From f990b32f2b1b852a7c12392ff7e572370dd76f74 Mon Sep 17 00:00:00 2001 From: Sambit Biswas Date: Sun, 27 Sep 2026 11:41:44 -0400 Subject: [PATCH 1/6] feat(dictation): Parakeet idle unload and warm-up, tap-or-hold mode, custom dictionary - Unload the on-device Parakeet worker after a configurable idle period (default 10 minutes, or never) and preload it on hotkey press or mic start. - Add a hybrid "Tap or hold" shortcut mode alongside hold and toggle. - Apply a user dictionary of words and replacements to every transcript. Co-Authored-By: Claude Opus 5.5 --- .papercuts/troubleshooting.md | 4 + docs/plans/README.md | 1 + docs/plans/dictation-parakeet-modes-plan.md | 52 +++++ main/handlers/local-voice.ts | 17 +- main/handlers/providers.ts | 11 +- main/services/config-store-core.test.ts | 33 ++++ main/services/dictation-coordinator.test.ts | 170 ++++++++++++++++ main/services/dictation-coordinator.ts | 99 +++++++++- main/services/dictation.ts | 21 ++ main/services/parakeet-engine.ts | 5 + main/services/parakeet-idle-unload.test.ts | 131 ++++++++++++ main/services/parakeet-idle-unload.ts | 92 +++++++++ main/services/parakeet-process-core.ts | 6 + main/services/parakeet-protocol.test.ts | 14 ++ main/services/parakeet-protocol.ts | 11 ++ main/services/parakeet-worker.ts | 11 +- main/services/parakeet.ts | 100 ++++++++-- main/services/portable-config-core.ts | 12 ++ main/services/types.ts | 6 + package.json | 2 +- renderer/components/composer.tsx | 10 +- .../environment-subagents-contract.test.ts | 14 +- renderer/components/onboarding-flow.tsx | 2 +- .../dictation-dictionary-settings.test.tsx | 89 +++++++++ .../dictation-dictionary-settings.tsx | 186 ++++++++++++++++++ .../settings/dictation-shortcut-settings.tsx | 36 +++- .../settings/local-voice-settings.tsx | 71 ++++++- .../components/settings/voice-settings.tsx | 2 + renderer/lib/ipc.ts | 2 + renderer/lib/types.ts | 3 + renderer/lib/use-voice-recorder.ts | 5 + renderer/shared/dictation-dictionary.test.ts | 95 +++++++++ renderer/shared/dictation-dictionary.ts | 121 ++++++++++++ renderer/shared/dictation-preferences.test.ts | 59 ++++++ renderer/shared/dictation-preferences.ts | 93 +++++++++ 35 files changed, 1549 insertions(+), 37 deletions(-) create mode 100644 docs/plans/dictation-parakeet-modes-plan.md create mode 100644 main/services/parakeet-idle-unload.test.ts create mode 100644 main/services/parakeet-idle-unload.ts create mode 100644 renderer/components/settings/dictation-dictionary-settings.test.tsx create mode 100644 renderer/components/settings/dictation-dictionary-settings.tsx create mode 100644 renderer/shared/dictation-dictionary.test.ts create mode 100644 renderer/shared/dictation-dictionary.ts create mode 100644 renderer/shared/dictation-preferences.test.ts create mode 100644 renderer/shared/dictation-preferences.ts diff --git a/.papercuts/troubleshooting.md b/.papercuts/troubleshooting.md index ceeb89458..0e329993e 100644 --- a/.papercuts/troubleshooting.md +++ b/.papercuts/troubleshooting.md @@ -1404,3 +1404,7 @@ because their native file-mutator test binary had not been built. Run ## 2026-09-26 PR #121 merge of #251 (Remote contract revision 14) - A PR that adds to the Remote contract has to renumber when main bumps `contractRevision`. The conflicts show up in 7 files: both fixtures, the TS/iOS/Android fixture assertions and the iOS fixture CodingKeys. After resolving, `cmp` the Android copy against the shared fixture. Plan docs that name the revision also go stale. + +## 2026-09-27 feature/dictation-parakeet-modes +- The worktree-isolation guard refuses Bash heredocs whose body contains backticks or template literals (e.g. `cat >> file < engineStatus()); + // Preload the recognizer when the composer mic starts. Best effort: a failed + // warm-up is reported by the transcription that follows, not here. + ipcMain.handle("localVoice:warm", async (_event, id: unknown) => { + const modelId = asString(id, "id"); + try { + await warmLocalVoice(modelId); + } catch { + // Ignored: transcription surfaces the actionable error. + } + }); // ── Model management ───────────────────────────────────────────────── ipcMain.handle("localModels:list", async () => listModels()); diff --git a/main/handlers/providers.ts b/main/handlers/providers.ts index 90a279964..b8cbf927c 100644 --- a/main/handlers/providers.ts +++ b/main/handlers/providers.ts @@ -68,6 +68,8 @@ import { import { isGenerationThinkingLevel } from "../../renderer/shared/generation-thinking.js"; import { isGeminiUsageScope } from "../../renderer/shared/gemini-usage-scope.js"; import { isGeminiTranscriptionModel } from "../../renderer/shared/voice-models.js"; +import { parseDictationDictionary } from "../../renderer/shared/dictation-dictionary.js"; +import { parseDictationPreferencePatch } from "../../renderer/shared/dictation-preferences.js"; const appearancePreview = new AppearancePreviewState(); @@ -518,8 +520,9 @@ export function registerProviderHandlers(): void { if (typeof p.shortcutEnabled === "boolean") next.shortcutEnabled = p.shortcutEnabled; if (typeof p.shortcutAccelerator === "string") next.shortcutAccelerator = p.shortcutAccelerator; if (typeof p.dictationEnabled === "boolean") next.dictationEnabled = p.dictationEnabled; - if (typeof p.dictationHoldToTalk === "boolean") - next.dictationHoldToTalk = p.dictationHoldToTalk; + Object.assign(next, parseDictationPreferencePatch(p)); + if (p.dictationDictionary !== undefined) + next.dictationDictionary = parseDictationDictionary(p.dictationDictionary); if (typeof p.dictationSilenceStop === "boolean") next.dictationSilenceStop = p.dictationSilenceStop; if (typeof p.dictationCleanup === "boolean") next.dictationCleanup = p.dictationCleanup; @@ -548,6 +551,10 @@ export function registerProviderHandlers(): void { const saved = process.platform === "linux" && next.dictationHoldToTalk !== undefined ? await linuxHoldSettings.apply(next.dictationHoldToTalk, (isCurrent) => configStore.setSettings(next, isCurrent)) : await configStore.setSettings(next); + if (next.localVoiceIdleUnloadMinutes !== undefined) { + const { reconfigureParakeetIdleUnload } = await import("../services/parakeet.js"); + void reconfigureParakeetIdleUnload(); + } if (next.skillsEnabled !== undefined) { skillRegistry.invalidate(); invalidateBotRuntimeInventoryAuthority("skill_configuration"); diff --git a/main/services/config-store-core.test.ts b/main/services/config-store-core.test.ts index 4b65b1239..9c8e23621 100644 --- a/main/services/config-store-core.test.ts +++ b/main/services/config-store-core.test.ts @@ -559,6 +559,39 @@ test("reads and writes survive a restart of the whole store", async (t) => { }); }); +test("hand-edited dictation preferences reach consumers only in supported shapes", async (t) => { + const h = await harness(t); + await h.store.setSettings({ exaEnabled: true }); + const file = await readJson>(h.settingsFile); + const settings = (file.settings ?? file) as Record; + Object.assign(settings, { + dictationActivationMode: "double-tap", + localVoiceIdleUnloadMinutes: -5, + dictationDictionary: [ + { from: " aiden ", to: "Aiden" }, + { from: 42, to: "nope" }, + { from: "AIDEN", to: "duplicate" }, + { from: "pie", to: "Pi" }, + ], + }); + await fs.writeFile(h.settingsFile, JSON.stringify(file, null, 2), "utf-8"); + + const restarted = createConfigStore( + createPortableConfigStores( + () => path.dirname(h.portableFile), + () => path.dirname(h.localFile), + ), + fakeSecrets().port, + ); + const runtime = await restarted.getSettings(); + assert.equal(runtime.dictationActivationMode, undefined); + assert.equal(runtime.localVoiceIdleUnloadMinutes, undefined); + assert.deepEqual(runtime.dictationDictionary, [ + { from: "aiden", to: "Aiden" }, + { from: "pie", to: "Pi" }, + ]); +}); + test("every install ends up with at least one workspace", async (t) => { const h = await harness(t); const workspaces = await h.store.listWorkspaces(); diff --git a/main/services/dictation-coordinator.test.ts b/main/services/dictation-coordinator.test.ts index 87d58c32b..466c5451f 100644 --- a/main/services/dictation-coordinator.test.ts +++ b/main/services/dictation-coordinator.test.ts @@ -7,6 +7,7 @@ import { TRANSCRIPTION_WATCHDOG_MS, type DictationCoordinatorDeps, } from "./dictation-coordinator.js"; +import { HYBRID_TAP_THRESHOLD_MS } from "../../renderer/shared/dictation-preferences.js"; function deferred() { let resolve!: (value: T) => void; @@ -404,3 +405,172 @@ test("desktop session failure during startup stops at first recorder readiness", await subject.coordinator.press(); fail(); await subject.coordinator.ready(); assert.equal(subject.coordinator.currentStage, "transcribing"); subject.coordinator.dispose(); }); + +function hybridHarness(overrides: Partial = {}) { + let clock = 10_000; + let grace: (() => void) | undefined; + const watches: Array<{ onRelease: () => void; stopped: boolean }> = []; + const subject = harness({ + now: () => clock, + getActivationMode: () => "hybrid", + getHoldKeyCode: () => 2, + startHoldWatch: (_keyCode, onRelease) => { + const watch = { onRelease, stopped: false }; + watches.push(watch); + return () => { + watch.stopped = true; + }; + }, + setTimer: (callback, delayMs) => { + if (delayMs === HOLD_RELEASE_GRACE_MS) grace = callback; + return dormantTimer(); + }, + ...overrides, + }); + return { + ...subject, + advance: (ms: number) => { + clock += ms; + }, + watches, + fireGrace: async () => { + assert.ok(grace, "release grace was scheduled"); + grace(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }, + }; +} + +test("hybrid tap latches recording on until the next press", async () => { + const subject = hybridHarness(); + await subject.coordinator.ready(); + await subject.coordinator.press(); + assert.equal(subject.watches.length, 1, "hybrid watches the key from the first press"); + subject.advance(HYBRID_TAP_THRESHOLD_MS - 50); + await subject.coordinator.release(); + assert.equal(subject.coordinator.currentStage, "recording", "a tap does not stop capture"); + assert.equal(subject.watches[0]?.stopped, true); + subject.advance(4_000); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "transcribing"); + assert.deepEqual( + subject.events.map((event) => event.state), + ["recording", "stopping"], + ); +}); + +test("hybrid hold behaves as push-to-talk and ignores repeats while held", async () => { + const subject = hybridHarness(); + await subject.coordinator.ready(); + await subject.coordinator.press(); + subject.advance(100); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "recording", "a repeat while held is ignored"); + subject.advance(HYBRID_TAP_THRESHOLD_MS + 500); + await subject.coordinator.release(); + await subject.fireGrace(); + assert.equal(subject.coordinator.currentStage, "transcribing"); +}); + +test("hybrid measures the tap from key-down even when the pill starts slowly", async () => { + const shown = deferred(); + const subject = hybridHarness({ showPill: () => shown.promise }); + const pressed = subject.coordinator.press(); + await new Promise((resolve) => setImmediate(resolve)); + subject.advance(120); + subject.watches[0]?.onRelease(); + subject.advance(900); + shown.resolve(true); + await pressed; + await subject.coordinator.ready(); + assert.equal(subject.coordinator.currentStage, "recording", "the quick tap latched on"); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "transcribing"); +}); + +test("hybrid long hold released during cold startup stops at recorder readiness", async () => { + const shown = deferred(); + const subject = hybridHarness({ showPill: () => shown.promise }); + const pressed = subject.coordinator.press(); + await new Promise((resolve) => setImmediate(resolve)); + subject.advance(HYBRID_TAP_THRESHOLD_MS + 200); + subject.watches[0]?.onRelease(); + shown.resolve(true); + await pressed; + await subject.coordinator.ready(); + assert.equal(subject.coordinator.currentStage, "transcribing"); +}); + +test("hybrid without a key watch degrades to toggle", async () => { + const subject = hybridHarness({ startHoldWatch: () => null }); + await subject.coordinator.ready(); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "recording"); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "transcribing"); +}); + +test("toggle mode ignores key-up events entirely", async () => { + const subject = harness({ getActivationMode: () => "toggle", startHoldWatch: () => () => {} }); + await subject.coordinator.ready(); + await subject.coordinator.press(); + await subject.coordinator.release(); + assert.equal(subject.coordinator.currentStage, "recording"); + assert.equal(subject.coordinator.currentMode, "toggle"); +}); + +test("each recording starts a best-effort model warm-up that cannot block capture", async () => { + let warmups = 0; + const logged: string[] = []; + const subject = harness({ + warmUp: async () => { + warmups += 1; + throw new Error("model missing"); + }, + logError: (message) => logged.push(message), + }); + await subject.coordinator.ready(); + await subject.coordinator.press(); + assert.equal(subject.coordinator.currentStage, "recording"); + await subject.coordinator.press(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(warmups, 1, "stopping does not warm again"); + assert.equal(logged.length, 1); +}); + +test("the dictionary corrects the delivered transcript after cleanup", async () => { + const pasted: string[] = []; + const subject = harness({ + shouldCleanup: () => true, + cleanupTranscript: async (text) => `${text}.`, + applyDictionary: (text) => text.replace(/\baiden\b/giu, "Aiden"), + paste: async (text) => { + pasted.push(text); + return "pasted"; + }, + }); + await subject.coordinator.ready(); + await subject.coordinator.press(); + await subject.coordinator.press(); + await subject.coordinator.result("ask aiden", subject.coordinator.currentOperationId); + assert.deepEqual(pasted, ["ask Aiden."]); +}); + +test("a failing dictionary never loses the transcript", async () => { + const pasted: string[] = []; + const subject = harness({ + applyDictionary: () => { + throw new Error("bad rules"); + }, + paste: async (text) => { + pasted.push(text); + return "pasted"; + }, + }); + await subject.coordinator.ready(); + await subject.coordinator.press(); + await subject.coordinator.press(); + await subject.coordinator.result("hello there", subject.coordinator.currentOperationId); + assert.deepEqual(pasted, ["hello there"]); +}); diff --git a/main/services/dictation-coordinator.ts b/main/services/dictation-coordinator.ts index fa3b5ea8e..970db0652 100644 --- a/main/services/dictation-coordinator.ts +++ b/main/services/dictation-coordinator.ts @@ -1,4 +1,8 @@ import type { DictationProgress, DictationStatePayload } from "../../renderer/shared/dictation.js"; +import { + HYBRID_TAP_THRESHOLD_MS, + type DictationActivationMode, +} from "../../renderer/shared/dictation-preferences.js"; import type { PasteDeliveryResult, PasteOutcome } from "./dictation-paste.js"; export type DictationStage = "idle" | "starting" | "recording" | "transcribing" | "delivering"; @@ -15,6 +19,12 @@ export interface DictationCoordinatorDeps { logError: (message: string, error: unknown) => void; now?: () => number; isHoldToTalk?: () => boolean | Promise; + /** Preferred over isHoldToTalk; `hybrid` treats a quick tap as toggle and a hold as push-to-talk. */ + getActivationMode?: () => DictationActivationMode | Promise; + /** Best-effort preload of the transcription model when a recording starts. */ + warmUp?: () => void | Promise; + /** User dictionary applied to the final transcript before delivery. */ + applyDictionary?: (text: string) => string | Promise; /** Optional polish after STT; must return the original text on failure. */ cleanupTranscript?: (text: string) => Promise; shouldCleanup?: () => boolean | Promise; @@ -49,7 +59,10 @@ export class DictationCoordinator { private watchdogTimer: NodeJS.Timeout | null = null; private queue: Promise = Promise.resolve(); private disposed = false; + /** Whether releasing the shortcut stops this operation (hold, or hybrid until a tap latches it). */ private holdToTalk = false; + private mode: DictationActivationMode = "toggle"; + private pressedAt = 0; private holdKeyCode: number | null = null; private holdWatchActive = false; private pendingRelease = false; @@ -67,6 +80,29 @@ export class DictationCoordinator { return this.operationId; } + /** Activation mode frozen for the current (or most recent) operation. */ + get currentMode(): DictationActivationMode { + return this.mode; + } + + private now(): number { + return (this.deps.now ?? Date.now)(); + } + + /** Hybrid only: the shortcut stopped being watched, so behave as a latched toggle. */ + private latchHybridToggle(): boolean { + if (this.mode !== "hybrid") return false; + this.holdToTalk = false; + if (this.stage === "recording") { + this.deps.broadcast({ + state: "recording", + operationId: this.operationId ?? undefined, + message: "Press the shortcut again to stop.", + }); + } + return true; + } + private enqueue(operation: () => Promise | void): Promise { const pending = this.queue.then(async () => { if (!this.disposed) await operation(); @@ -149,6 +185,7 @@ export class DictationCoordinator { () => { this.holdWatchActive = false; this.stopHoldWatch = null; + if (this.latchHybridToggle()) return; this.deps.broadcast({ state: "recording", operationId: this.operationId ?? undefined, @@ -157,6 +194,7 @@ export class DictationCoordinator { }, ); if (typeof stop !== "function") { + if (this.latchHybridToggle()) return; this.deps.broadcast({ state: "recording", operationId: this.operationId ?? undefined, @@ -169,6 +207,7 @@ export class DictationCoordinator { } catch (error) { this.holdWatchActive = false; this.deps.logError("Could not watch the dictation shortcut for release.", error); + if (this.latchHybridToggle()) return; this.deps.broadcast({ state: "recording", operationId: this.operationId ?? undefined, @@ -177,6 +216,21 @@ export class DictationCoordinator { } } + private watchesFromPress(): boolean { + return Boolean(this.deps.startReleaseWatch) || this.mode === "hybrid"; + } + + private startWarmUp(): void { + if (!this.deps.warmUp) return; + try { + void Promise.resolve(this.deps.warmUp()).catch((error: unknown) => { + this.deps.logError("Could not warm up the transcription model.", error); + }); + } catch (error) { + this.deps.logError("Could not warm up the transcription model.", error); + } + } + private scheduleHide(delayMs: number): void { this.clearHideTimer(); this.hideTimer = this.deps.setTimer(() => { @@ -186,7 +240,13 @@ export class DictationCoordinator { } private async refreshHoldMode(): Promise { - this.holdToTalk = (await this.deps.isHoldToTalk?.()) === true; + if (this.deps.getActivationMode) { + const mode = await this.deps.getActivationMode(); + this.mode = mode === "hold" || mode === "hybrid" ? mode : "toggle"; + } else { + this.mode = (await this.deps.isHoldToTalk?.()) === true ? "hold" : "toggle"; + } + this.holdToTalk = this.mode !== "toggle"; const code = await this.deps.getHoldKeyCode?.(); this.holdKeyCode = typeof code === "number" ? code : null; } @@ -201,6 +261,9 @@ export class DictationCoordinator { /** Hotkey press. Toggle mode starts/stops; hold mode starts, ignores down-repeats, and stops once release is in flight. */ press(): Promise { + // Stamp the physical press now, not when the queue reaches it, so a + // hybrid tap is measured from the user's key-down. + const pressedAt = this.now(); return this.enqueue(async () => { this.clearHideTimer(); this.clearReleaseTimer(); @@ -210,9 +273,13 @@ export class DictationCoordinator { await this.refreshHoldMode(); this.stage = "starting"; this.pendingRelease = false; + this.pressedAt = pressedAt; this.operationSequence += 1; - this.operationId = `${(this.deps.now ?? Date.now)()}-${this.operationSequence}`; - if (this.deps.startReleaseWatch) this.beginHoldWatch(); + this.operationId = `${this.now()}-${this.operationSequence}`; + this.startWarmUp(); + // Hybrid must observe the key-up from the start to tell a tap from a + // hold; plain hold keeps watching only once capture is live. + if (this.watchesFromPress()) this.beginHoldWatch(); try { const created = await this.deps.showPill(); if (created) this.pillReady = false; @@ -226,12 +293,15 @@ export class DictationCoordinator { if (this.stage === "starting" && this.pillReady) { this.stage = "recording"; this.deps.broadcast({ state: "recording", operationId: this.operationId ?? undefined }); - if (!this.deps.startReleaseWatch) this.beginHoldWatch(); + if (!this.watchesFromPress()) this.beginHoldWatch(); if (this.pendingRelease) this.stopIfRecording(); } return; } if (this.stage === "starting") { + // Hybrid watches the key from the first press, so a press while it is + // still held is an OS repeat, not a stop request. + if (this.mode === "hybrid" && this.holdToTalk && this.holdWatchActive) return; // A toggle-mode second press and a hold-mode release can arrive while // permission/settings/microphone startup is still in flight. Latch it // so the first recorder frame cannot outlive the user's stop action. @@ -265,9 +335,20 @@ export class DictationCoordinator { /** Hold-to-talk key-up, with a short grace so OS repeats do not cut capture. */ release(expectedOperationId?: string | null): Promise { + const releasedAt = this.now(); return this.enqueue(async () => { if (expectedOperationId !== undefined && expectedOperationId !== this.operationId) return; if (!this.holdToTalk) return; + if ( + this.mode === "hybrid" && + (this.stage === "starting" || this.stage === "recording") && + releasedAt - this.pressedAt < HYBRID_TAP_THRESHOLD_MS + ) { + // A tap: keep recording until the next press, exactly like toggle. + this.endHoldWatch(); + this.holdToTalk = false; + return; + } if (this.stage === "starting") { this.pendingRelease = true; return; @@ -301,7 +382,7 @@ export class DictationCoordinator { if (this.stage === "starting") { this.stage = "recording"; this.deps.broadcast({ state: "recording", operationId: this.operationId ?? undefined }); - if (!this.deps.startReleaseWatch) this.beginHoldWatch(); + if (!this.watchesFromPress()) this.beginHoldWatch(); if (this.pendingRelease) this.stopIfRecording(); } }); @@ -352,6 +433,14 @@ export class DictationCoordinator { } catch (error) { this.deps.logError("Dictation cleanup failed; using the original transcript.", error); } + if (this.deps.applyDictionary) { + try { + const corrected = (await this.deps.applyDictionary(transcript)).trim(); + if (corrected) transcript = corrected.slice(0, MAX_TRANSCRIPT_LENGTH); + } catch (error) { + this.deps.logError("Dictation dictionary failed; using the unmodified transcript.", error); + } + } const pasteResult = await this.deps.paste(transcript); const outcome = typeof pasteResult === "string" ? pasteResult : pasteResult.outcome; this.stage = "idle"; diff --git a/main/services/dictation.ts b/main/services/dictation.ts index 2160392c2..3490e23f8 100644 --- a/main/services/dictation.ts +++ b/main/services/dictation.ts @@ -14,6 +14,8 @@ import { acceleratorPrimaryMacKeyCode } from "./dictation-keycode.js"; import { dictationPlatformBehavior } from "./dictation-platform.js"; import { pasteTranscript, runAtomicMacPaste, type PasteDeps } from "./dictation-paste.js"; import { DictationCoordinator } from "./dictation-coordinator.js"; +import { applyDictationDictionary, parseDictationDictionary } from "../../renderer/shared/dictation-dictionary.js"; +import { resolveDictationActivationMode } from "../../renderer/shared/dictation-preferences.js"; import { activeLinuxDictationHoldShortcut, initLinuxDictationSessionLost, subscribeLinuxDictationRelease } from "./shortcut.js"; @@ -60,6 +62,25 @@ const coordinator = new DictationCoordinator({ activeLinuxDictationHoldShortcut() || (dictationPlatformBehavior().holdToTalk && (await configStore.getSettings()).dictationHoldToTalk === true), + getActivationMode: async () => { + const settings = await configStore.getSettings(); + // Linux reports releases only while the portal bound a hold shortcut; macOS + // watches the physical key when the platform supports it. + const linuxHold = activeLinuxDictationHoldShortcut(); + const releaseCapable = linuxHold || dictationPlatformBehavior().holdToTalk; + return resolveDictationActivationMode( + linuxHold ? { ...settings, dictationHoldToTalk: true } : settings, + releaseCapable, + ); + }, + warmUp: async () => { + const settings = await configStore.getSettings(); + if (settings.voiceProvider !== "local" || !settings.localVoiceModel) return; + const { warmLocalVoice } = await import("./parakeet.js"); + await warmLocalVoice(settings.localVoiceModel); + }, + applyDictionary: async (text) => + applyDictationDictionary(text, parseDictationDictionary((await configStore.getSettings()).dictationDictionary)), shouldCleanup: async () => (await configStore.getSettings()).dictationCleanup === true, cleanupTranscript: cleanupDictationTranscript, ...(process.platform === "linux" ? { startReleaseWatch: subscribeLinuxDictationRelease } : {}), diff --git a/main/services/parakeet-engine.ts b/main/services/parakeet-engine.ts index 8c0a35e6e..8565b16a5 100644 --- a/main/services/parakeet-engine.ts +++ b/main/services/parakeet-engine.ts @@ -71,6 +71,11 @@ function getRecognizer(modelId: string, modelDirectory: string): OfflineRecogniz return recognizer; } +/** Load (or reuse) the recognizer so the first transcription skips model load. */ +export function warmRecognizer(modelId: string, modelDirectory: string): void { + getRecognizer(modelId, modelDirectory); +} + export function releaseRecognizer(modelId: string): void { recognizers.delete(modelId); } diff --git a/main/services/parakeet-idle-unload.test.ts b/main/services/parakeet-idle-unload.test.ts new file mode 100644 index 000000000..c348c864c --- /dev/null +++ b/main/services/parakeet-idle-unload.test.ts @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ParakeetIdleUnloader } from "./parakeet-idle-unload.js"; + +interface FakeTimer { + callback: () => void; + delayMs: number; + cleared: boolean; +} + +function harness(initialIdleMs: number | null | (() => Promise) = 60_000) { + const timers: FakeTimer[] = []; + let unloads = 0; + let idle = initialIdleMs; + const unloader = new ParakeetIdleUnloader({ + setTimer: (callback, delayMs) => { + const timer = { callback, delayMs, cleared: false }; + timers.push(timer); + return timer; + }, + clearTimer: (timer) => { + timer.cleared = true; + }, + idleMs: () => (typeof idle === "function" ? idle() : idle), + unload: () => { + unloads += 1; + }, + }); + const live = () => timers.filter((timer) => !timer.cleared); + return { + unloader, + live, + unloads: () => unloads, + setIdle: (value: number | null) => { + idle = value; + }, + /** Fire every armed timer, as if the idle period elapsed. */ + elapse: () => { + for (const timer of live()) { + timer.cleared = true; + timer.callback(); + } + }, + }; +} + +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +test("the model unloads once after the idle period that follows its last use", async () => { + const h = harness(5 * 60_000); + const end = h.unloader.begin(); + assert.equal(h.live().length, 0, "no countdown while transcription is in flight"); + end(); + await settle(); + assert.deepEqual( + h.live().map((timer) => timer.delayMs), + [5 * 60_000], + ); + h.elapse(); + assert.equal(h.unloads(), 1); + await h.unloader.reconfigure(); + assert.equal(h.live().length, 0, "nothing left to unload until the model is used again"); +}); + +test("new use cancels a pending unload and restarts the countdown afterwards", async () => { + const h = harness(60_000); + h.unloader.begin()(); + await settle(); + const first = h.live()[0]; + const end = h.unloader.begin(); + assert.equal(first?.cleared, true); + end(); + await settle(); + assert.equal(h.live().length, 1); + h.elapse(); + assert.equal(h.unloads(), 1); +}); + +test("overlapping work keeps the model loaded until the last lease ends", async () => { + const h = harness(60_000); + const warm = h.unloader.begin(); + const transcribe = h.unloader.begin(); + warm(); + await settle(); + assert.equal(h.live().length, 0); + warm(); + await settle(); + assert.equal(h.live().length, 0, "ending a lease twice does not release another caller"); + transcribe(); + await settle(); + assert.equal(h.live().length, 1); +}); + +test("a lease that starts while the idle period is being read wins", async () => { + let resolveIdle!: (value: number) => void; + const h = harness(() => new Promise((resolve) => (resolveIdle = resolve))); + h.unloader.begin()(); + const next = h.unloader.begin(); + resolveIdle(1_000); + await settle(); + assert.equal(h.live().length, 0, "the stale read cannot arm an unload under active work"); + next(); +}); + +test("'never' keeps the model loaded and a Settings change re-arms the countdown", async () => { + const h = harness(null); + h.unloader.begin()(); + await settle(); + assert.equal(h.live().length, 0); + h.setIdle(2 * 60_000); + await h.unloader.reconfigure(); + assert.deepEqual( + h.live().map((timer) => timer.delayMs), + [2 * 60_000], + ); + h.setIdle(null); + await h.unloader.reconfigure(); + assert.equal(h.live().length, 0); + assert.equal(h.unloads(), 0); +}); + +test("an externally released model is not unloaded again", async () => { + const h = harness(60_000); + h.unloader.begin()(); + await settle(); + h.unloader.forget(); + assert.equal(h.live().length, 0); + await h.unloader.reconfigure(); + assert.equal(h.live().length, 0); + assert.equal(h.unloads(), 0); +}); diff --git a/main/services/parakeet-idle-unload.ts b/main/services/parakeet-idle-unload.ts new file mode 100644 index 000000000..1b8c14e7f --- /dev/null +++ b/main/services/parakeet-idle-unload.ts @@ -0,0 +1,92 @@ +// Idle unload policy for the on-device Parakeet recognizer. Electron-free so +// the timer/in-flight rules are testable without a utility process. + +export interface ParakeetIdleUnloaderDeps { + setTimer: (callback: () => void, delayMs: number) => Timer; + clearTimer: (timer: Timer) => void; + /** Current idle period, or null to keep the model loaded indefinitely. */ + idleMs: () => number | null | Promise; + /** Free the recognizer (terminate the worker process). */ + unload: () => void; + logError?: (message: string, error: unknown) => void; +} + +/** + * Tracks model use and unloads after a quiet period. Work in flight (warm-up + * or transcription) always holds the model; the countdown starts only after + * the last lease ends, and any new lease cancels a pending unload — even one + * whose idle period is still being read. + */ +export class ParakeetIdleUnloader { + private active = 0; + private loaded = false; + private timer: Timer | null = null; + private revision = 0; + + constructor(private readonly deps: ParakeetIdleUnloaderDeps) {} + + get inFlight(): number { + return this.active; + } + + get armed(): boolean { + return this.timer !== null; + } + + private cancelTimer(): void { + this.revision += 1; + if (this.timer === null) return; + this.deps.clearTimer(this.timer); + this.timer = null; + } + + /** Mark the model busy. Call the returned function exactly once when done. */ + begin(): () => void { + this.cancelTimer(); + this.active += 1; + this.loaded = true; + let ended = false; + return () => { + if (ended) return; + ended = true; + this.active = Math.max(0, this.active - 1); + if (this.active === 0) void this.arm(); + }; + } + + /** Re-read the idle period (e.g. after a Settings change) and restart the countdown. */ + reconfigure(): Promise { + if (this.active > 0 || !this.loaded) return Promise.resolve(); + return this.arm(); + } + + /** The model was released by other means (process exit, app shutdown, model delete). */ + forget(): void { + this.cancelTimer(); + this.loaded = false; + } + + private async arm(): Promise { + this.cancelTimer(); + const revision = this.revision; + let delay: number | null; + try { + delay = await this.deps.idleMs(); + } catch (error) { + this.deps.logError?.("Could not read the on-device voice idle period.", error); + return; + } + if (revision !== this.revision || this.active > 0 || !this.loaded) return; + if (delay === null || !Number.isFinite(delay) || delay < 0) return; + this.timer = this.deps.setTimer(() => { + this.timer = null; + if (this.active > 0 || !this.loaded) return; + this.loaded = false; + try { + this.deps.unload(); + } catch (error) { + this.deps.logError?.("Could not unload the idle on-device voice model.", error); + } + }, delay); + } +} diff --git a/main/services/parakeet-process-core.ts b/main/services/parakeet-process-core.ts index 01b702a5a..ddb48f28b 100644 --- a/main/services/parakeet-process-core.ts +++ b/main/services/parakeet-process-core.ts @@ -49,6 +49,7 @@ export class ParakeetProcessClient { message: | { kind: "status" } | { kind: "release"; modelId: string } + | { kind: "warm"; modelId: string; modelDirectory: string } | { kind: "transcribe"; modelId: string; @@ -112,6 +113,11 @@ export class ParakeetProcessClient { return result.text ?? ""; } + async warm(modelId: string, modelDirectory: string): Promise { + const result = await this.request({ kind: "warm", modelId, modelDirectory }); + if (result.kind === "failure") throw new Error(result.message); + } + async release(modelId: string): Promise { const result = await this.request({ kind: "release", modelId }); if (result.kind === "failure") throw new Error(result.message); diff --git a/main/services/parakeet-protocol.test.ts b/main/services/parakeet-protocol.test.ts index 23171317c..f61cdf173 100644 --- a/main/services/parakeet-protocol.test.ts +++ b/main/services/parakeet-protocol.test.ts @@ -67,3 +67,17 @@ test("parakeet protocol accepts only versioned request and result frames", () => true, ); }); + +test("parakeet warm requests must name the model directory to preload", () => { + const warm = { + version: PARAKEET_PROTOCOL_VERSION, + kind: "warm", + requestId: "w1", + modelId: "parakeet-v3", + modelDirectory: "/tmp/model", + }; + assert.equal(isParakeetParentMessage(warm), true); + const { modelDirectory: _omitted, ...withoutDirectory } = warm; + assert.equal(isParakeetParentMessage(withoutDirectory), false); + assert.equal(isParakeetParentMessage({ ...warm, version: 2 }), false); +}); diff --git a/main/services/parakeet-protocol.ts b/main/services/parakeet-protocol.ts index 42221c4cc..44a7629a3 100644 --- a/main/services/parakeet-protocol.ts +++ b/main/services/parakeet-protocol.ts @@ -20,6 +20,14 @@ export type ParakeetParentMessage = kind: "release"; requestId: string; modelId: string; + } + | { + version: typeof PARAKEET_PROTOCOL_VERSION; + /** Load the recognizer ahead of the first transcription. */ + kind: "warm"; + requestId: string; + modelId: string; + modelDirectory: string; }; export type ParakeetWorkerMessage = @@ -47,6 +55,9 @@ export function isParakeetParentMessage(value: unknown): value is ParakeetParent if (typeof value.requestId !== "string" || value.requestId.length === 0) return false; if (value.kind === "status") return true; if (value.kind === "release") return typeof value.modelId === "string"; + if (value.kind === "warm") { + return typeof value.modelId === "string" && typeof value.modelDirectory === "string"; + } return ( value.kind === "transcribe" && typeof value.modelId === "string" && diff --git a/main/services/parakeet-worker.ts b/main/services/parakeet-worker.ts index d726ce624..9f4671ea4 100644 --- a/main/services/parakeet-worker.ts +++ b/main/services/parakeet-worker.ts @@ -1,6 +1,6 @@ import { pcmToFloat32 } from "../handlers/voice-codec.js"; import { decodeAidenRemotePcm16 } from "./aiden-remote-speech-codec.js"; -import { engineStatus, releaseRecognizer, transcribePcm } from "./parakeet-engine.js"; +import { engineStatus, releaseRecognizer, transcribePcm, warmRecognizer } from "./parakeet-engine.js"; import { isParakeetParentMessage, PARAKEET_PROTOCOL_VERSION, @@ -35,6 +35,15 @@ parentPort.on("message", (event) => { }); return; } + if (message.kind === "warm") { + warmRecognizer(message.modelId, message.modelDirectory); + post({ + version: PARAKEET_PROTOCOL_VERSION, + kind: "result", + requestId: message.requestId, + }); + return; + } if (message.kind === "release") { releaseRecognizer(message.modelId); post({ diff --git a/main/services/parakeet.ts b/main/services/parakeet.ts index 1d2193ad4..8d2cd47cf 100644 --- a/main/services/parakeet.ts +++ b/main/services/parakeet.ts @@ -10,15 +10,57 @@ import { engineStatus as engineStatusInProcess, releaseRecognizer as releaseRecognizerInProcess, transcribePcm as transcribePcmInProcess, + warmRecognizer as warmRecognizerInProcess, } from "./parakeet-engine.js"; +import { ParakeetIdleUnloader } from "./parakeet-idle-unload.js"; import { ParakeetProcessClient } from "./parakeet-process-core.js"; import { ParakeetTranscriptionLane } from "./parakeet-transcription-lane.js"; +import { localVoiceIdleUnloadMs } from "../../renderer/shared/dictation-preferences.js"; let client: ParakeetProcessClient | null = null; let child: UtilityProcess | null = null; let launching: Promise | null = null; let processGeneration = 0; const transcriptionLane = new ParakeetTranscriptionLane(); +/** Models loaded by the in-process fallback (tests / non-Electron hosts). */ +const inProcessModels = new Set(); + +// Terminating the utility process is the only way to return the native +// sherpa-onnx allocations to the OS, so idle unload disposes the worker. +const idleUnloader = new ParakeetIdleUnloader({ + setTimer: (callback, delayMs) => { + const timer = setTimeout(callback, delayMs); + timer.unref?.(); + return timer; + }, + clearTimer: (timer) => clearTimeout(timer), + idleMs: async () => { + const { configStore } = await import("./config-store.js"); + return localVoiceIdleUnloadMs((await configStore.getSettings()).localVoiceIdleUnloadMinutes); + }, + unload: () => unloadIdleModel(), +}); + +function unloadIdleModel(): void { + processGeneration += 1; + const current = client; + client = null; + launching = null; + current?.dispose(); + child = null; + for (const modelId of inProcessModels) releaseRecognizerInProcess(modelId); + inProcessModels.clear(); +} + +/** Hold the model loaded for the duration of `operation`, then restart the idle countdown. */ +async function withModelLease(operation: () => Promise): Promise { + const end = idleUnloader.begin(); + try { + return await operation(); + } finally { + end(); + } +} function attachUtilityProcess(processHandle: UtilityProcess): ParakeetProcessClient { return new ParakeetProcessClient({ @@ -60,7 +102,10 @@ async function launchClient(generation: number): Promise } launched.on("exit", () => { if (child === launched) child = null; - if (client === created) client = null; + if (client === created) { + client = null; + if (idleUnloader.inFlight === 0) idleUnloader.forget(); + } }); child = launched; client = created; @@ -94,12 +139,41 @@ function disposeClientIfCurrent(expected: ParakeetProcessClient): void { } export async function engineStatus(): Promise<{ ready: boolean; error: string | null }> { - try { - return await (await getClient()).status(); - } catch (error) { - if (isolationUnavailable(error)) return engineStatusInProcess(); - return { ready: false, error: error instanceof Error ? error.message : String(error) }; - } + return withModelLease(async () => { + try { + return await (await getClient()).status(); + } catch (error) { + if (isolationUnavailable(error)) return engineStatusInProcess(); + return { ready: false, error: error instanceof Error ? error.message : String(error) }; + } + }); +} + +/** + * Preload the recognizer so the first transcription after a hotkey press or + * microphone start does not pay the model-load cost. Best effort: a missing + * model or failed load is reported to the caller, which should ignore it — + * the real transcription reports the actionable error. + */ +export async function warmLocalVoice(modelId: string): Promise { + const directory = modelDir(modelId); + if (!directory || !isModelInstalled(modelId)) return; + await transcriptionLane.run(() => + withModelLease(async () => { + try { + await (await getClient()).warm(modelId, directory); + } catch (error) { + if (!isolationUnavailable(error)) throw error; + warmRecognizerInProcess(modelId, directory); + inProcessModels.add(modelId); + } + }), + ); +} + +/** Settings changed the idle period: restart the countdown with the new value. */ +export function reconfigureParakeetIdleUnload(): Promise { + return idleUnloader.reconfigure(); } export async function releaseRecognizer(modelId: string): Promise { @@ -108,6 +182,7 @@ export async function releaseRecognizer(modelId: string): Promise { } catch (error) { if (isolationUnavailable(error)) { releaseRecognizerInProcess(modelId); + inProcessModels.delete(modelId); return; } throw error; @@ -125,7 +200,7 @@ export async function transcribePcmBase64( } let activeClient: ParakeetProcessClient | null = null; return transcriptionLane.run( - async () => { + () => withModelLease(async () => { try { activeClient = await getClient(); if (signal?.aborted) { @@ -141,11 +216,12 @@ export async function transcribePcmBase64( } catch (error) { if (isolationUnavailable(error)) { signal?.throwIfAborted(); + inProcessModels.add(modelId); return transcribePcmInProcess(pcmToFloat32(pcmBase64), modelId, directory); } throw error; } - }, + }), { signal, onCancelActive: () => { @@ -160,7 +236,7 @@ export async function transcribePcm16Base64(pcmBase64: string, modelId: string): if (!directory || !isModelInstalled(modelId)) { throw new Error("The selected voice model isn't downloaded. Download it in Settings → Voice."); } - return transcriptionLane.run(async () => { + return transcriptionLane.run(() => withModelLease(async () => { try { return await ( await getClient() @@ -172,14 +248,16 @@ export async function transcribePcm16Base64(pcmBase64: string, modelId: string): }); } catch (error) { if (isolationUnavailable(error)) { + inProcessModels.add(modelId); return transcribePcmInProcess(decodeAidenRemotePcm16(pcmBase64), modelId, directory); } throw error; } - }); + })); } export function disposeParakeet(): void { + idleUnloader.forget(); processGeneration += 1; const current = client; client = null; diff --git a/main/services/portable-config-core.ts b/main/services/portable-config-core.ts index 4e3c3b39a..b2221c635 100644 --- a/main/services/portable-config-core.ts +++ b/main/services/portable-config-core.ts @@ -1,5 +1,7 @@ import { parseCustomModelOptions } from "../../renderer/shared/custom-model-options.js"; import { compactionEngineFrom } from "../../renderer/shared/compaction.js"; +import { parseDictationDictionary } from "../../renderer/shared/dictation-dictionary.js"; +import { isLocalVoiceIdleUnloadMinutes } from "../../renderer/shared/dictation-preferences.js"; // Splits Aiden's persisted configuration into a portable half and a // machine-local half, and migrates existing installs into that layout once. // @@ -699,6 +701,16 @@ export function runtimeSettingsFrom(settings: AppSettings): AppSettings { retainKnownValue("telegramActivity", ["quiet", "thinking", "tools", "verbose"]); retainKnownValue("telegramRendering", ["rich", "html"]); retainKnownValue("telegramVoiceMode", ["hidden", "mirror", "always"]); + retainKnownValue("dictationActivationMode", ["toggle", "hold", "hybrid"]); + if (settings.dictationDictionary !== undefined) { + runtime.dictationDictionary = parseDictationDictionary(settings.dictationDictionary); + } + if ( + settings.localVoiceIdleUnloadMinutes !== undefined && + !isLocalVoiceIdleUnloadMinutes(settings.localVoiceIdleUnloadMinutes) + ) { + delete runtime.localVoiceIdleUnloadMinutes; + } if (!Number.isSafeInteger(settings.telegramAllowedUserId)) delete runtime.telegramAllowedUserId; if (!isGenerationThinkingLevel(settings.telegramThinkingLevel)) delete runtime.telegramThinkingLevel; diff --git a/main/services/types.ts b/main/services/types.ts index 6ca5b1b96..61eb2976c 100644 --- a/main/services/types.ts +++ b/main/services/types.ts @@ -658,6 +658,12 @@ export interface AppSettings { dictationAccelerator?: string; /** Hold the dictation shortcut to record; release to transcribe. */ dictationHoldToTalk?: boolean; + /** Refines a release-capable shortcut: `hold` (push-to-talk) or `hybrid` (tap toggles, hold talks). */ + dictationActivationMode?: import("../../renderer/shared/dictation-preferences.js").DictationActivationMode; + /** Vocabulary and replacement rules applied to finished dictation transcripts. */ + dictationDictionary?: import("../../renderer/shared/dictation-dictionary.js").DictationDictionaryEntry[]; + /** Minutes an idle on-device voice model stays loaded; 0 keeps it loaded. */ + localVoiceIdleUnloadMinutes?: number; /** End dictation shortly after silence. */ dictationSilenceStop?: boolean; /** Polish the transcript with the current chat model before paste. */ diff --git a/package.json b/package.json index fe9c8a2ff..fabacf702 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,7 @@ "test:telegram": "tsx --test main/services/telegram/telegram-profile-mutation-fence.test.ts main/services/telegram/telegram-controls.test.ts main/services/telegram/telegram-inbound.test.ts main/services/telegram/telegram-outbound.test.ts main/services/telegram/telegram-queue.test.ts main/services/telegram/telegram-markdown.test.ts main/services/telegram/telegram-bot-api.test.ts main/services/telegram/telegram-turn.test.ts main/services/telegram/telegram-session.test.ts main/services/telegram/telegram-service-core.test.ts main/services/telegram/telegram-workspace-core.test.ts main/services/telegram/telegram-activity.test.ts main/services/telegram/telegram-profile-config.test.ts main/services/telegram/telegram-extension-registry.test.ts main/services/telegram/telegram-thread-store.test.ts main/services/telegram/telegram-ownership.test.ts main/services/telegram/telegram-agent-tools.test.ts main/services/telegram/telegram-bot-binding-store.test.ts main/services/telegram/telegram-bot-chat-lifecycle.test.ts main/services/telegram/telegram-bot-binding-reconciliation.test.ts main/services/telegram/telegram-bot-binding-validation.test.ts main/services/telegram/telegram-bot-binding-authority.test.ts renderer/lib/telegram-workspace-options.test.ts", "test:bots": "tsx --test main/services/telegram/telegram-bot-chat-lifecycle.test.ts main/services/telegram/telegram-bot-binding-reconciliation.test.ts main/services/telegram/telegram-bot-binding-validation.test.ts main/services/telegram/telegram-bot-binding-authority.test.ts main/services/bot-avatar-generator-core.test.ts main/services/bot-avatar-operation-registry.test.ts main/services/bot-avatar-store.test.ts main/services/bot-store-core.test.ts main/services/bot-chat-store.test.ts main/services/bot-mutation-gate.test.ts main/services/bot-inbox-projection.test.ts main/services/bot-system-prompt.test.ts main/services/bot-generation-preparation.test.ts main/services/bot-inbound-attachment-home.test.ts main/services/bot-file-tool-router.test.ts main/services/bot-tool-authority.test.ts main/services/bot-capability-store-core.test.ts main/services/bot-capability-store.test.ts main/services/bot-capability-state-checkpoint.test.ts main/services/bot-capability-keychain-anchor.test.ts main/services/bot-capability-secret-service-anchor.test.ts main/services/bot-capability-lease.test.ts main/services/bot-runtime-inventory-lease.test.ts main/services/bot-runtime-inventory-publication.test.ts main/services/bot-runtime-authority.test.ts main/services/bot-capability-catalog-core.test.ts main/services/bot-capability-bindings.test.ts main/services/bot-capability-key-store.test.ts main/services/bot-capability-migration-seal.test.ts main/services/bot-capability-incarnation-store.test.ts main/services/bot-capability-inventory-ports.test.ts main/services/bot-capability-production-shape.test.ts main/services/bot-mcp-inventory.test.ts main/services/bot-skill-inventory.test.ts main/services/bot-skill-content-watcher.test.ts main/services/bot-managed-workspace-core.test.ts main/services/bot-lifecycle-journal-core.test.ts main/services/bot-application-service.test.ts main/handlers/bot-params.test.ts main/handlers/bots.contract.test.ts renderer/main/bots-view.test.tsx renderer/lib/model-picker-data.test.ts renderer/lib/command-system-core.test.ts renderer/shared/bot-capabilities.test.ts", "test:bots:coverage": "node scripts/run-registered-tests-with-coverage.mjs test:bots", - "test:voice": "tsx --test main/services/dictation-hold-settings.test.ts main/services/linux-dictation-portal.test.ts main/services/transcription-core.test.ts main/services/gemini-live-transcription-core.test.ts main/services/dictation-coordinator.test.ts main/services/dictation-paste.test.ts main/services/parakeet-protocol.test.ts main/services/parakeet-process-core.test.ts main/services/parakeet-transcription-lane.test.ts renderer/shared/voice-models.test.ts renderer/shared/gemini-usage-scope.test.ts renderer/components/settings/gemini-voice-setup.test.tsx renderer/lib/accessibility-permission-core.test.ts renderer/lib/accessibility-refresh.test.ts renderer/lib/dictation-operation-gate.test.ts renderer/lib/gemini-recorded-retry.test.ts renderer/lib/live-pcm-capture.test.ts renderer/lib/voice-recorder-core.test.ts renderer/lib/wav-audio.test.ts", + "test:voice": "tsx --test main/services/dictation-hold-settings.test.ts main/services/linux-dictation-portal.test.ts main/services/transcription-core.test.ts main/services/gemini-live-transcription-core.test.ts main/services/dictation-coordinator.test.ts main/services/dictation-paste.test.ts main/services/parakeet-protocol.test.ts main/services/parakeet-process-core.test.ts main/services/parakeet-transcription-lane.test.ts renderer/shared/voice-models.test.ts renderer/shared/gemini-usage-scope.test.ts renderer/components/settings/gemini-voice-setup.test.tsx renderer/lib/accessibility-permission-core.test.ts renderer/lib/accessibility-refresh.test.ts renderer/lib/dictation-operation-gate.test.ts renderer/lib/gemini-recorded-retry.test.ts renderer/lib/live-pcm-capture.test.ts renderer/lib/voice-recorder-core.test.ts renderer/lib/wav-audio.test.ts main/services/parakeet-idle-unload.test.ts renderer/shared/dictation-dictionary.test.ts renderer/shared/dictation-preferences.test.ts renderer/components/settings/dictation-dictionary-settings.test.tsx", "test:tts": "tsx --test main/services/tts/authority-contract.test.ts renderer/components/settings/tts-settings.test.tsx main/services/tts/audio-store.test.ts renderer/lib/tts-client.test.ts renderer/lib/tts-player.test.ts main/services/tts/speech-text.test.ts main/services/tts/source.test.ts main/services/tts/gemini-wire.test.ts main/services/tts/service.test.ts main/services/tts/gemini-provider.loopback.test.ts renderer/components/message-bubble.test.tsx renderer/shared/tts.test.ts", "test:diagnostics": "tsx --test main/services/diagnostics-contract.test.ts main/services/diagnostic-health.test.ts main/services/diagnostic-journal.test.ts main/services/diagnostic-support.test.ts main/services/dev-log.test.ts main/services/process-diagnostics.test.ts main/services/renderer-crash-recovery.test.ts main/services/renderer-diagnostic-rate.test.ts main/services/subagents/subagent-runtime-diagnostics.test.ts renderer/components/settings/diagnostics-settings.test.tsx && node --test scripts/diagnostic-policy.test.mjs", "diagnostics:failure-receipt": "node scripts/write-diagnostic-failure-receipt.mjs", diff --git a/renderer/components/composer.tsx b/renderer/components/composer.tsx index 683613874..7da6f08b9 100644 --- a/renderer/components/composer.tsx +++ b/renderer/components/composer.tsx @@ -52,6 +52,7 @@ import { browserAnnotationAttachments, browserAnnotationContext } from "../lib/b import { browserAnnotationDelivery } from "../lib/browser-annotation-delivery"; import { composerImageAttach } from "../lib/composer-attach"; import type { BrowserAnnotation } from "../shared/browser"; +import { applyDictationDictionary, parseDictationDictionary } from "../shared/dictation-dictionary"; import { useDiscoveredSkills, useSettings } from "../lib/queries"; import type { Attachment, Chat, Workspace, WorkspacePermission } from "../lib/types"; import { composerSubmissionAllowed, computerUseControlState } from "../lib/computer-use-control"; @@ -535,6 +536,9 @@ export function Composer({ !sessionCommandBusy && !(isGenerating && stoppingGeneration); const settings = useSettings(); + // Read at transcript time so dictionary edits apply to an in-flight recording. + const dictationDictionaryRef = React.useRef(undefined); + dictationDictionaryRef.current = settings.data?.dictationDictionary; const skillCatalog = useDiscoveredSkills(workspace?.id); const selectedSkillState = React.useMemo( () => @@ -560,7 +564,11 @@ export function Composer({ !composing && (!selectedSkillState || selectedSkillState.state === "valid"); const voice = useVoiceRecorder( - (transcript) => { + (heard) => { + const transcript = applyDictationDictionary( + heard, + parseDictationDictionary(dictationDictionaryRef.current), + ); if (!firstSendPendingRef.current) setText((prev) => (prev.trim() ? `${prev.trim()} ${transcript}` : transcript)); }, { diff --git a/renderer/components/environment-subagents-contract.test.ts b/renderer/components/environment-subagents-contract.test.ts index 3e4d44dca..59ab9e46b 100644 --- a/renderer/components/environment-subagents-contract.test.ts +++ b/renderer/components/environment-subagents-contract.test.ts @@ -23,6 +23,7 @@ import { replaceSubagentStopPendingOwner, } from "../lib/subagent-stop-pending.js"; import { visibleSubagentReferences } from "../lib/subagent-feature-gate.js"; +import { resolveDictationActivationMode } from "../shared/dictation-preferences.js"; import type { ChatMessage } from "../lib/types.js"; function source(relativePath: string): string { @@ -635,12 +636,13 @@ test("dictation settings match host support even with a saved hold preference", assert.equal(parseAppCapabilities({ platform: "linux", dictationHoldToTalk: false }).dictationHoldToTalk, false); assert.equal(parseAppCapabilities({ platform: "darwin", dictationHoldToTalk: true }).dictationHoldToTalk, true); assert.equal(parseAppCapabilities({ dictationHoldToTalk: "true" }).dictationHoldToTalk, false); - const source = readFileSync(new URL("./settings/dictation-shortcut-settings.tsx", import.meta.url), "utf8"); - assert.match(source, /const holdToTalk = capabilities\.dictationHoldToTalk && settings\.data\?\.dictationHoldToTalk === true/u); - assert.match(source, /canChooseHold \? \([\s\S]*? { diff --git a/renderer/components/onboarding-flow.tsx b/renderer/components/onboarding-flow.tsx index 2abcbdcbe..fd7d62e83 100644 --- a/renderer/components/onboarding-flow.tsx +++ b/renderer/components/onboarding-flow.tsx @@ -374,7 +374,7 @@ const featureBentos: FeatureBento[] = [ group: "control", title: "Voice & Dictation", description: - "Speak in the composer or dictate system-wide. Choose shortcut behavior in Voice settings. Keep audio on-device with Parakeet, or explicitly connect cloud transcription and review what it can access.", + "Speak in the composer or dictate system-wide. Tap, hold, or both, and teach Aiden your names and terms in Voice settings. Keep audio on-device with Parakeet, or explicitly connect cloud transcription and review what it can access.", icon: Mic2, imageUrl: FEATURE_ILLUSTRATIONS.voice, size: "standard", diff --git a/renderer/components/settings/dictation-dictionary-settings.test.tsx b/renderer/components/settings/dictation-dictionary-settings.test.tsx new file mode 100644 index 000000000..c3104c272 --- /dev/null +++ b/renderer/components/settings/dictation-dictionary-settings.test.tsx @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { + DictationDictionaryView, + type DictationDictionaryViewProps, +} from "./dictation-dictionary-settings.js"; + +const noop = () => undefined; + +function props(extra: Partial = {}): DictationDictionaryViewProps { + return { + entries: [], + heard: "", + replacement: "", + error: null, + saving: false, + onHeardChange: noop, + onReplacementChange: noop, + onAdd: noop, + onRemove: noop, + ...extra, + }; +} + +type Element = React.ReactElement & { children?: React.ReactNode }>; + +function findAll(node: React.ReactNode, match: (element: Element) => boolean): Element[] { + const found: Element[] = []; + const visit = (current: React.ReactNode) => { + if (Array.isArray(current)) return current.forEach(visit); + if (!React.isValidElement(current)) return; + const element = current as Element; + if (match(element)) found.push(element); + visit(element.props.children); + }; + visit(node); + return found; +} + +test("an empty dictionary explains itself and cannot add a blank word", () => { + const html = renderToStaticMarkup(); + assert.match(html, /No custom words yet\./); + assert.match(html, /aria-label="Heard as"/); + assert.match(html, /aria-label="Replace with"/); + assert.match(html, /]*type="submit"[^>]*disabled=""/); +}); + +test("saved rules list the spoken form, its replacement, and a named remove action", () => { + const removed: number[] = []; + const tree = DictationDictionaryView( + props({ + entries: [ + { from: "aiden", to: "Aiden" }, + { from: "Kubernetes", to: "" }, + ], + onRemove: (index) => removed.push(index), + }), + ); + const html = renderToStaticMarkup(tree); + assert.match(html, /aria-label="Custom dictionary"/); + assert.match(html, /aiden.*Aiden/s); + assert.match(html, /Kubernetes.*exact spelling/s); + const [removeSecond] = findAll(tree, (element) => element.props["aria-label"] === "Remove Kubernetes"); + assert.ok(removeSecond); + (removeSecond.props.onClick as () => void)(); + assert.deepEqual(removed, [1]); +}); + +test("a rejected entry is announced and tied to the input", () => { + const html = renderToStaticMarkup( + , + ); + assert.match(html, /role="alert"[^>]*>Remove one first\./); + assert.match(html, /aria-invalid="true"/); +}); + +test("submitting the form adds the rule without reloading the page", () => { + let added = 0; + const tree = DictationDictionaryView(props({ heard: "aiden", onAdd: () => (added += 1) })); + const [form] = findAll(tree, (element) => element.type === "form"); + let prevented = false; + (form!.props.onSubmit as (event: { preventDefault: () => void }) => void)({ + preventDefault: () => (prevented = true), + }); + assert.equal(prevented, true); + assert.equal(added, 1); +}); diff --git a/renderer/components/settings/dictation-dictionary-settings.tsx b/renderer/components/settings/dictation-dictionary-settings.tsx new file mode 100644 index 000000000..5edc0b0b1 --- /dev/null +++ b/renderer/components/settings/dictation-dictionary-settings.tsx @@ -0,0 +1,186 @@ +// Custom dictation dictionary: words the recognizer should spell a specific +// way ("aiden" -> "Aiden") and phrase replacements. Applied to every finished +// transcript, from the global shortcut and the composer microphone alike. + +import * as React from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { ArrowRight, Plus, Trash2 } from "lucide-react"; +import { Button, Field, FieldSet, Input, Text, toast } from "../ui"; +import { settingsApi } from "../../lib/ipc"; +import { queryKeys, useSettings } from "../../lib/queries"; +import { + addDictationDictionaryEntry, + parseDictationDictionary, + type DictationDictionaryEntry, +} from "../../shared/dictation-dictionary"; + +// One dictionary editor exists per Settings page, so a fixed id is unique. +const errorId = "dictation-dictionary-error"; + +export interface DictationDictionaryViewProps { + entries: readonly DictationDictionaryEntry[]; + heard: string; + replacement: string; + error: string | null; + saving: boolean; + onHeardChange: (value: string) => void; + onReplacementChange: (value: string) => void; + onAdd: () => void; + onRemove: (index: number) => void; +} + +export function DictationDictionaryView({ + entries, + heard, + replacement, + error, + saving, + onHeardChange, + onReplacementChange, + onAdd, + onRemove, +}: DictationDictionaryViewProps) { + return ( +
+ +
{ + event.preventDefault(); + onAdd(); + }} + > + onHeardChange(event.currentTarget.value)} + /> + onReplacementChange(event.currentTarget.value)} + /> + +
+ {error ? ( + + {error} + + ) : null} + {entries.length === 0 ? ( + + No custom words yet. + + ) : ( +
    + {entries.map((entry, index) => ( +
  • + + {entry.from} + + {entry.to ? ( + <> + + + {entry.to} + + + ) : ( + + exact spelling + + )} + +
    • + ))} +
+ )} + +
+ ); +} + +export function DictationDictionarySettings() { + const qc = useQueryClient(); + const settings = useSettings(); + const entries = React.useMemo( + () => parseDictationDictionary(settings.data?.dictationDictionary), + [settings.data?.dictationDictionary], + ); + const [heard, setHeard] = React.useState(""); + const [replacement, setReplacement] = React.useState(""); + const [error, setError] = React.useState(null); + const [saving, setSaving] = React.useState(false); + + const save = async (next: DictationDictionaryEntry[]): Promise => { + setSaving(true); + try { + await settingsApi.set({ dictationDictionary: next }); + await qc.invalidateQueries({ queryKey: queryKeys.settings }); + return true; + } catch { + toast.error("Aiden couldn’t save the custom dictionary."); + return false; + } finally { + setSaving(false); + } + }; + + const add = async () => { + const edit = addDictationDictionaryEntry(entries, heard, replacement); + if (!edit.ok) { + setError(edit.error); + return; + } + setError(null); + if (await save(edit.entries)) { + setHeard(""); + setReplacement(""); + } + }; + + return ( + { + setHeard(value); + setError(null); + }} + onReplacementChange={setReplacement} + onAdd={() => void add()} + onRemove={(index) => void save(entries.filter((_, position) => position !== index))} + /> + ); +} diff --git a/renderer/components/settings/dictation-shortcut-settings.tsx b/renderer/components/settings/dictation-shortcut-settings.tsx index 538bc6311..a722bb09d 100644 --- a/renderer/components/settings/dictation-shortcut-settings.tsx +++ b/renderer/components/settings/dictation-shortcut-settings.tsx @@ -28,6 +28,11 @@ import { type AccessibilityPermissionState, } from "../../lib/accessibility-permission-core"; import { useAppCapabilities } from "../../lib/app-capabilities"; +import { + isDictationActivationMode, + resolveDictationActivationMode, + type DictationActivationMode, +} from "../../shared/dictation-preferences"; async function openAccessibilitySettings(): Promise { try { @@ -225,18 +230,21 @@ export function DictationShortcutSettings() { const settings = useSettings(); const [holdBusy, setHoldBusy] = React.useState(false); const canChooseHold = capabilities.dictationHoldToTalk || capabilities.dictationHoldSetup; - const holdToTalk = capabilities.dictationHoldToTalk && settings.data?.dictationHoldToTalk === true; + const activationMode: DictationActivationMode = resolveDictationActivationMode( + settings.data ?? {}, + capabilities.dictationHoldToTalk, + ); const silenceStop = settings.data?.dictationSilenceStop === true; const cleanup = settings.data?.dictationCleanup === true; const sounds = settings.data?.dictationSounds === true; const patch = async (next: { - dictationHoldToTalk?: boolean; + dictationActivationMode?: DictationActivationMode; dictationSilenceStop?: boolean; dictationCleanup?: boolean; dictationSounds?: boolean; }) => { - if (next.dictationHoldToTalk !== undefined) setHoldBusy(true); + if (next.dictationActivationMode !== undefined) setHoldBusy(true); try { await settingsApi.set(next); await qc.invalidateQueries({ queryKey: queryKeys.settings }); @@ -265,8 +273,13 @@ export function DictationShortcutSettings() { orientation="vertical" > void patch({ dictationHoldToTalk: canChooseHold && value === "hold" })} + value={activationMode} + onValueChange={(value) => + void patch({ + dictationActivationMode: + canChooseHold && isDictationActivationMode(value) ? value : "toggle", + }) + } orientation="vertical" aria-label="Dictation shortcut behavior" disabled={holdBusy} @@ -284,6 +297,17 @@ export function DictationShortcutSettings() { ) : null} + {canChooseHold ? ( + + ) : null}