From b7d43a4399a137689859d6052bcf97dc3a65ea6b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 2 Oct 2026 08:24:41 +0200 Subject: [PATCH 1/5] Speak templated progress during long Brunch voice turns Co-authored-by: Amp --- .../local-storage-demo-app.tsx | 4 + .../live-brunch-bridge.test.ts | 213 ++++++++++++- .../app/voice-interview/live-brunch-bridge.ts | 148 ++++++++- .../live-conversation-control.test.tsx | 137 +++++++++ .../live-conversation-control.tsx | 26 +- .../voice-interview/live-conversation.test.ts | 30 ++ .../live-progress-policy.test.ts | 260 ++++++++++++++++ .../voice-interview/live-progress-policy.ts | 286 ++++++++++++++++++ .../live-speech-captions.test.ts | 64 ++++ .../voice-interview/live-speech-captions.ts | 19 +- .../voice-interview-control.tsx | 3 + .../server/voice/openai-live-session.test.ts | 3 +- .../src/server/voice/openai-live-session.ts | 5 +- 13 files changed, 1186 insertions(+), 12 deletions(-) create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index b6c08f4025a..289eef0fb17 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -234,6 +234,7 @@ export const getBrunchVoiceMode = ( settlements?: readonly FlueConversationSettlement[], snapshot?: FlueConversationState, mediationHistory?: VoiceMediationHistory, + isToolAwaitingApproval?: (toolCallId: string) => boolean, ): PetrinautAiVoiceMode | undefined => { if (!config) return undefined; @@ -261,6 +262,7 @@ export const getBrunchVoiceMode = ( {...context} config={config} mediationHistory={mediationHistory} + isToolAwaitingApproval={isToolAwaitingApproval} settlements={settlements} // Voice only observes this snapshot. Message replacement remains gated // independently by followMessages.canReplace below. @@ -822,6 +824,7 @@ export const LocalStorageDemoApp = ({ flueHistory.settlements, flueHistory.snapshot, mediationHistory, + (toolCallId) => mutationApproval.coordinator.hasPending(toolCallId), ), [ brunchSelected, @@ -829,6 +832,7 @@ export const LocalStorageDemoApp = ({ flueHistory.settlements, flueHistory.snapshot, mediationHistory, + mutationApproval, openAIVoiceConfig, realtimeEnabled, realtimePreferenceReady, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index e452caac022..dbe984b11fe 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -11,6 +11,8 @@ beforeEach(() => { }); afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); }); @@ -34,6 +36,7 @@ const setup = ( ); const notice = vi.fn(); const speechPending = vi.fn(() => true); + const isToolAwaitingApproval = vi.fn(() => false); const submit = vi.fn( async ( input: Parameters< @@ -54,6 +57,7 @@ const setup = ( appendThinking, notice, speechPending, + isToolAwaitingApproval, submit, mediation, }); @@ -76,6 +80,7 @@ const setup = ( appendThinking, notice, speechPending, + isToolAwaitingApproval, submit, update, }; @@ -1927,6 +1932,12 @@ test.each(["button", "snapshot", "aborted"] as const)( }, ]; fixture.update({ status: "streaming", messages }); + // Quiet phase publication is independent of the interruption context. + expect(fixture.appendThinking).toHaveBeenCalledWith( + expect.stringContaining('"progress"'), + null, + ); + fixture.appendThinking.mockClear(); if (stopKind === "button") fixture.bridge.stopResponse(); else if (stopKind === "snapshot") fixture.update({ stopped: true, messages }); @@ -2433,7 +2444,11 @@ test.each([ ? [] : [[expect.stringContaining(expectedStatus), "request"]], ); - expect(fixture.appendThinking).not.toHaveBeenCalled(); + expect( + fixture.appendThinking.mock.calls.every(([text]) => + text.startsWith('{"progress":'), + ), + ).toBe(true); expect( fixture.appendInstructions.mock.calls.flat().join(" "), ).not.toContain("Ask the person to continue"); @@ -2451,3 +2466,199 @@ test("textless and dropped inputs without delegations do not inject session-wide expect(fixture.appendInstructions).not.toHaveBeenCalled(); expect(fixture.appendCommentary).not.toHaveBeenCalled(); }); + +const runningTool = (toolName = "createExperiment") => ({ + id: "answer", + role: "assistant" as const, + parts: [ + { + type: "dynamic-tool" as const, + toolName, + toolCallId: "call", + state: "input-available" as const, + input: {}, + }, + ], +}); + +test("progress leaves the delegation open and settlement waits for Live speech", async () => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Compare staffing")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming", messages: [runningTool()] }); + fixture.bridge.liveSpeaking(true); + await vi.advanceTimersByTimeAsync(2_000); + fixture.bridge.liveSpeaking(false); + await vi.advanceTimersByTimeAsync(5_999); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(251); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Setting up the comparison.", + null, + ); + fixture.bridge.liveSpeaking(true); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + await vi.advanceTimersByTimeAsync(3_000); + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + fixture.bridge.liveSpeaking(false); + await vi.advanceTimersByTimeAsync(250); + expect(fixture.appendCommentary).toHaveBeenLastCalledWith( + segment().text, + "delegation", + ); + await vi.advanceTimersByTimeAsync(40_000); + expect(fixture.appendCommentary).toHaveBeenCalledTimes(2); + fixture.bridge.stop(); + expect(vi.getTimerCount()).toBe(0); +}); + +test("approval changes are polled even without a new message snapshot", async () => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Remove that place")); + fixture.speechPending.mockReturnValue(false); + fixture.isToolAwaitingApproval.mockReturnValue(true); + fixture.bridge.responseStarted(started); + fixture.update({ + status: "streaming", + messages: [runningTool("removePlace")], + }); + await vi.advanceTimersByTimeAsync(40_000); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect( + fixture.appendThinking.mock.calls.some(([text]) => + text.includes('"phase":"awaiting-approval"'), + ), + ).toBe(true); + fixture.isToolAwaitingApproval.mockReturnValue(false); + await vi.advanceTimersByTimeAsync(2_500); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(250); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Making those changes now.", + null, + ); + fixture.bridge.stop(); +}); + +test.each(["speech", "stop", "error", "settlement"] as const)( + "%s cancels future progress", + async (cause) => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Compare staffing")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming", messages: [runningTool()] }); + await vi.advanceTimersByTimeAsync(5_750); + if (cause === "speech") fixture.bridge.speechStarted(); + else if (cause === "stop") fixture.bridge.stopResponse(); + else if (cause === "error") fixture.update({ status: "error" }); + else wrapUp(fixture); + await vi.advanceTimersByTimeAsync(60_000); + expect( + fixture.appendCommentary.mock.calls.filter( + ([text]) => text === "Setting up the comparison.", + ), + ).toEqual([]); + fixture.bridge.stop(); + }, +); + +test("historical and unrelated tools cannot supply progress for this turn", async () => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.update({ messages: [runningTool()] }); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Explain this")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted({ ...started, messageId: "new-answer" }); + fixture.update({ + status: "streaming", + messages: [runningTool(), { ...runningTool(), id: "unrelated" }], + }); + await vi.advanceTimersByTimeAsync(6_250); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Give me a moment on this one.", + null, + ); + fixture.bridge.stop(); +}); + +test("a tool continuation can report progress after its initial submission settled", async () => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Compare staffing")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ status: "streaming", settlements: completed }); + await vi.advanceTimersByTimeAsync(7_000); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.bridge.responseStarted({ + ...started, + submissionId: "continuation", + position: { batch: 3, index: 0 }, + }); + fixture.update({ + status: "streaming", + settlements: completed, + messages: [runningTool()], + }); + await vi.advanceTimersByTimeAsync(250); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Setting up the comparison.", + null, + ); + fixture.bridge.stop(); +}); + +test.each([true, false])( + "output-available applied=%s uses reported edit outcomes, not the transport state", + async (applied) => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Edit three places")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.update({ + status: "streaming", + messages: [ + { + id: "answer", + role: "assistant", + parts: ["one", "two", "three"].map((toolCallId) => ({ + type: "dynamic-tool", + toolName: "addPlace", + toolCallId, + state: "output-available", + input: {}, + output: { applied: toolCallId === "two" ? applied : true }, + })), + }, + ], + }); + await vi.advanceTimersByTimeAsync(6_250); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + applied + ? "The edits are in. Give me a moment." + : "Give me a moment on this one.", + null, + ); + fixture.bridge.stop(); + }, +); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts index ca89969f732..9013c795474 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -1,9 +1,12 @@ +import { getToolName, isToolUIPart } from "ai"; + import { serializeVoiceBrief } from "../../../shared/voice-mediation"; import { selectCanonicalSpeech } from "./canonical-speech"; import { liveUtteranceStages, routeUtterance, } from "./live-brunch-bridge/utterance-pipeline"; +import { ProgressPolicy } from "./live-progress-policy"; import { logLiveDiagnostic } from "./shared/live-diagnostic"; import type { VoiceBriefFields } from "../../../shared/voice-mediation"; @@ -44,6 +47,8 @@ interface Turn { readonly baseline: ReadonlySet; readonly baselineMessages: ReadonlySet; delegationId: string | null; + readonly progress: ProgressPolicy; + progressOffered: boolean; submitted?: boolean; submissionId?: string; /** The conversation history the turn began in, so a switch cannot split it. */ @@ -82,12 +87,17 @@ interface Dependencies { text: string, delegationId: string | null, ) => boolean; - readonly appendInstructions: (text: string, delegationId: string) => boolean; - /** Quiet interruption context, never spoken or bound to a delegation. */ + readonly appendInstructions: ( + text: string, + delegationId: string | null, + ) => boolean; + /** Quiet progress/interruption context, never spoken or bound to a delegation. */ readonly appendThinking: (text: string, delegationId: null) => boolean; readonly notice: (message: string | null) => void; /** Transcription speech has started and its transcript isn't finalized yet. */ readonly speechPending: () => boolean; + /** Browser approval authority; input-available alone also means executing. */ + readonly isToolAwaitingApproval?: (toolCallId: string) => boolean; } /** GPT-Live accepts at most 500 tokens per append; stay well inside it. */ @@ -130,6 +140,9 @@ export class LiveBrunchBridge { > >(); #waitingForComposer: Turn | undefined; + #progressTimer: ReturnType | undefined; + #liveSpeaking = false; + #progressPendingUntil = 0; #chat: Chat = { canAcceptVoiceInput: false, segments: [], @@ -141,6 +154,100 @@ export class LiveBrunchBridge { this.#dependencies = dependencies; } + /** Acoustic activity is a hold, not proof of provider playback completion. */ + public liveSpeaking(on: boolean): void { + if (this.#liveSpeaking && !on) { + for (const turn of this.#turns) { + if (turn.delegationId !== null && !turn.progressOffered) + turn.progress.acknowledged(Date.now()); + } + } + this.#liveSpeaking = on; + if (on) this.#progressPendingUntil = 0; + } + + #stopProgress(): void { + clearInterval(this.#progressTimer); + this.#progressTimer = undefined; + } + + #updateProgress(): void { + if (this.#abort.signal.aborted || this.#turns.size === 0) { + this.#stopProgress(); + return; + } + const now = Date.now(); + for (const turn of this.#turns) { + if (turn.superseded || !turn.submissionId || turn.delegationId === null) + continue; + const { messages, required } = this.#responseScope(turn.submissionId); + // Do not narrate a completed backend turn while its final rendering or + // mediation catches up. Client continuations still have missing settlements. + if ( + this.#chat.settlements.some( + (entry) => + required.has(entry.submissionId) && entry.outcome !== "completed", + ) || + [...required].every((id) => + this.#chat.settlements.some((entry) => entry.submissionId === id), + ) + ) { + continue; + } + for (const message of this.#chat.messages ?? []) { + if ( + message.role !== "assistant" || + !messages.has(message.id) || + turn.baselineMessages.has(message.id) + ) + continue; + for (const part of message.parts) { + if (!isToolUIPart(part)) continue; + const state = + this.#dependencies.isToolAwaitingApproval?.(part.toolCallId) || + part.state === "approval-requested" + ? "awaiting-approval" + : part.state === "input-streaming" + ? "preparing" + : "running"; + turn.progress.toolStarted( + part.toolCallId, + getToolName(part), + now, + state, + ); + if ( + part.state === "output-available" || + part.state === "output-error" || + part.state === "output-denied" + ) { + const output: unknown = + part.state === "output-available" ? part.output : undefined; + const failed = + typeof output === "object" && + output !== null && + (("applied" in output && output.applied === false) || + ("success" in output && output.success === false) || + ("status" in output && + ["invalid", "error", "cancelled", "failed"].includes( + String(output.status), + ))); + turn.progress.toolFinished( + part.toolCallId, + now, + part.state === "output-available" && !failed, + ); + } + } + } + turn.progress.userSpeaking(this.#dependencies.speechPending()); + turn.progress.liveSpeaking( + this.#liveSpeaking || now < this.#progressPendingUntil, + ); + turn.progress.evaluate(now); + } + } + public stop(): void { this.#closeDeferredDelegations(); this.#abort.abort(); @@ -151,6 +258,8 @@ export class LiveBrunchBridge { /** Stop future speech offers, not work already admitted by Brunch. */ public speechStarted(): void { + this.#stopProgress(); + this.#progressPendingUntil = 0; for (const preparation of this.#preparations) preparation.abort(); this.#preparations.clear(); let withdrew = false; @@ -273,6 +382,9 @@ export class LiveBrunchBridge { ); if (turn) { turn.delegationId = delegationId; + // Live's prompt acknowledges before delegating. Receipt is a conservative + // clock proxy; observed acknowledgement audio ending moves it later. + turn.progress.acknowledged(Date.now()); logLiveDiagnostic("delegation.matched", { delegationId, inputId: turn.inputId, @@ -416,6 +528,23 @@ export class LiveBrunchBridge { superseded: input.superseded, preparation: new AbortController(), delegationId, + progressOffered: false, + progress: new ProgressPolicy({ + commentary: (line) => { + // Keep this selector local: null-delegation commentary is awaiting + // real-provider verification; instructions is a one-line fallback. + const sent = this.#dependencies.appendCommentary(line, null); + turn.progressOffered = true; + // Allow output to begin before sending a simultaneously settled wrap-up. + // A provider that stays silent must not block the summary indefinitely. + if (sent) this.#progressPendingUntil = Date.now() + 5_000; + return sent; + }, + thinking: (context) => { + this.#dependencies.appendThinking(JSON.stringify(context), null); + }, + diagnostic: logLiveDiagnostic, + }), history: this.#dependencies.mediation.history, baseline: new Set(this.#chat.segments.map((segment) => segment.id)), baselineMessages: new Set([ @@ -426,6 +555,9 @@ export class LiveBrunchBridge { }; this.#waitingForComposer = turn; this.#turns.add(turn); + turn.progress.startTurn(Date.now()); + if (delegationId !== null) turn.progress.acknowledged(Date.now()); + this.#progressTimer ??= setInterval(() => this.#updateProgress(), 250); this.#preparations.add(turn.preparation); try { let text = input.text; @@ -559,9 +691,12 @@ export class LiveBrunchBridge { return; } this.#settle(); + this.#updateProgress(); } #interruptTurns(reason: "stopped" | "error"): void { + this.#stopProgress(); + this.#progressPendingUntil = 0; for (const preparation of this.#preparations) preparation.abort(); this.#preparations.clear(); for (const turn of this.#turns) { @@ -909,6 +1044,7 @@ export class LiveBrunchBridge { continue; } for (const segment of segments) this.#offeredSegments.add(segment.id); + turn.progress.settled(); this.#turns.delete(turn); // Freeze complete prose once. Sending is neither exact relay nor playback proof. const source = segments.map((segment) => segment.text).join("\n\n"); @@ -946,6 +1082,14 @@ export class LiveBrunchBridge { source, turn.preparation.signal, ); + while ( + turn.progressOffered && + (this.#liveSpeaking || Date.now() < this.#progressPendingUntil) && + !turn.preparation.signal.aborted && + !this.#abort.signal.aborted + ) { + await new Promise((resolve) => setTimeout(resolve, 250)); + } if (turn.preparation.signal.aborted || this.#abort.signal.aborted) return; // Only provider output transcripts prove that these words were spoken. mediation.offered(turn.inputId); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index e99d153e696..2a3b1704214 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -1442,6 +1442,11 @@ test("Stop sends the partial answer to Live quietly and Continue admits one new config={config} />, ); + expect(session.appendThinking).toHaveBeenCalledWith( + expect.stringContaining('"progress"'), + null, + ); + vi.mocked(session.appendThinking).mockClear(); act(() => tracker.recordStopRequested()); expect(session.appendThinking).toHaveBeenCalledExactlyOnceWith( expect.stringContaining( @@ -1781,3 +1786,135 @@ test.each(["commentary", "instructions"] as const)( expect(props.reportVoiceSessionState).not.toHaveBeenCalled(); }, ); + +test("Live progress observes the host approval gate and stays out of written voice history", async () => { + mockMediation(); + const tracker = new BrunchPanelConversationTracker(); + const history = new VoiceMediationHistory("progress-test"); + const props = context(); + props.submitVoiceInput = vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-input", + })); + const awaitingApproval = vi.fn(() => true); + const wiring = { + mediationHistory: history, + isToolAwaitingApproval: awaitingApproval, + resolveInputSubmission: () => "root", + subscribeToResponseMessageStarted: + tracker.subscribeToResponseMessageStarted.bind(tracker), + }; + const { rerender, unmount } = render( + , + ); + await start(); + vi.useFakeTimers(); + try { + const call = vi.mocked(createLiveConversation).mock.lastCall!; + const session = vi.mocked(createLiveConversation).mock.results.at(-1)! + .value as ReturnType; + act(() => { + call[0]({ phase: "connected", message: null }); + call[6]?.input({ + id: "input", + text: "Remove that place", + startMs: 100, + endMs: 200, + }); + call[3]("delegation"); + call[2]({ + id: "voice-input", + text: "Remove that place", + startedDuringOutput: false, + }); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(0); + }); + vi.mocked(session.speechPending).mockReturnValue(false); + act(() => + tracker.recordResponse({ + messageId: "answer", + submissionId: "root", + position: { batch: 1, index: 0 }, + }), + ); + rerender( + , + ); + await act(async () => { + await vi.advanceTimersByTimeAsync(30_000); + }); + expect(awaitingApproval).toHaveBeenCalledWith("delete"); + expect(session.appendCommentary).not.toHaveBeenCalled(); + awaitingApproval.mockReturnValue(false); + act(() => + call[0]({ + phase: "connected", + message: null, + activity: { microphoneLevel: 0, outputActive: true }, + }), + ); + await act(async () => { + await vi.advanceTimersByTimeAsync(10_000); + }); + expect(session.appendCommentary).not.toHaveBeenCalled(); + act(() => + call[0]({ + phase: "connected", + message: null, + activity: { microphoneLevel: 0, outputActive: false }, + }), + ); + await act(async () => { + await vi.advanceTimersByTimeAsync(6_000); + }); + expect(session.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Making those changes now.", + null, + ); + act(() => { + call[4]({ + eventId: "progress", + kind: "commentary", + delegationId: null, + status: "accepted", + startMs: 46_000, + }); + call[6]?.output({ + id: "progress-output", + text: "Making those changes now.", + startMs: 46_100, + endMs: 47_000, + }); + }); + expect(JSON.stringify(history.project([]))).not.toContain( + "Making those changes now.", + ); + expect(screen.queryByText("Making those changes now.")).toBeNull(); + unmount(); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index 4bd632fe8c5..9a0fe978018 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -38,6 +38,7 @@ type LiveControlsContext = PetrinautAiVoiceModeContext & | "subscribeToResponseMessageStarted" | "subscribeToResponseMessageCompleted" | "subscribeToStopRequested" + | "isToolAwaitingApproval" > & { readonly mediationHistory?: VoiceMediationHistory; readonly acknowledgeDisclosure: () => void; @@ -85,6 +86,7 @@ export const LiveConversationControl = ({ subscribeToResponseMessageStarted, subscribeToResponseMessageCompleted, subscribeToStopRequested, + isToolAwaitingApproval, }: LiveControlsContext) => { const [localHistory] = useState(() => new VoiceMediationHistory("session")); const history = mediationHistory ?? localHistory; @@ -127,6 +129,7 @@ export const LiveConversationControl = ({ const latest = useRef({ submit, messages, + isToolAwaitingApproval, chat: { status, stopped, @@ -163,6 +166,7 @@ export const LiveConversationControl = ({ latest.current = { submit, messages, + isToolAwaitingApproval, chat: { status, stopped, @@ -183,6 +187,7 @@ export const LiveConversationControl = ({ resolveResponseSubmission, settlements, snapshot, + isToolAwaitingApproval, ]); useEffect( @@ -240,6 +245,7 @@ export const LiveConversationControl = ({ const next = createLiveConversation( (nextState) => { if (session.current !== next) return; + bridge.current?.liveSpeaking(nextState.activity?.outputActive === true); if ( nextState.phase !== "connected" || nextState.activity?.outputActive @@ -290,13 +296,19 @@ export const LiveConversationControl = ({ offeredInput = undefined; } const inputId = appendInputs.get(result.eventId); - if ( - inputId && - result.status === "accepted" && - result.startMs !== undefined - ) - captions.wrapUp(inputId, result.startMs); + if (result.status === "accepted" && result.startMs !== undefined) { + if (inputId) captions.wrapUp(inputId, result.startMs); + else if (result.delegationId === null) + captions.progress(result.startMs); + } if (result.status !== "unknown") appendInputs.delete(result.eventId); + } else if ( + result.kind === "instructions" && + result.delegationId === null && + result.status === "accepted" && + result.startMs !== undefined + ) { + captions.progress(result.startMs); } // Every successful local send starts as unknown. Neither waiting // for acceptance nor acceptance itself is an error or resolves a @@ -358,6 +370,8 @@ export const LiveConversationControl = ({ appendThinking: next.appendThinking, notice: setWarningMessage, speechPending: next.speechPending, + isToolAwaitingApproval: (toolCallId) => + latest.current.isToolAwaitingApproval?.(toolCallId) ?? false, }); bridge.current.update(latest.current.chat); session.current = next; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts index 34f60e89d22..517d33eaf82 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -1518,6 +1518,36 @@ test("telemetry shows activity but silence and late samples never settle or revi expect(fixture.onState).toHaveBeenCalledTimes(calls); }); +test("progress commentary has a null delegation and only the later wrap-up closes the local delegation", async () => { + const fixture = setup(); + await connect(fixture); + fixture.emit(0, { + type: "session.delegation.created", + delegation: { id: "opaque", target: "client" }, + }); + fixture.conversation.appendCommentary("Give me a moment on this one.", null); + const progress = fixture.onAppendResult.mock.lastCall![0]; + expect(JSON.parse(fixture.sent[0][0]!)).toMatchObject({ + type: "session.commentary.append", + delegation_id: null, + content: "Give me a moment on this one.", + }); + fixture.emit(0, { + type: "session.commentary.appended", + client_event_id: progress.eventId, + start_ms: 1_000, + }); + expect(fixture.conversation.openDelegations.has("opaque")).toBe(true); + fixture.conversation.appendCommentary("The model is ready.", "opaque"); + const wrapUp = fixture.onAppendResult.mock.lastCall![0]; + fixture.emit(0, { + type: "session.commentary.appended", + client_event_id: wrapUp.eventId, + start_ms: 2_000, + }); + expect(fixture.conversation.openDelegations.has("opaque")).toBe(false); +}); + test("quiet interruption context requires its own acknowledgement and leaves the delegation open", async () => { const fixture = setup(); await connect(fixture); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts new file mode 100644 index 00000000000..d0b96b9b152 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts @@ -0,0 +1,260 @@ +import { expect, test, vi } from "vitest"; + +import { ProgressPolicy } from "./live-progress-policy"; + +const tool = (at: number, name: string, duration = 400) => ({ + at, + name, + duration, +}); +const burst = (at: number, names: string[], step: number, duration: number) => + names.map((name, index) => tool(at + index * step, name, duration)); +const build = [ + tool(1_500, "getLatestNetDefinition", 600), + tool(2_400, "mutate_workpiece", 1_200), + ...burst( + 7_000, + [ + "addPlace", + "addTransition", + "addArc", + "addArc", + "addParameter", + "addParameter", + "addTransition", + "addArc", + ], + 600, + 400, + ), + tool(19_000, "addMetric", 500), + tool(20_200, "updateTransition", 500), + tool(24_500, "applyAutoLayout", 900), + tool(27_000, "getNetCompilationErrors", 600), +]; + +// The six prototype timelines, with the locked plan's immediate experiment gate. +const scenarios = [ + { name: "quick answer", tools: [], settleAt: 3_000, expected: [] }, + { + name: "fast burst", + tools: [ + tool(1_200, "getLatestNetDefinition", 500), + ...burst( + 2_200, + [ + "addPlace", + "addTransition", + "addArc", + "addArc", + "updateTransition", + "addPlace", + "addArc", + "addArc", + ], + 450, + 300, + ), + ], + settleAt: 9_000, + expected: [], + }, + { + name: "build the net", + tools: build, + settleAt: 31_000, + expected: ["The edits are in. Give me a moment."], + }, + { + name: "long experiment draft", + tools: [ + tool(1_500, "getLatestNetDefinition", 600), + tool(4_800, "createExperiment", 35_000), + ], + settleAt: 45_000, + expected: [ + "Setting up the comparison.", + "Still setting up the comparison.", + ], + }, + { + name: "thinking only", + tools: [], + settleAt: 26_000, + expected: ["Give me a moment on this one."], + }, + { + name: "interruption held", + tools: build, + settleAt: 31_000, + speech: { from: 15_000, to: 19_000 }, + expected: [], + }, +]; + +const setup = () => { + type Output = ConstructorParameters[0]; + const commentary = vi.fn(() => true); + const thinking = vi.fn(); + const diagnostic = vi.fn(); + const policy = new ProgressPolicy({ commentary, thinking, diagnostic }); + policy.startTurn(0); + policy.acknowledged(600); + return { policy, commentary, thinking, diagnostic }; +}; + +test.each(scenarios)( + "$name matches the prototype without speaking near settlement", + (scenario) => { + const { policy, commentary } = setup(); + const spokenAt: number[] = []; + // 50 ms inputs retain the scripted boundaries; evaluation is every 250 ms. + for (let now = 0; now <= scenario.settleAt + 40_000; now += 50) { + for (const [index, call] of scenario.tools.entries()) { + if (now === call.at) policy.toolStarted(String(index), call.name, now); + if (now === call.at + call.duration) + policy.toolFinished(String(index), now, true); + } + policy.userSpeaking( + !!scenario.speech && + now >= scenario.speech.from && + now < scenario.speech.to, + ); + if (now === scenario.settleAt) policy.settled(); + if (now % 250 === 0 && policy.evaluate(now)?.speak) spokenAt.push(now); + } + expect(commentary.mock.calls.map(([line]) => line)).toEqual( + scenario.expected, + ); + expect(spokenAt.every((at) => at < scenario.settleAt - 1_000)).toBe(true); + }, +); + +test("approval waits stay silent, then execution gets its own running clock", () => { + const { policy, commentary, thinking } = setup(); + policy.toolStarted("delete", "removePlace", 1_000, "awaiting-approval"); + for (let now = 1_000; now <= 60_000; now += 250) policy.evaluate(now); + expect(commentary).not.toHaveBeenCalled(); + expect( + thinking.mock.calls.map(([context]) => context.progress.phase), + ).toEqual(["awaiting-approval"]); + policy.toolStarted("delete", "removePlace", 60_000); + expect(policy.evaluate(62_499)?.speak).toBe(false); + expect(policy.evaluate(62_500)?.speak).toBe(true); + expect(commentary).toHaveBeenCalledWith("Making those changes now."); +}); + +test("ack, running, gap, repeat and cap gates include their exact boundaries", () => { + const { policy, commentary } = setup(); + policy.toolStarted("one", "addPlace", 4_100); + expect(policy.evaluate(6_599)?.speak).toBe(false); + expect(policy.evaluate(6_600)?.speak).toBe(true); + expect(policy.evaluate(26_600)?.speak).toBe(false); + expect(policy.evaluate(36_599)?.speak).toBe(false); + expect(policy.evaluate(36_600)?.speak).toBe(true); + policy.toolStarted("two", "createExperiment", 60_000); + expect(policy.evaluate(100_000)?.speak).toBe(false); + expect(commentary.mock.calls).toEqual([ + ["Making those changes now."], + ["Still working through the changes."], + ]); +}); + +test.each([ + ["createExperiment", "Setting up the comparison."], + ["draft_petrinaut_experiment", "Setting up the comparison."], + ["task", "Working through the details."], + ["mutate_workpiece", "Working through the details."], +])("%s qualifies immediately once acknowledged", (name, line) => { + const { policy, commentary } = setup(); + policy.toolStarted("one", name, 7_000); + expect(policy.evaluate(7_000)?.speak).toBe(true); + expect(commentary).toHaveBeenCalledWith(line); +}); + +test("a changed phase still waits twenty seconds and Live speech holds it", () => { + const { policy } = setup(); + policy.evaluate(6_600); + policy.toolStarted("experiment", "createExperiment", 26_000); + expect(policy.evaluate(26_599)?.speak).toBe(false); + policy.liveSpeaking(true); + expect(policy.evaluate(26_600)?.speak).toBe(false); + policy.liveSpeaking(false); + expect(policy.evaluate(26_601)?.speak).toBe(true); +}); + +test("phase changes publish quiet state even before speech qualifies; duplicates do not reset running time", () => { + const { policy, thinking, diagnostic } = setup(); + policy.evaluate(1_000); + policy.toolStarted("one", "addPlace", 4_500); + policy.evaluate(4_500); + policy.toolStarted("one", "addPlace", 6_500); + expect(policy.evaluate(6_999)?.speak).toBe(false); + expect(policy.evaluate(7_000)?.speak).toBe(true); + expect(thinking.mock.calls.map(([value]) => value.progress.phase)).toEqual([ + "thinking", + "mutation", + "mutation", + ]); + const suppressions = diagnostic.mock.calls.filter( + ([name]) => name === "progress.suppressed", + ); + policy.evaluate(7_001); + policy.evaluate(7_002); + expect( + diagnostic.mock.calls.filter(([name]) => name === "progress.suppressed"), + ).toHaveLength(suppressions.length + 1); +}); + +test.each([ + [ + "getLatestNetDefinition", + true, + "I've had a look at the model. Give me a moment.", + ], + [ + "draft_petrinaut_experiment", + true, + "The comparison is drafted. One moment.", + ], + ["createExperiment", true, "Give me a moment on this one."], + ["createExperiment", false, "Give me a moment on this one."], +])( + "idle after %s success=%s only describes reported success", + (name, succeeded, line) => { + const { policy, commentary } = setup(); + policy.toolStarted("one", name, 1_000); + policy.toolFinished("one", 2_000, succeeded); + expect(policy.evaluate(6_999)?.speak).toBe(false); + expect(policy.evaluate(7_000)?.speak).toBe(true); + expect(commentary).toHaveBeenCalledWith(line); + }, +); + +test("concurrent same-name calls and failed edits cannot imply completed edits", () => { + const { policy, commentary } = setup(); + policy.toolStarted("one", "addPlace", 1_000); + policy.toolStarted("two", "addPlace", 2_000); + policy.toolStarted("three", "addPlace", 3_000); + policy.toolFinished("one", 4_000, true); + policy.toolFinished("three", 5_000, true); + expect(policy.evaluate(6_600)?.phase.state).toBe("running"); + policy.toolFinished("two", 7_000, false); + policy.evaluate(36_600); + expect(commentary).not.toHaveBeenCalledWith( + "The edits are in. Give me a moment.", + ); +}); + +test("input streaming does not count as executing, and a new turn resets gates", () => { + const { policy, commentary } = setup(); + policy.toolStarted("one", "addPlace", 1_000, "preparing"); + expect(policy.evaluate(60_000)?.speak).toBe(false); + policy.endTurn(); + expect(policy.evaluate(61_000)).toBeNull(); + policy.startTurn(62_000); + expect(policy.evaluate(70_000)?.speak).toBe(false); + policy.acknowledged(71_000); + expect(policy.evaluate(77_000)?.speak).toBe(true); + expect(commentary).toHaveBeenCalledOnce(); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts new file mode 100644 index 00000000000..ae9d01baf14 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts @@ -0,0 +1,286 @@ +import { petrinautToolEffects } from "@hashintel/brunch-agent-plugin-sdcpn"; +import { brunchTools } from "@hashintel/brunch-agent/constants"; + +type Capability = "read" | "mutation" | "command" | "experiment" | "substrate"; +type ToolState = "preparing" | "running" | "awaiting-approval"; + +const capabilityOf = (name: string): Capability => { + if (name === brunchTools.draftPetrinautExperiment) return "experiment"; + if (Object.hasOwn(petrinautToolEffects, name)) { + const { capability } = + petrinautToolEffects[name as keyof typeof petrinautToolEffects]; + return { + "petrinaut-read": "read", + "petrinaut-mutation": "mutation", + "petrinaut-command": "command", + "petrinaut-experiment": "experiment", + }[capability] as Capability; + } + // Non-Petrinaut catalogue classes (substrate, ledger, explanation and + // diagnostic), and unknown tools, get the non-specific substrate phrase. + return "substrate"; +}; + +const phrases = { + read: [ + "Looking over the model as it stands.", + "Still reading through the model.", + ], + mutation: ["Making those changes now.", "Still working through the changes."], + command: ["Tidying up the layout.", "Still tidying the layout."], + experiment: [ + "Setting up the comparison.", + "Still setting up the comparison.", + ], + substrate: [ + "Working through the details.", + "Still working through the details.", + ], + thinking: ["Give me a moment on this one.", "Still thinking this through."], + "thinking-after-edits": [ + "The edits are in. Give me a moment.", + "Still thinking this through.", + ], + "thinking-after-read": [ + "I've had a look at the model. Give me a moment.", + "Still thinking this through.", + ], + "thinking-after-experiment": [ + "The comparison is drafted. One moment.", + "Still thinking this through.", + ], +} as const; + +interface Phase { + readonly phase: keyof typeof phrases | "awaiting-approval"; + readonly state: ToolState | "idle"; + readonly runningFor: number; + readonly idleFor: number; +} + +interface Tool { + readonly capability: Capability; + readonly draftsExperiment: boolean; + startedAt: number; + state: ToolState; + endedAt: number | null; + succeeded: boolean; +} + +interface Turn { + readonly startedAt: number; + ackAt: number | null; + readonly tools: Map; + readonly lines: { at: number; phase: Phase["phase"] }[]; + lastPhase: string | null; + settled: boolean; + userSpeaking: boolean; + liveSpeaking: boolean; +} + +interface Output { + readonly commentary: (line: string) => boolean; + readonly thinking: (context: { + progress: { + phase: Phase["phase"]; + state: Phase["state"]; + toolsSoFar: number; + sinceAckMs: number | null; + spokenLines: number; + }; + }) => void; + readonly diagnostic: ( + event: "progress.spoken" | "progress.suppressed", + metadata: Record, + ) => void; +} + +/** Prototype policy, driven entirely by observed tool transitions and supplied time. */ +export class ProgressPolicy { + readonly #out: Output; + #turn: Turn | null = null; + #lastSuppression: string | null = null; + + public constructor(out: Output) { + this.#out = out; + } + + public startTurn(now: number): void { + this.#turn = { + startedAt: now, + ackAt: null, + tools: new Map(), + lines: [], + lastPhase: null, + settled: false, + userSpeaking: false, + liveSpeaking: false, + }; + this.#lastSuppression = null; + } + + public acknowledged(now: number): void { + if (this.#turn) this.#turn.ackAt = now; + } + + /** Call identity, not name, keeps overlapping calls and repeated snapshots distinct. */ + public toolStarted( + id: string, + name: string, + now: number, + state: ToolState = "running", + ): void { + const turn = this.#turn; + if (!turn || turn.settled) return; + const previous = turn.tools.get(id); + if (previous) { + if (previous.endedAt !== null || previous.state === state) return; + previous.state = state; + previous.startedAt = now; + } else { + turn.tools.set(id, { + capability: capabilityOf(name), + draftsExperiment: name === brunchTools.draftPetrinautExperiment, + startedAt: now, + state, + endedAt: null, + succeeded: false, + }); + } + } + + public toolFinished(id: string, now: number, succeeded: boolean): void { + const tool = this.#turn?.tools.get(id); + if (tool && tool.endedAt === null) { + tool.endedAt = now; + tool.succeeded = succeeded; + } + } + + public userSpeaking(on: boolean): void { + if (this.#turn) this.#turn.userSpeaking = on; + } + public liveSpeaking(on: boolean): void { + if (this.#turn) this.#turn.liveSpeaking = on; + } + public settled(): void { + if (this.#turn) this.#turn.settled = true; + } + public endTurn(): void { + this.#turn = null; + } + + #phase(turn: Turn, now: number): Phase { + const tools = [...turn.tools.values()]; + const active = tools.filter((tool) => tool.endedAt === null); + // Any approval blocks narration, including when another call is executing. + if (active.some((tool) => tool.state === "awaiting-approval")) + return { + phase: "awaiting-approval", + state: "awaiting-approval", + runningFor: 0, + idleFor: 0, + }; + const latest = active.at(-1) ?? tools.at(-1); + if (active.length && latest) + return { + phase: latest.capability, + state: latest.state, + runningFor: now - latest.startedAt, + idleFor: 0, + }; + const mutations = tools.filter((tool) => tool.capability === "mutation"); + const phase = latest?.succeeded + ? latest.capability === "mutation" && + mutations.length >= 3 && + mutations.every((tool) => tool.succeeded) + ? "thinking-after-edits" + : latest.capability === "read" + ? "thinking-after-read" + : latest.draftsExperiment + ? "thinking-after-experiment" + : "thinking" + : "thinking"; + const lastActivity = Math.max( + turn.startedAt, + ...tools.flatMap((tool) => [tool.startedAt, tool.endedAt ?? -Infinity]), + ); + return { phase, state: "idle", runningFor: 0, idleFor: now - lastActivity }; + } + + #thinking(turn: Turn, info: Phase, now: number): void { + this.#out.thinking({ + progress: { + phase: info.phase, + state: info.state, + toolsSoFar: turn.tools.size, + sinceAckMs: turn.ackAt === null ? null : now - turn.ackAt, + spokenLines: turn.lines.length, + }, + }); + } + + public evaluate( + now: number, + ): { speak: boolean; phase: Phase; reason: string | null } | null { + const turn = this.#turn; + if (!turn) return null; + const info = this.#phase(turn, now); + if (!turn.settled && turn.lastPhase !== `${info.phase}|${info.state}`) { + this.#thinking(turn, info, now); + turn.lastPhase = `${info.phase}|${info.state}`; + } + const lastLine = turn.lines.at(-1); + const sinceLine = lastLine ? now - lastLine.at : Infinity; + const slowClass = info.phase === "experiment" || info.phase === "substrate"; + const checks: readonly (readonly [string, boolean])[] = [ + ["settled", !turn.settled], + ["person-speaking", !turn.userSpeaking], + ["live-speaking", !turn.liveSpeaking], + ["awaiting-approval", info.state !== "awaiting-approval"], + ["unacknowledged", turn.ackAt !== null], + ["quiet-after-ack", turn.ackAt !== null && now - turn.ackAt >= 6_000], + ["minimum-gap", sinceLine >= 20_000], + [ + "activity", + info.state === "running" + ? info.runningFor >= (slowClass ? 0 : 2_500) + : info.state === "idle" && info.idleFor >= 5_000, + ], + ["line-cap", turn.lines.length < 2], + ["same-phase", lastLine?.phase !== info.phase || sinceLine >= 30_000], + ]; + const reason = checks.find(([, passed]) => !passed)?.[0] ?? null; + if (reason !== null) { + const key = `${reason}|${info.phase}`; + if ( + this.#lastSuppression !== key && + turn.ackAt !== null && + !turn.settled && + !turn.liveSpeaking + ) { + this.#lastSuppression = key; + this.#out.diagnostic("progress.suppressed", { + reason, + phase: info.phase, + sinceAckMs: now - turn.ackAt, + }); + } + return { speak: false, phase: info, reason }; + } + if (info.phase === "awaiting-approval") + return { speak: false, phase: info, reason: "awaiting-approval" }; + const line = phrases[info.phase][lastLine?.phase === info.phase ? 1 : 0]; + // A refused append consumes its slot too: never retry or replay automatically. + turn.lines.push({ at: now, phase: info.phase }); + this.#thinking(turn, info, now); + const sent = this.#out.commentary(line); + this.#lastSuppression = null; + this.#out.diagnostic(sent ? "progress.spoken" : "progress.suppressed", { + phase: info.phase, + sinceAckMs: turn.ackAt === null ? null : now - turn.ackAt, + ...(sent ? {} : { reason: "send-failed" }), + }); + return { speak: sent, phase: info, reason: sent ? null : "send-failed" }; + } +} diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts index 981ecb919be..10ee18e15d0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts @@ -2,6 +2,70 @@ import { expect, test, vi } from "vitest"; import { LiveSpeechCaptions } from "./live-speech-captions"; +test.each(["before", "after"] as const)( + "progress accepted %s its transcript stays out of saved captions", + (timing) => { + const caption = vi.fn(); + const captions = new LiveSpeechCaptions(caption); + captions.input({ id: "input", text: "Build it", startMs: 100, endMs: 200 }); + captions.begin("request"); + captions.output({ + id: "ack", + text: "I'll build it.", + startMs: 300, + endMs: 600, + }); + if (timing === "before") captions.progress(6_000); + captions.output({ + id: "progress", + text: "Making those changes now.", + startMs: 6_100, + endMs: 7_000, + }); + if (timing === "after") captions.progress(6_000); + captions.progress(26_000); + captions.output({ + id: "still", + text: "Still working through the changes.", + startMs: 26_100, + endMs: 27_000, + }); + captions.wrapUp("request", 30_000); + captions.output({ + id: "result", + text: "The model is ready.", + startMs: 30_100, + endMs: 31_000, + }); + expect(caption.mock.calls.slice(-2)).toEqual([ + ["request", "reply", { text: "I'll build it.", state: "done" }], + [ + "request", + "wrapUp", + { text: "The model is ready.", state: "streaming" }, + ], + ]); + captions.speechStarted(); + captions.input({ + id: "next", + text: "Change it", + startMs: 32_000, + endMs: 33_000, + }); + captions.begin("next-request"); + captions.output({ + id: "next-ack", + text: "I'll change it.", + startMs: 34_000, + endMs: 35_000, + }); + expect(caption).toHaveBeenCalledWith("next-request", "reply", { + text: "I'll change it.", + state: "streaming", + }); + }, +); + test("streams time-ordered input once per event and retires the preview on finalization", () => { const input = { update: vi.fn<(id: string, text: string) => void>(), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts index af0f923cc76..704eb2367f0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts @@ -35,6 +35,7 @@ export class LiveSpeechCaptions { ) => void; readonly #windows: Window[] = []; readonly #fragments = new Map(); + readonly #progressStarts = new Set(); readonly #inputFragments = new Map(); readonly #seenInputIds = new Set(); readonly #input?: InputPreview; @@ -136,6 +137,12 @@ export class LiveSpeechCaptions { window.wrapUpMs = startMs; this.#publish(); } + /** Progress is spoken-only; the settled wrap-up reopens saved captions. */ + public progress(startMs: number): void { + if (this.#closed) return; + this.#progressStarts.add(startMs); + this.#publish(); + } public output(fragment: LiveTranscriptFragment): void { if (this.#closed || this.#fragments.has(fragment.id)) return; this.#fragments.set(fragment.id, fragment); @@ -155,10 +162,20 @@ export class LiveSpeechCaptions { ); for (const window of this.#windows) { if (!window.id) continue; + const progressStart = Math.min( + ...[...this.#progressStarts].filter( + (start) => + start >= window.startMs && + (window.endMs === undefined || start < window.endMs), + ), + ); const matching = fragments.filter( (fragment) => fragment.startMs >= window.startMs && - (window.endMs === undefined || fragment.startMs < window.endMs), + (window.endMs === undefined || fragment.startMs < window.endMs) && + (fragment.startMs < progressStart || + (window.wrapUpMs !== undefined && + fragment.startMs >= window.wrapUpMs)), ); const text = matching.map((fragment) => fragment.text).join(""); let replyEnd = text.length; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index cbcf6c0e74b..1fb1dafcb73 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -632,6 +632,7 @@ const AvailableVoiceInterviewControl = ({ const PinnedVoiceInterviewControl = ({ config, mediationHistory, + isToolAwaitingApproval, resolveInputSubmission, resolveResponseSubmission, settlements, @@ -645,6 +646,7 @@ const PinnedVoiceInterviewControl = ({ }: PetrinautAiVoiceModeContext & { readonly config: OpenAIVoiceConfig; readonly mediationHistory?: VoiceMediationHistory; + readonly isToolAwaitingApproval?: (toolCallId: string) => boolean; readonly resolveInputSubmission?: ResolveSubmission; readonly resolveResponseSubmission?: ResolveSubmissions; readonly settlements?: readonly VoiceSubmissionSettlement[]; @@ -671,6 +673,7 @@ const PinnedVoiceInterviewControl = ({ Date: Fri, 2 Oct 2026 09:14:35 +0200 Subject: [PATCH 2/5] Preserve wrap-up caption overlaps after progress Co-authored-by: Amp --- .../live-speech-captions.test.ts | 55 +++++++++++++++++++ .../voice-interview/live-speech-captions.ts | 3 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts index 10ee18e15d0..40177e622b3 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.test.ts @@ -66,6 +66,61 @@ test.each(["before", "after"] as const)( }, ); +test.each([ + { + split: "whole sentence", + acknowledgement: "I'll check. ", + reply: "I'll check. ", + wrapUp: "The duration is 2.5 hours, not 25.", + }, + { + split: "zero-offset fallback", + acknowledgement: "", + reply: "The duration is 2.", + wrapUp: "5 hours, not 25.", + }, +])( + "preserves wrap-up overlap after progress with the $split split", + ({ acknowledgement, reply, wrapUp }) => { + const lines = { reply: "", wrapUp: "" }; + const captions = new LiveSpeechCaptions((_id, kind, line) => { + lines[kind] = line.text; + }); + captions.input({ id: "input", text: "Check it", startMs: 100, endMs: 200 }); + captions.begin("turn"); + if (acknowledgement) { + captions.output({ + id: "ack", + text: acknowledgement, + startMs: 300, + endMs: 600, + }); + } + captions.progress(26_000); + captions.output({ + id: "progress", + text: "Still thinking this through.", + startMs: 26_100, + endMs: 30_000, + }); + captions.output({ + id: "overlap", + text: "The duration is 2.", + startMs: 29_900, + endMs: 30_100, + }); + captions.output({ + id: "rest", + text: "5 hours, not 25.", + startMs: 30_200, + endMs: 31_000, + }); + expect(lines).toEqual({ reply: acknowledgement, wrapUp: "" }); + captions.wrapUp("turn", 30_000); + expect(lines).toEqual({ reply, wrapUp }); + }, +); + test("streams time-ordered input once per event and retires the preview on finalization", () => { const input = { update: vi.fn<(id: string, text: string) => void>(), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts index 704eb2367f0..fdf81831ce6 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-speech-captions.ts @@ -175,7 +175,8 @@ export class LiveSpeechCaptions { (window.endMs === undefined || fragment.startMs < window.endMs) && (fragment.startMs < progressStart || (window.wrapUpMs !== undefined && - fragment.startMs >= window.wrapUpMs)), + (fragment.startMs >= window.wrapUpMs || + fragment.endMs > window.wrapUpMs))), ); const text = matching.map((fragment) => fragment.text).join(""); let replyEnd = text.length; From 9f6f52f04d68f9d93245817c1c93eac159162776 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 2 Oct 2026 09:50:50 +0200 Subject: [PATCH 3/5] Fix Live progress gates and pause handling Co-authored-by: Amp --- .../live-brunch-bridge.test.ts | 78 ++++++++++++++++++- .../app/voice-interview/live-brunch-bridge.ts | 13 +++- .../live-progress-policy.test.ts | 40 +++++++++- .../voice-interview/live-progress-policy.ts | 31 ++++++-- 4 files changed, 149 insertions(+), 13 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index dbe984b11fe..45f149d860c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -2508,7 +2508,16 @@ test("progress leaves the delegation open and settlement waits for Live speech", await vi.advanceTimersByTimeAsync(3_000); expect(fixture.appendCommentary).toHaveBeenCalledOnce(); fixture.bridge.liveSpeaking(false); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(1_000); + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + // A short acoustic pause is not completion. Resumed output extends the hold, + // even when it outlasts the initial five-second allowance for speech to start. + fixture.bridge.liveSpeaking(true); + await vi.advanceTimersByTimeAsync(2_000); + fixture.bridge.liveSpeaking(false); + await vi.advanceTimersByTimeAsync(1_499); + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(fixture.appendCommentary).toHaveBeenLastCalledWith( segment().text, "delegation", @@ -2519,6 +2528,73 @@ test("progress leaves the delegation open and settlement waits for Live speech", expect(vi.getTimerCount()).toBe(0); }); +test.each(["silent", "delayed"] as const)( + "settlement waits for %s progress output with a bounded silent fallback", + async (output) => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Compare staffing")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming", messages: [runningTool()] }); + await vi.advanceTimersByTimeAsync(6_000); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Setting up the comparison.", + null, + ); + wrapUp(fixture); + await vi.advanceTimersByTimeAsync(4_000); + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + if (output === "delayed") { + fixture.bridge.liveSpeaking(true); + await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + fixture.bridge.liveSpeaking(false); + await vi.advanceTimersByTimeAsync(1_499); + } else { + await vi.advanceTimersByTimeAsync(999); + } + expect(fixture.appendCommentary).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + expect(fixture.appendCommentary).toHaveBeenLastCalledWith( + segment().text, + "delegation", + ); + fixture.bridge.stop(); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test.each(["speech", "stop", "teardown"] as const)( + "%s cancels the summary while progress speech is held", + async (cause) => { + vi.useFakeTimers(); + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept(speech("one", "Compare staffing")); + fixture.speechPending.mockReturnValue(false); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming", messages: [runningTool()] }); + await vi.advanceTimersByTimeAsync(6_000); + fixture.bridge.liveSpeaking(true); + wrapUp(fixture); + await vi.advanceTimersByTimeAsync(1_000); + if (cause === "speech") fixture.bridge.speechStarted(); + else if (cause === "stop") fixture.bridge.stopResponse(); + else fixture.bridge.stop(); + fixture.bridge.liveSpeaking(false); + await vi.advanceTimersByTimeAsync(10_000); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Setting up the comparison.", + null, + ); + expect(fixture.mediation.offered).not.toHaveBeenCalled(); + fixture.bridge.stop(); + expect(vi.getTimerCount()).toBe(0); + }, +); + test("approval changes are polled even without a new message snapshot", async () => { vi.useFakeTimers(); const fixture = setup(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts index 9013c795474..1b50833fc11 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -143,6 +143,7 @@ export class LiveBrunchBridge { #progressTimer: ReturnType | undefined; #liveSpeaking = false; #progressPendingUntil = 0; + #liveSpeechHoldUntil = 0; #chat: Chat = { canAcceptVoiceInput: false, segments: [], @@ -157,13 +158,15 @@ export class LiveBrunchBridge { /** Acoustic activity is a hold, not proof of provider playback completion. */ public liveSpeaking(on: boolean): void { if (this.#liveSpeaking && !on) { + // Live has no playback-complete event. Require sustained quiet rather + // than releasing a pending summary at the first pause in a sentence. + this.#liveSpeechHoldUntil = Date.now() + 1_500; for (const turn of this.#turns) { if (turn.delegationId !== null && !turn.progressOffered) turn.progress.acknowledged(Date.now()); } } this.#liveSpeaking = on; - if (on) this.#progressPendingUntil = 0; } #stopProgress(): void { @@ -242,7 +245,9 @@ export class LiveBrunchBridge { } turn.progress.userSpeaking(this.#dependencies.speechPending()); turn.progress.liveSpeaking( - this.#liveSpeaking || now < this.#progressPendingUntil, + this.#liveSpeaking || + now < this.#progressPendingUntil || + now < this.#liveSpeechHoldUntil, ); turn.progress.evaluate(now); } @@ -1084,7 +1089,9 @@ export class LiveBrunchBridge { ); while ( turn.progressOffered && - (this.#liveSpeaking || Date.now() < this.#progressPendingUntil) && + (this.#liveSpeaking || + Date.now() < this.#progressPendingUntil || + Date.now() < this.#liveSpeechHoldUntil) && !turn.preparation.signal.aborted && !this.#abort.signal.aborted ) { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts index d0b96b9b152..d9cec61370e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.test.ts @@ -164,7 +164,8 @@ test.each([ ["createExperiment", "Setting up the comparison."], ["draft_petrinaut_experiment", "Setting up the comparison."], ["task", "Working through the details."], - ["mutate_workpiece", "Working through the details."], + ["activate_skill", "Working through the details."], + ["read_skill_resource", "Working through the details."], ])("%s qualifies immediately once acknowledged", (name, line) => { const { policy, commentary } = setup(); policy.toolStarted("one", name, 7_000); @@ -172,6 +173,43 @@ test.each([ expect(commentary).toHaveBeenCalledWith(line); }); +test.each([ + "mutate_workpiece", + "read_workpiece", + "query_workpiece", + "ping", + "unknown_tool", +])("%s keeps the normal running delay", (name) => { + const { policy, commentary } = setup(); + policy.toolStarted("one", name, 7_000); + expect(policy.evaluate(7_000)?.reason).toBe("activity"); + expect(policy.evaluate(9_499)?.reason).toBe("activity"); + expect(policy.evaluate(9_500)?.speak).toBe(true); + expect(commentary).toHaveBeenCalledExactlyOnceWith( + "Working through the details.", + ); +}); + +test("a fast non-substrate burst after acknowledgement stays silent", () => { + const { policy, commentary } = setup(); + for (const [index, name] of [ + "read_workpiece", + "mutate_workpiece", + "query_workpiece", + "ping", + "unknown_tool", + ].entries()) { + const now = 6_600 + index * 200; + policy.toolStarted(name, name, now); + policy.evaluate(now); + policy.toolFinished(name, now + 100, true); + policy.evaluate(now + 100); + } + policy.settled(); + policy.evaluate(20_000); + expect(commentary).not.toHaveBeenCalled(); +}); + test("a changed phase still waits twenty seconds and Live speech holds it", () => { const { policy } = setup(); policy.evaluate(6_600); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts index ae9d01baf14..728c4510295 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-progress-policy.ts @@ -1,11 +1,23 @@ import { petrinautToolEffects } from "@hashintel/brunch-agent-plugin-sdcpn"; import { brunchTools } from "@hashintel/brunch-agent/constants"; -type Capability = "read" | "mutation" | "command" | "experiment" | "substrate"; +type Capability = + | "read" + | "mutation" + | "command" + | "experiment" + | "substrate" + | "other"; type ToolState = "preparing" | "running" | "awaiting-approval"; const capabilityOf = (name: string): Capability => { if (name === brunchTools.draftPetrinautExperiment) return "experiment"; + if ( + name === "task" || + name === brunchTools.activateSkill || + name === brunchTools.readSkillResource + ) + return "substrate"; if (Object.hasOwn(petrinautToolEffects, name)) { const { capability } = petrinautToolEffects[name as keyof typeof petrinautToolEffects]; @@ -16,11 +28,16 @@ const capabilityOf = (name: string): Capability => { "petrinaut-experiment": "experiment", }[capability] as Capability; } - // Non-Petrinaut catalogue classes (substrate, ledger, explanation and - // diagnostic), and unknown tools, get the non-specific substrate phrase. - return "substrate"; + // Ledger, explanation, diagnostic and unknown tools share a generic phrase, + // but must not inherit substrate's immediate running gate. + return "other"; }; +const genericToolPhrases = [ + "Working through the details.", + "Still working through the details.", +] as const; + const phrases = { read: [ "Looking over the model as it stands.", @@ -32,10 +49,8 @@ const phrases = { "Setting up the comparison.", "Still setting up the comparison.", ], - substrate: [ - "Working through the details.", - "Still working through the details.", - ], + substrate: genericToolPhrases, + other: genericToolPhrases, thinking: ["Give me a moment on this one.", "Still thinking this through."], "thinking-after-edits": [ "The edits are in. Give me a moment.", From b45a7758244af460bdc33ddb8fd03644f0ce970e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 2 Oct 2026 17:07:32 +0200 Subject: [PATCH 4/5] Ignore Live progress accepted after the person speaks again Co-authored-by: Cursor --- .../live-conversation-control.test.tsx | 72 +++++++++++++++++-- .../live-conversation-control.tsx | 10 ++- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 2a3b1704214..59d45a50a31 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -341,6 +341,68 @@ test("captions a Live turn in the history it began in after a switch", async () expect(second.project([])).toEqual([]); }); +test("progress accepted after the person speaks again does not hide the new reply", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => new Promise(() => {})), + ); + const props = context(); + const history = new VoiceMediationHistory("standalone"); + render( + , + ); + await start(); + const call = vi.mocked(createLiveConversation).mock.lastCall!; + const progress = { + eventId: "progress", + kind: "commentary" as const, + delegationId: null, + }; + act(() => { + call[6]?.started(); + call[6]?.input({ + id: "one", + text: "Compare four", + startMs: 100, + endMs: 300, + }); + call[2]({ id: "first", text: "Compare four", startedDuringOutput: false }); + call[4]({ ...progress, status: "unknown" }); + call[6]?.started(); + call[6]?.input({ + id: "two", + text: "Make it seven", + startMs: 1000, + endMs: 1200, + }); + call[2]({ + id: "second", + text: "Make it seven", + startedDuringOutput: false, + }); + call[4]({ ...progress, status: "accepted", startMs: 1300 }); + call[6]?.output({ + id: "reply", + text: "Seven it is.", + startMs: 1400, + endMs: 1600, + }); + }); + expect( + history.project([]).find((message) => message.id === "voice-reply:second") + ?.parts, + ).toEqual([ + { + type: "data-voiceAgentReply", + data: { text: "Seven it is.", state: "streaming" }, + }, + ]); +}); + test("retires a Live preview from the history it began in after a switch", async () => { const { call, first, second, switchConversation } = await renderSwitchingLive(); @@ -1894,13 +1956,13 @@ test("Live progress observes the host approval gate and stays out of written voi null, ); act(() => { - call[4]({ + const progress = { eventId: "progress", - kind: "commentary", + kind: "commentary" as const, delegationId: null, - status: "accepted", - startMs: 46_000, - }); + }; + call[4]({ ...progress, status: "unknown" }); + call[4]({ ...progress, status: "accepted", startMs: 46_000 }); call[6]?.output({ id: "progress-output", text: "Making those changes now.", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index 9a0fe978018..62d8b88725e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -242,6 +242,9 @@ export const LiveConversationControl = ({ ); let offeredInput: string | undefined; const appendInputs = new Map(); + // Appends sent before the person last started speaking belong to the + // interrupted turn; a late acceptance must not caption the new one. + const turnAppends = new Set(); const next = createLiveConversation( (nextState) => { if (session.current !== next) return; @@ -290,7 +293,10 @@ export const LiveConversationControl = ({ }, (result) => { if (session.current !== next) return; - if (result.kind === "commentary") { + if (result.status === "unknown") turnAppends.add(result.eventId); + const currentTurn = turnAppends.has(result.eventId); + if (result.status !== "unknown") turnAppends.delete(result.eventId); + if (currentTurn && result.kind === "commentary") { if (offeredInput) { appendInputs.set(result.eventId, offeredInput); offeredInput = undefined; @@ -303,6 +309,7 @@ export const LiveConversationControl = ({ } if (result.status !== "unknown") appendInputs.delete(result.eventId); } else if ( + currentTurn && result.kind === "instructions" && result.delegationId === null && result.status === "accepted" && @@ -334,6 +341,7 @@ export const LiveConversationControl = ({ bridge.current?.speechStarted(); offeredInput = undefined; appendInputs.clear(); + turnAppends.clear(); }, input: (fragment) => { if (session.current === next) captions.input(fragment); From 18488e58b6ecf546c3782b2fa38311a91b186438 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 2 Oct 2026 19:20:26 +0200 Subject: [PATCH 5/5] Keep consuming the offered wrap-up when a Live append fails locally Co-authored-by: Cursor --- .../live-conversation-control.test.tsx | 45 ++++++++++++++++++- .../live-conversation-control.tsx | 4 +- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 59d45a50a31..a88776f223f 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -1233,7 +1233,7 @@ const mockMediation = () => { return fetch; }; -test("the prepared brief enters the real admission helper and only its settled canonical prose is summarized for Live", async () => { +const offerSettledWrapUp = async () => { const fetch = mockMediation(); const tracker = new BrunchPanelConversationTracker(); const history = new VoiceMediationHistory("test"); @@ -1370,6 +1370,12 @@ test("the prepared brief enters the real admission helper and only its settled c }), }), ); + return { call, history, messages, props, session, tracker }; +}; + +test("the prepared brief enters the real admission helper and only its settled canonical prose is summarized for Live", async () => { + const { call, history, messages, props, session, tracker } = + await offerSettledWrapUp(); act(() => { const append = { eventId: "summary", @@ -1423,6 +1429,43 @@ test("the prepared brief enters the real admission helper and only its settled c await waitFor(() => expect(props.submitVoiceInput).toHaveBeenCalledTimes(2)); }); +test("a summary that could not be sent does not caption later progress as its wrap-up", async () => { + const { call, history, messages } = await offerSettledWrapUp(); + const progress = { + eventId: "progress", + kind: "commentary" as const, + delegationId: null, + }; + act(() => { + call[4]({ + eventId: "summary", + kind: "commentary", + delegationId: "delegation-1", + status: "local-failure", + }); + call[4]({ ...progress, status: "unknown" }); + call[4]({ ...progress, status: "accepted", startMs: 500 }); + call[6]?.output({ + id: "spoken", + text: "Still checking the reviewers.", + startMs: 600, + endMs: 900, + }); + }); + expect( + history + .project([ + { + id: "utterance-1", + role: "user", + parts: [{ type: "text", text: "Canonical brief" }], + }, + ...messages, + ]) + .map((entry) => entry.id), + ).not.toContain("voice-wrap-up:utterance-1"); +}); + test("Stop sends the partial answer to Live quietly and Continue admits one new turn", async () => { const fetch = mockMediation(); const tracker = new BrunchPanelConversationTracker(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index 62d8b88725e..7291e5e72ff 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -294,7 +294,9 @@ export const LiveConversationControl = ({ (result) => { if (session.current !== next) return; if (result.status === "unknown") turnAppends.add(result.eventId); - const currentTurn = turnAppends.has(result.eventId); + // A local failure is reported synchronously, without a prior unknown. + const currentTurn = + result.status === "local-failure" || turnAppends.has(result.eventId); if (result.status !== "unknown") turnAppends.delete(result.eventId); if (currentTurn && result.kind === "commentary") { if (offeredInput) {