diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 86990c2bfc7..1bf3b44f813 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -261,14 +261,14 @@ gets through, open DevTools, enable the **Verbose** console level and filter by it. - `input.ignored` with `reason: "short-during-output"` marks a short transcript that started during output and was not sent to Brunch. -- `filter.shadow` with `stage: "echo"` marks a transcript the echo check would +- `filter.shadow` with `reason: "echo"` marks a transcript the echo check would skip. The check runs in shadow, so the transcript is still handled as before and can also appear as `input.ignored`. It compares speech that overlapped output, in memory only, with Live's output transcript from three seconds before the speech started until it stopped. Six or more words count as a repeat when they mostly match Live's words in order; fewer count only when they appear together, in order, in Live's words. -- `filter.shadow` with `stage: "doubtful-short-during-output"` tries a narrower +- `filter.shadow` with `reason: "doubtful-short-during-output"` tries a narrower three-word rule: short speech during output counts only when its least likely token has a log probability below -1.9 (`minLogprob`) or it repeats Live's words as the echo check defines them. The three-word rule still diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index e452caac022..c99d7624d89 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -1384,13 +1384,9 @@ test("an input repeating Live's overlapping words is traced by the echo stage in expect( records.filter((record) => record.event === "filter.shadow"), ).toMatchObject([ - { inputId: "echo", stage: "echo", reason: "echo" }, - { inputId: "short-echo", stage: "echo", reason: "echo" }, - { - inputId: "short-echo", - stage: "doubtful-short-during-output", - reason: "doubtful-short-during-output", - }, + { inputId: "echo", reason: "echo" }, + { inputId: "short-echo", reason: "echo" }, + { inputId: "short-echo", reason: "doubtful-short-during-output" }, ]); expect( records.filter((record) => record.event === "input.ignored"), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.test.ts index 395819c8e5b..6233ed565b9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.test.ts @@ -24,17 +24,19 @@ const input = { id: "one", text: "PRIVATE okay", startedDuringOutput: true }; test("sends when no stage skips", () => { expect( - routeUtterance(input, [{ name: "probe", mode: "on", skip: () => null }]), + routeUtterance(input, [ + { reason: "empty", mode: "on", skips: () => false }, + ]), ).toBeNull(); }); test("the first stage that is on and skips decides; later stages never run", () => { - const later = vi.fn(() => "empty" as const); + const later = vi.fn(() => true); expect( routeUtterance(input, [ - { name: "quiet", mode: "on", skip: () => null }, - { name: "decides", mode: "on", skip: () => "short-during-output" }, - { name: "later", mode: "on", skip: later }, + { reason: "echo", mode: "on", skips: () => false }, + { reason: "short-during-output", mode: "on", skips: () => true }, + { reason: "empty", mode: "on", skips: later }, ]), ).toBe("short-during-output"); expect(later).not.toHaveBeenCalled(); @@ -45,20 +47,19 @@ test("a shadow stage traces what it would skip without deciding", () => { const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); expect( routeUtterance(input, [ - { name: "trial", mode: "shadow", skip: () => "empty" }, - { name: "decides", mode: "on", skip: () => "short-during-output" }, + { reason: "empty", mode: "shadow", skips: () => true }, + { reason: "short-during-output", mode: "on", skips: () => true }, ]), ).toBe("short-during-output"); const records = traceRecords(debug.mock.calls); expect(records).toMatchObject([ - { event: "filter.shadow", inputId: "one", stage: "trial", reason: "empty" }, + { event: "filter.shadow", inputId: "one", reason: "empty" }, ]); expect(Object.keys(records[0] ?? {}).sort()).toEqual([ "at", "event", "inputId", "reason", - "stage", ]); expect(JSON.stringify(debug.mock.calls)).not.toContain("PRIVATE"); }); @@ -68,22 +69,22 @@ test("a shadow stage that would not skip leaves no trace", () => { const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); expect( routeUtterance(input, [ - { name: "trial", mode: "shadow", skip: () => null }, + { reason: "empty", mode: "shadow", skips: () => false }, ]), ).toBeNull(); expect(debug).not.toHaveBeenCalled(); }); test("a stage that is off never runs", () => { - const off = vi.fn(() => "empty" as const); + const off = vi.fn(() => true); expect( - routeUtterance(input, [{ name: "off", mode: "off", skip: off }]), + routeUtterance(input, [{ reason: "empty", mode: "off", skips: off }]), ).toBeNull(); expect(off).not.toHaveBeenCalled(); }); test("stages see Live's words when the speech overlapped its output", () => { - const skip = vi.fn(() => null); + const skips = vi.fn(() => false); routeUtterance( { id: "one", @@ -91,9 +92,9 @@ test("stages see Live's words when the speech overlapped its output", () => { startedDuringOutput: true, liveOutputText: "Okay?", }, - [{ name: "probe", mode: "on", skip }], + [{ reason: "empty", mode: "on", skips }], ); - expect(skip).toHaveBeenCalledWith({ + expect(skips).toHaveBeenCalledWith({ inputId: "one", text: "Okay", words: 1, @@ -109,11 +110,11 @@ test.each([ ])( "stages see %j with its word count, counting contractions as one word", (text, startedDuringOutput, words) => { - const skip = vi.fn(() => null); + const skips = vi.fn(() => false); routeUtterance({ id: "one", text, startedDuringOutput }, [ - { name: "probe", mode: "on", skip }, + { reason: "empty", mode: "on", skips }, ]); - expect(skip).toHaveBeenCalledWith({ + expect(skips).toHaveBeenCalledWith({ inputId: "one", text, words, @@ -123,12 +124,12 @@ test.each([ ); test("stages see the transcript's lowest token log probability", () => { - const skip = vi.fn(() => null); + const skips = vi.fn(() => false); routeUtterance( { id: "one", text: "Okay", startedDuringOutput: false, minLogprob: -2.5 }, - [{ name: "probe", mode: "on", skip }], + [{ reason: "empty", mode: "on", skips }], ); - expect(skip).toHaveBeenCalledWith({ + expect(skips).toHaveBeenCalledWith({ inputId: "one", text: "Okay", words: 1, @@ -141,7 +142,7 @@ const doubtfulShadows = (calls: readonly (readonly unknown[])[]) => traceRecords(calls).filter( (record) => record.event === "filter.shadow" && - record.stage === "doubtful-short-during-output", + record.reason === "doubtful-short-during-output", ); test.each([ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.ts index 1556bcdb8a6..15a738f7268 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge/utterance-pipeline.ts @@ -22,10 +22,10 @@ export interface Utterance { } export interface UtteranceStage { - readonly name: string; + readonly reason: SkipReason; /** A shadow stage traces what it would skip and never decides. */ readonly mode: "on" | "shadow" | "off"; - readonly skip: (utterance: Utterance) => SkipReason | null; + readonly skips: (utterance: Utterance) => boolean; } export const doubtfulBelowLogprob = -1.9; @@ -40,35 +40,32 @@ const isShortDuringOutput = ({ startedDuringOutput, words }: Utterance) => export const liveUtteranceStages: readonly UtteranceStage[] = [ // Leaked Live audio can finalize as a longer repeat of Live's own words. { - name: "echo", + reason: "echo", mode: "shadow", - skip: (utterance) => (repeatsOverlappingOutput(utterance) ? "echo" : null), + skips: repeatsOverlappingOutput, }, // Only short speech during output that also looks invented or repeats Live; // a shadow stage must run before the on stage that skips the same input. { - name: "doubtful-short-during-output", + reason: "doubtful-short-during-output", mode: "shadow", - skip: (utterance) => + skips: (utterance) => isShortDuringOutput(utterance) && ((utterance.minLogprob !== undefined && utterance.minLogprob < doubtfulBelowLogprob) || - repeatsOverlappingOutput(utterance)) - ? "doubtful-short-during-output" - : null, + repeatsOverlappingOutput(utterance)), }, // Leaked Live audio finalizes as phantoms of a few words. { - name: "short-during-output", + reason: "short-during-output", mode: "on", - skip: (utterance) => - isShortDuringOutput(utterance) ? "short-during-output" : null, + skips: isShortDuringOutput, }, // Transcription can finalize noise as punctuation alone, such as ".". { - name: "empty", + reason: "empty", mode: "on", - skip: ({ words }) => (words === 0 ? "empty" : null), + skips: ({ words }) => words === 0, }, ]; @@ -91,14 +88,11 @@ export const routeUtterance = ( minLogprob: input.minLogprob, }; for (const stage of stages) { - if (stage.mode === "off") continue; - const reason = stage.skip(utterance); - if (reason === null) continue; - if (stage.mode === "on") return reason; + if (stage.mode === "off" || !stage.skips(utterance)) continue; + if (stage.mode === "on") return stage.reason; logLiveDiagnostic("filter.shadow", { inputId: utterance.inputId, - stage: stage.name, - reason, + reason: stage.reason, }); } return null; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts index 34f60e89d22..6219e94cc40 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -1485,7 +1485,7 @@ test("telemetry shows activity but silence and late samples never settle or revi await vi.advanceTimersByTimeAsync(1); expect(fixture.onState.mock.lastCall?.[0].activity?.outputActive).toBe(false); expect(fixture.onFinalizedInput).not.toHaveBeenCalled(); - getStats.mockRejectedValueOnce(new Error("Optional telemetry failed")); + getStats.mockRejectedValueOnce(new Error("Stats unavailable")); await vi.advanceTimersByTimeAsync(100); expect(fixture.input.stop).not.toHaveBeenCalled(); getStats.mockResolvedValueOnce( @@ -1756,66 +1756,6 @@ test("summarises echo evidence for each stretch of audible Live output without p await stopped; }); -test("does not count speech as started during output while the speaker is muted or at zero volume", async () => { - vi.useFakeTimers(); - const fixture = setup(); - Object.assign(fixture.peers[0]!, { - getStats: vi.fn( - async (): Promise> => - new Map([ - ["output", { type: "inbound-rtp", kind: "audio", audioLevel: 0.2 }], - ]), - ), - }); - await connect(fixture); - fixture.peers[0]!.dispatchEvent( - Object.assign(new Event("track"), { - track: fixture.outputs[0], - streams: [fixture.stream], - }), - ); - const speakers = [ - { itemId: "muted", muted: true, volume: 1 }, - { itemId: "silent", muted: false, volume: 0 }, - { itemId: "audible", muted: false, volume: 1 }, - ]; - for (const { itemId, muted, volume } of speakers) { - fixture.conversation.setSpeakerMuted(muted); - fixture.conversation.setSpeakerVolume(volume); - await vi.advanceTimersByTimeAsync(100); - fixture.emit(1, { - type: "input_audio_buffer.speech_started", - item_id: itemId, - }); - } - let previousItemId: string | null = null; - for (const { itemId } of speakers) { - fixture.emit(1, { - type: "input_audio_buffer.committed", - item_id: itemId, - previous_item_id: previousItemId, - }); - previousItemId = itemId; - } - for (const { itemId } of speakers) { - fixture.emit(1, { - type: "conversation.item.input_audio_transcription.completed", - item_id: itemId, - content_index: 0, - transcript: "Yes.", - }); - } - - expect( - fixture.onFinalizedInput.mock.calls.map( - ([input]) => input.startedDuringOutput, - ), - ).toEqual([false, false, true]); - const stopped = fixture.conversation.stop(); - fixture.emit(0, { type: "session.closed" }); - await stopped; -}); - test("records transcription confidence on input.finalized from numbers only", async () => { vi.stubEnv("DEV", true); const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); @@ -2067,7 +2007,7 @@ test("passes Live's recent words only with speech that overlapped its audible ou await stopped; }); -test("does not count speech as overlapping output while the speaker is muted or at zero volume", async () => { +test("does not count speech as started during or overlapping output while the speaker is muted or at zero volume", async () => { vi.useFakeTimers(); const fixture = setup(); Object.assign(fixture.peers[0]!, { @@ -2123,6 +2063,11 @@ test("does not count speech as overlapping output while the speaker is muted or }); } + expect( + fixture.onFinalizedInput.mock.calls.map( + ([input]) => input.startedDuringOutput, + ), + ).toEqual([false, false, true]); expect( fixture.onFinalizedInput.mock.calls.map(([input]) => input.liveOutputText), ).toEqual([undefined, undefined, "Yes."]); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts index d500bb50229..e3044791b60 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts @@ -321,7 +321,9 @@ export const createLiveConversation = ( if (report.type === "inbound-rtp") outputLevel = report.audioLevel; }); } catch { - // Optional telemetry must not affect the session lifetime. + // A failed sample reads as inaudible output. The short-during-output + // filter and echo checks depend on this sampling, but its failure must + // not end the session. } if (abort.signal.aborted) return; activityTimer = setTimeout(() => void sampleActivity(), 100); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-mediation-history.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-mediation-history.ts index bdf024ab41a..c90555bb429 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-mediation-history.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-mediation-history.ts @@ -17,7 +17,6 @@ const turnSchema = z.object({ submissionId: z.string().optional(), canonicalId: z.string().optional(), responseIds: z.array(z.string()), - anchorOnly: z.boolean().optional(), fields: z.record(z.string(), z.string()).optional(), preparationFailed: z.boolean().optional(), reply: lineSchema.optional(), @@ -52,7 +51,7 @@ export class VoiceMediationHistory { .max(100) .parse(JSON.parse(storage?.getItem(this.#key) ?? "[]")); for (const turn of saved) { - if (!turn.submissionId && !turn.anchorOnly) continue; + if (!turn.submissionId) continue; if (turn.reply) turn.reply.state = "done"; if (turn.wrapUp) turn.wrapUp.state = "done"; this.#turns.set(turn.id, turn); @@ -77,7 +76,7 @@ export class VoiceMediationHistory { this.#key, JSON.stringify( [...this.#turns.values()] - .filter((turn) => turn.submissionId || turn.anchorOnly) + .filter((turn) => turn.submissionId) .slice(-100), ), );