diff --git a/crates/agent-gui/src/lib/cancellation/abortRace.ts b/crates/agent-gui/src/lib/cancellation/abortRace.ts new file mode 100644 index 000000000..f4720a1c5 --- /dev/null +++ b/crates/agent-gui/src/lib/cancellation/abortRace.ts @@ -0,0 +1,43 @@ +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("Aborted", "AbortError"); +} + +/** + * Resolves/rejects with `operation` unless `signal` aborts first. The source + * promise remains observed after an abort, so a late rejection never becomes + * an unhandled rejection. + */ +export function raceWithAbort(operation: PromiseLike | T, signal?: AbortSignal): Promise { + const source = Promise.resolve(operation); + if (!signal) return source; + if (signal.aborted) { + // The caller may already have started an IPC/network promise before it + // checks cancellation. Keep that promise observed even though this race + // has already been decided, otherwise a late rejection becomes unhandled. + void source.catch(() => undefined); + return Promise.reject(abortReason(signal)); + } + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: (value: T) => void, value: T) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(value); + }; + const fail = (error: unknown) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + reject(error); + }; + const onAbort = () => fail(abortReason(signal)); + + signal.addEventListener("abort", onAbort, { once: true }); + source.then( + (value) => finish(resolve, value), + (error) => fail(error), + ); + }); +} diff --git a/crates/agent-gui/src/lib/chat/compaction/controller.ts b/crates/agent-gui/src/lib/chat/compaction/controller.ts index 1c050675f..f2af7d374 100644 --- a/crates/agent-gui/src/lib/chat/compaction/controller.ts +++ b/crates/agent-gui/src/lib/chat/compaction/controller.ts @@ -5,6 +5,7 @@ import { positiveTokenCount, } from "@liveagent/ui/lib/chat/contextUsage"; import type { PendingUploadedFile } from "@liveagent/ui/lib/chat/uploadedFiles"; +import { raceWithAbort } from "../../cancellation/abortRace"; import type { StreamDebugLogger } from "../../debug/agentDebug"; import type { ProviderId } from "../../settings"; import { type ConversationViewState, getActiveSegment } from "../conversation/conversationState"; @@ -87,6 +88,8 @@ export type CompactionTurnBinding = { cancellation: TurnCancellation; debugLogger?: StreamDebugLogger; complete?: CompleteAssistantFn; + /** Observer owned by this turn; prevents a late old turn from reporting into a replacement. */ + observer?: CompactionObserver; sinks: CompactionSinks; buildPreparedContext: ( state: ConversationViewState, @@ -186,6 +189,11 @@ export class CompactionController { private pressure = createCompactionPressure(); private readonly ledger = new TokenLedger(); private binding: CompactionTurnBinding | null = null; + // A conversation can start a replacement turn after force-stop while an old + // provider task is still unwinding. The lease keeps that old task from + // clearing or rolling back the replacement turn's binding. + private bindingGeneration = 0; + private activeBindingGeneration: number | null = null; private rollbackSnapshot: RollbackSnapshot | null = null; private inFlight = false; private statusPhase: CompactionStatus["phase"] = "idle"; @@ -213,18 +221,30 @@ export class CompactionController { bindTurn(binding: CompactionTurnBinding) { // A defensive rebind must not strand the previous observer interval. this.settleAbortedIfRunning(); + const generation = ++this.bindingGeneration; this.binding = binding; + this.activeBindingGeneration = generation; this.rollbackSnapshot = null; this.inFlight = false; + return generation; } - unbindTurn() { + isTurnBound(generation: number) { + return this.binding !== null && this.activeBindingGeneration === generation; + } + + unbindTurn(expectedGeneration?: number) { + if (expectedGeneration !== undefined && !this.isTurnBound(expectedGeneration)) { + return false; + } // Every published start receives exactly one terminal notification, even when a caller // tears down the turn without first reaching the ordinary completion path. this.settleAbortedIfRunning(); this.binding = null; + this.activeBindingGeneration = null; this.rollbackSnapshot = null; this.inFlight = false; + return true; } get stats() { @@ -235,7 +255,14 @@ export class CompactionController { binding: CompactionTurnBinding, state: ConversationViewState, ): Promise { - const persisted = await binding.sinks.persist?.(state); + // Checkpoint durability matters, but an already-started write must not + // keep a Stop request from releasing the run. raceWithAbort continues to + // observe the underlying write after cancellation, so a late failure is + // not left as an unhandled rejection. + const persisted = await raceWithAbort( + binding.sinks.persist?.(state), + binding.cancellation.userStop.signal, + ); if (persisted === false || persisted === null) { throw new Error("compaction checkpoint persistence failed"); } @@ -324,8 +351,10 @@ export class CompactionController { includeUploadedFilesMetadata?: boolean; }): Promise { const binding = this.binding; + const bindingGeneration = this.activeBindingGeneration; const presend = binding?.presend; - if (!binding || !presend) return false; + if (!binding || !presend || bindingGeneration === null) return false; + const ownsBinding = () => this.isTurnBound(bindingGeneration); if (binding.cancellation.userStop.signal.aborted) { throw createCompactionAbortError(); } @@ -420,6 +449,9 @@ export class CompactionController { ); return true; } catch (error) { + if (!ownsBinding()) { + throw createCompactionAbortError(); + } if (this.isAbortOutcome(scope.controller.signal, error)) { throw error; } @@ -441,8 +473,10 @@ export class CompactionController { return false; } finally { scope.release(); - this.inFlight = false; - this.binding?.sinks.setBridgeToolStatus?.(null); + if (ownsBinding()) { + this.inFlight = false; + this.binding?.sinks.setBridgeToolStatus?.(null); + } } } @@ -458,9 +492,11 @@ export class CompactionController { manualContextUsage?: ManualContextUsageSnapshot; }): Promise { const binding = this.binding; - if (!binding) { + const bindingGeneration = this.activeBindingGeneration; + if (!binding || bindingGeneration === null) { return { context: null, shouldDisableProtection: false, outcome: "skipped" }; } + const ownsBinding = () => this.isTurnBound(bindingGeneration); // 覆盖"mid-stream abort 后、summarizer 启动前"用户恰好点停止的间隙。 if (binding.cancellation.userStop.signal.aborted) { throw createCompactionAbortError(); @@ -594,6 +630,9 @@ export class CompactionController { ); return { context: resumeContext, shouldDisableProtection: false, outcome: "compacted" }; } catch (error) { + if (!ownsBinding()) { + throw createCompactionAbortError(); + } if (this.isAbortOutcome(scope.controller.signal, error)) { throw error; } @@ -630,8 +669,10 @@ export class CompactionController { : { context: null, shouldDisableProtection: false, outcome: "failed" }; } finally { scope.release(); - this.inFlight = false; - this.binding?.sinks.setBridgeToolStatus?.(null); + if (ownsBinding()) { + this.inFlight = false; + this.binding?.sinks.setBridgeToolStatus?.(null); + } } } @@ -653,7 +694,7 @@ export class CompactionController { }, ): Promise { if (this.binding || this.inFlight) return { status: "busy" }; - this.bindTurn(binding); + const bindingGeneration = this.bindTurn(binding); try { const probe = this.probeManualDecision(binding, state, contextUsage, options?.tools); if (!probe.shouldCompact) { @@ -681,12 +722,12 @@ export class CompactionController { } } catch { // 中止或意外异常：走统一善后（回滚快照 / running 态复位 idle）。 - await this.handleTurnAbort(); + await this.handleTurnAbort(bindingGeneration); return binding.cancellation.userStop.signal.aborted ? { status: "failed", aborted: true } : { status: "failed" }; } finally { - this.unbindTurn(); + this.unbindTurn(bindingGeneration); } } @@ -721,7 +762,10 @@ export class CompactionController { } // 用户中止后的统一善后：有快照则回滚（恢复状态/输入框/可选持久化）并返回 true。 - async handleTurnAbort(): Promise { + async handleTurnAbort(expectedGeneration?: number): Promise { + if (expectedGeneration !== undefined && !this.isTurnBound(expectedGeneration)) { + return false; + } const binding = this.binding; const snapshot = this.rollbackSnapshot; this.rollbackSnapshot = null; @@ -808,6 +852,10 @@ export class CompactionController { this.binding?.sinks.publishStatus?.(status); } + private activeObserver() { + return this.binding?.observer ?? this.observer; + } + private publishRunning( trigger: CompactionTrigger, sourceSegmentIndex: number, @@ -818,7 +866,7 @@ export class CompactionController { this.observedTrigger = trigger; this.observedTokensBefore = decision.totalTokens; this.notifyObserver(() => - this.observer?.onStart({ trigger, tokensBefore: decision.totalTokens }), + this.activeObserver()?.onStart({ trigger, tokensBefore: decision.totalTokens }), ); this.publishStatus({ phase: "running", @@ -842,7 +890,7 @@ export class CompactionController { // was unwinding. Late completion is then operationally stale and must not emit a second end. if (this.observedTrigger !== trigger || this.observedOperationId !== operationId) return; this.notifyObserver(() => - this.observer?.onEnd({ + this.activeObserver()?.onEnd({ trigger, status: "complete", ...(this.observedTokensBefore === undefined @@ -866,7 +914,7 @@ export class CompactionController { private settleFailed(trigger: CompactionTrigger, message: string, operationId: number) { if (this.observedTrigger !== trigger || this.observedOperationId !== operationId) return; this.notifyObserver(() => - this.observer?.onEnd({ + this.activeObserver()?.onEnd({ trigger, status: "error", ...(this.observedTokensBefore === undefined @@ -886,7 +934,7 @@ export class CompactionController { return false; } this.notifyObserver(() => - this.observer?.onEnd({ + this.activeObserver()?.onEnd({ trigger, status: "aborted", ...(this.observedTokensBefore === undefined diff --git a/crates/agent-gui/src/lib/chat/conversation/run/gatewayBridgeEvents.ts b/crates/agent-gui/src/lib/chat/conversation/run/gatewayBridgeEvents.ts index 2fc3b35fc..a60d4b7db 100644 --- a/crates/agent-gui/src/lib/chat/conversation/run/gatewayBridgeEvents.ts +++ b/crates/agent-gui/src/lib/chat/conversation/run/gatewayBridgeEvents.ts @@ -218,12 +218,17 @@ export function createGatewayBridgeEventController( } }, emitError(message: string, conversationIdOverride?: string) { - queueEvent({ + const sendResult = queueEvent({ type: "error", message, conversation_id: conversationIdOverride ?? params.resolveErrorConversationId?.() ?? params.conversationId, }); + if (sendResult && typeof (sendResult as Promise).then === "function") { + (sendResult as Promise).catch((error) => { + console.warn("error event failed", error); + }); + } }, close() { streamClosed = true; diff --git a/crates/agent-gui/src/lib/chat/runner/agentRunner.ts b/crates/agent-gui/src/lib/chat/runner/agentRunner.ts index 6218434b6..e9ea1c000 100644 --- a/crates/agent-gui/src/lib/chat/runner/agentRunner.ts +++ b/crates/agent-gui/src/lib/chat/runner/agentRunner.ts @@ -14,7 +14,12 @@ import { mergeHostedSearchBlocks, } from "@liveagent/ui/lib/chat/hostedSearch"; import type { PreparedProxyRequest } from "@liveagent/ui/lib/providers/proxy"; -import { buildStreamRequestDebugPayload, type StreamDebugLogger } from "../../debug/agentDebug"; +import { raceWithAbort } from "../../cancellation/abortRace"; +import { + buildStreamRequestDebugPayload, + flushDebugLoggerInBackground, + type StreamDebugLogger, +} from "../../debug/agentDebug"; import { capturePrefixShape, comparePrefixShape } from "../../debug/prefixCacheShape"; import { readPreviousPrefixShape, recordPrefixShape } from "../../debug/prefixShapeStore"; import { @@ -855,7 +860,10 @@ export async function runAssistantWithTools(params: { // Await the round's probe finalization (message_end already queued this // exact promise) so the coverage decision reads the complete in-band // search metadata instead of racing the response-clone parser. - const blocks = await finishHostedSearchRound(currentRound, "completed"); + const blocks = await raceWithAbort( + finishHostedSearchRound(currentRound, "completed"), + params.signal, + ); return blocks.some((block) => block.status === "completed" && block.sources.length > 0); } // web_fetch bridges never add new information; once the model has @@ -1038,7 +1046,7 @@ export async function runAssistantWithTools(params: { ) { const finalization = finishHostedSearchRound(round, mode) .then((hostedSearchBlocks) => { - if (!assistantRef) return; + if (!assistantRef || params.signal?.aborted) return; const nextAssistant = applyHostedSearchBlocksToAssistant( assistantRef.current, round, @@ -1064,7 +1072,15 @@ export async function runAssistantWithTools(params: { async function waitForHostedSearchFinalizations() { while (hostedSearchFinalizations.size > 0) { - await Promise.allSettled([...hostedSearchFinalizations]); + const pending = Promise.allSettled([...hostedSearchFinalizations]); + try { + await raceWithAbort(pending, params.signal); + } catch (error) { + // Probe finalization unregisters before it waits for a response + // clone. A cancelled turn must not wait for that clone to close. + if (params.signal?.aborted) return; + throw error; + } } } @@ -1955,7 +1971,7 @@ export async function runAssistantWithTools(params: { throw new Error(normalizeErrorMessage(assistant.errorMessage, "Cancelled")); } - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "agent runner"); return { messages, assistant, @@ -1967,7 +1983,7 @@ export async function runAssistantWithTools(params: { nativeWebSearchStatusController.finish(); params.onToolStatus?.(null); params.debugLogger?.logError(error); - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "agent runner"); throw error; } finally { queueAllHostedSearchFinalizations("dispose"); diff --git a/crates/agent-gui/src/lib/debug/agentDebug.ts b/crates/agent-gui/src/lib/debug/agentDebug.ts index 164a905f8..d72d8ea14 100644 --- a/crates/agent-gui/src/lib/debug/agentDebug.ts +++ b/crates/agent-gui/src/lib/debug/agentDebug.ts @@ -26,6 +26,25 @@ export type StreamDebugLogger = { flush: () => Promise; }; +/** + * Agent dev logging is diagnostic only. Keep its queued IPC writes out of + * request completion and cancellation paths, while still observing a late + * failure so it cannot become an unhandled rejection. + */ +export function flushDebugLoggerInBackground( + logger: StreamDebugLogger | undefined, + context: string, +): void { + if (!logger) return; + try { + void Promise.resolve(logger.flush()).catch((error) => { + console.warn(`Agent dev debug ${context} flush failed`, error); + }); + } catch (error) { + console.warn(`Agent dev debug ${context} flush failed`, error); + } +} + const writeQueues = new Map>(); const REDACTED_DEBUG_CREDENTIAL = "[redacted credential]"; diff --git a/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts b/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts index 958453832..1b0d7bd0f 100644 --- a/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts +++ b/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts @@ -5,6 +5,7 @@ import { createAssistantMessageEventStream, isRetryableAssistantError, } from "@earendil-works/pi-ai"; +import { raceWithAbort } from "../../cancellation/abortRace"; /** * Provider auto-failover runtime (cc-switch inspired). @@ -274,6 +275,25 @@ function buildStartFailureAssistantMessage( } as AssistantMessage; } +function buildAbortedAssistantMessage(candidate: ProviderFailoverCandidate): AssistantMessage { + return { + ...buildStartFailureAssistantMessage(candidate, new Error("Cancelled")), + stopReason: "aborted", + } as AssistantMessage; +} + +function observeSourceResult(source: AssistantMessageEventStream): void { + try { + // A terminal event is authoritative for the wrapped output. Keep a + // non-conforming source result observed without making terminal delivery + // depend on it, so a later rejection cannot become unhandled. + void Promise.resolve(source.result()).catch(() => undefined); + } catch { + // Custom stream implementations may throw from result(). Their terminal + // event has already determined the wrapped stream's outcome. + } +} + /** * Runs candidates in order with buffer-until-commit semantics. * @@ -387,11 +407,12 @@ export function withProviderFailover( } if (terminal?.type === "done") { + observeSourceResult(source); if (!committed) { options.onCommitted?.(index); for (const bufferedEvent of buffered) output.push(bufferedEvent); } - output.end(await source.result()); + output.end(); return; } @@ -399,6 +420,7 @@ export function withProviderFailover( if (!committed && terminalEligible && !isLastAttempt && !signal?.aborted) { // Discard this attempt's buffered events and try the next candidate. + observeSourceResult(source); continue; } @@ -407,7 +429,25 @@ export function withProviderFailover( if (!committed) { for (const bufferedEvent of buffered) output.push(bufferedEvent); } - output.end(await source.result()); + if (terminal) { + observeSourceResult(source); + output.end(); + return; + } + + try { + output.end(await raceWithAbort(source.result(), signal)); + } catch (error) { + const final = signal?.aborted + ? buildAbortedAssistantMessage(candidate) + : buildStartFailureAssistantMessage(candidate, error); + output.push({ + type: "error", + reason: signal?.aborted ? "aborted" : "error", + error: final, + }); + output.end(final); + } return; } })(); diff --git a/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts b/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts index aef4299d6..5fbbc0119 100644 --- a/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts +++ b/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts @@ -13,10 +13,13 @@ export const DEFAULT_STREAM_RETRY_MAX_ATTEMPTS = 6; const STREAM_RETRY_BASE_DELAY_MS = 200; const STREAM_RETRY_BACKOFF_FACTOR = 2; +const DEFAULT_STREAM_RETRY_IDLE_TIMEOUT_MS = 30_000; export type StreamRetryConfig = { maxAttempts?: number; disabled?: boolean; + /** Maximum time a provider attempt may wait for its next event or result. */ + idleTimeoutMs?: number; /** * Retry ordinal (1..maxRetries) about to be attempted, invoked before the * backoff sleep. `errorMessage` is the failure that triggered this retry. @@ -46,6 +49,12 @@ function terminalMessage(event: TerminalEvent) { return event.type === "done" ? event.message : event.error; } +function terminalAssistantMessage( + terminal: TerminalEvent | undefined, +): AssistantMessage | undefined { + return terminal ? (terminalMessage(terminal) as AssistantMessage) : undefined; +} + /** Codex-style backoff: base * factor^(attempt-1) * uniform(0.9, 1.1), uncapped. */ export function computeStreamRetryBackoffMs(attempt: number): number { const base = STREAM_RETRY_BASE_DELAY_MS * STREAM_RETRY_BACKOFF_FACTOR ** (attempt - 1); @@ -67,6 +76,59 @@ function buildAbortedAssistantMessage(previous: AssistantMessage | undefined): A } as AssistantMessage; } +function errorText(error: unknown): string { + if (error instanceof Error && error.message.trim()) return error.message; + if (typeof error === "string" && error.trim()) return error; + if (error && typeof error === "object") { + const candidate = error as { errorMessage?: unknown; message?: unknown; error?: unknown }; + if (typeof candidate.errorMessage === "string" && candidate.errorMessage.trim()) { + return candidate.errorMessage; + } + if (typeof candidate.message === "string" && candidate.message.trim()) { + return candidate.message; + } + if (typeof candidate.error === "string" && candidate.error.trim()) return candidate.error; + } + return String(error) || "Provider stream failed"; +} + +function buildTransportErrorMessage( + error: unknown, + previous?: AssistantMessage, +): AssistantMessage { + return { + ...(previous ?? {}), + role: "assistant", + content: previous?.content ?? [], + stopReason: "error", + errorMessage: errorText(error), + } as AssistantMessage; +} + +function isRetryableTransportFailure(error: unknown): boolean { + const assistantError = + error && typeof error === "object" && "role" in error + ? (error as AssistantMessage) + : undefined; + if (assistantError && isRetryableAssistantError(assistantError)) return true; + const message = errorText(error); + if (/\b(?:abort|aborted|cancel|cancelled|canceled)\b/i.test(message)) return false; + return /(?:fetch failed|network|timed?\s*out|timeout|econn(?:reset|refused|aborted)|connection|socket|stream|\b(?:408|425|429|500|502|503|504|522|524)\b|temporarily unavailable|service unavailable|overloaded)/i.test( + message, + ); +} + +function createSyntheticErrorStream( + error: unknown, + previous?: AssistantMessage, +): AssistantMessageEventStream { + const stream = createAssistantMessageEventStream(); + const failed = buildTransportErrorMessage(error, previous); + stream.push({ type: "error", reason: "error", error: failed }); + stream.end(failed); + return stream; +} + function sleepWithAbort(ms: number, signal: AbortSignal | undefined): Promise { if (signal?.aborted) return Promise.reject(signal.reason ?? new Error("Aborted")); if (ms <= 0) return Promise.resolve(); @@ -83,6 +145,87 @@ function sleepWithAbort(ms: number, signal: AbortSignal | undefined): Promise( + operation: PromiseLike | T, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + const source = Promise.resolve(operation); + if (signal?.aborted) { + // The provider read may already be in flight. Keep its late rejection + // observed even though the caller must return cancellation immediately. + void source.catch(() => undefined); + return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError")); + } + if (timeoutMs <= 0 && !signal) return source; + + return new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType | undefined; + const cleanup = () => { + if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }; + const succeed = (value: T) => { + if (settled) return; + settled = true; + cleanup(); + resolve(value); + }; + const fail = (error: unknown) => { + if (settled) return; + settled = true; + cleanup(); + reject(error); + }; + const onAbort = () => + fail(signal?.reason ?? new DOMException("Aborted", "AbortError")); + + if (signal) signal.addEventListener("abort", onAbort, { once: true }); + if (timeoutMs > 0) { + timer = setTimeout(() => fail(new Error("Provider stream idle timeout")), timeoutMs); + } + source.then(succeed, fail); + }); +} + +function readTerminalEventAfterAbort( + iterator: AsyncIterator, +): Promise> { + // A provider may have synchronously buffered its terminal error before the + // user pressed Stop. Preserve that terminal for retry accounting, but never + // forward a queued text/thinking/tool event after cancellation. + return new Promise((resolve) => { + let settled = false; + const finish = (value: IteratorResult) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(value); + }; + const timer = setTimeout(() => finish({ done: true, value: undefined }), 0); + Promise.resolve() + .then(() => iterator.next()) + .then( + (next) => { + if (next.done || isTerminalEvent(next.value)) finish(next); + else finish({ done: true, value: undefined }); + }, + () => finish({ done: true, value: undefined }), + ); + }); +} + +function abandonIterator(iterator: AsyncIterator) { + try { + // A hung provider iterator can keep its return() promise pending too. + // Request cleanup without making cancellation wait on provider code. + void Promise.resolve(iterator.return?.()).catch(() => undefined); + } catch { + // Some minimal provider/test iterators do not support return(). + } +} + /** * Wraps a fresh-stream factory with attempt-scoped retry for transient * provider/transport failures. @@ -111,10 +254,25 @@ export function withStreamRetry( ): AssistantMessageEventStream { const maxAttempts = Math.max(1, options?.maxAttempts ?? DEFAULT_STREAM_RETRY_MAX_ATTEMPTS); const disabled = options?.disabled ?? false; + const idleTimeoutMs = Math.max( + 0, + options?.idleTimeoutMs ?? DEFAULT_STREAM_RETRY_IDLE_TIMEOUT_MS, + ); const signal = options?.signal; const output = createAssistantMessageEventStream(); - const firstSource = factory(); + let firstSource: AssistantMessageEventStream; + try { + firstSource = factory(); + } catch (error) { + firstSource = createSyntheticErrorStream(error); + } + + const endAsAborted = (previous?: AssistantMessage) => { + const aborted = buildAbortedAssistantMessage(previous); + output.push({ type: "error", reason: "aborted", error: aborted }); + output.end(aborted); + }; void (async () => { let attempt = 1; @@ -125,8 +283,32 @@ export function withStreamRetry( let committed = false; const buffered: AssistantMessageEvent[] = []; let terminal: TerminalEvent | undefined; + let iteratorFailure = false; - for await (const event of source) { + const iterator = source[Symbol.asyncIterator](); + while (true) { + let next: IteratorResult; + try { + next = signal?.aborted + ? await readTerminalEventAfterAbort(iterator) + : await raceWithTimeout(iterator.next(), idleTimeoutMs, signal); + } catch (error) { + if (signal?.aborted) { + abandonIterator(iterator); + endAsAborted(terminalAssistantMessage(terminal)); + return; + } + abandonIterator(iterator); + iteratorFailure = true; + const failed = buildTransportErrorMessage(error, terminalAssistantMessage(terminal)); + terminal = { type: "error", reason: "error", error: failed }; + const failedEvent = terminal; + if (committed) output.push(failedEvent); + else buffered.push(failedEvent); + break; + } + if (next.done) break; + const event = next.value; if (!committed && COMMITTING_EVENT_TYPES.has(event.type)) { committed = true; for (const bufferedEvent of buffered.splice(0)) output.push(bufferedEvent); @@ -140,19 +322,62 @@ export function withStreamRetry( } else { buffered.push(event); } - if (isTerminalEvent(event)) terminal = event; + if (isTerminalEvent(event)) { + terminal = event; + abandonIterator(iterator); + break; + } + } + + // The abort-aware terminal drain may stop waiting before the provider iterator + // acknowledges cancellation. Ask it to release its transport without blocking the UI. + if (signal?.aborted) abandonIterator(iterator); + + let result: AssistantMessage | undefined; + if (!iteratorFailure && terminal === undefined) { + if (signal?.aborted) { + if (terminal === undefined) { + endAsAborted(); + return; + } + result = terminalMessage(terminal) as AssistantMessage; + } else { + try { + result = await raceWithTimeout(source.result(), idleTimeoutMs, signal); + } catch (error) { + if (signal?.aborted) { + endAsAborted(terminalAssistantMessage(terminal)); + return; + } + if (terminal === undefined) { + const failed = buildTransportErrorMessage(error); + terminal = { type: "error", reason: "error", error: failed }; + if (committed) output.push(terminal); + else buffered.push(terminal); + } + } + } + } + + // A few provider adapters finish iteration without emitting a terminal + // event and expose the failure only through result(). Treat that shape + // exactly like an in-stream error so an uncommitted network failure can + // still be retried. + if (terminal === undefined && result?.stopReason === "error") { + terminal = { type: "error", reason: "error", error: result }; + if (committed) output.push(terminal); + else buffered.push(terminal); } if (terminal?.type === "error" && !committed && !disabled && attempt < maxAttempts) { - if (isRetryableAssistantError(terminalMessage(terminal))) { + const terminalError = terminalMessage(terminal); + if (isRetryableAssistantError(terminalError) || isRetryableTransportFailure(terminalError)) { const errorMessage = terminalMessage(terminal)?.errorMessage || "Unknown error"; attempt += 1; options?.onRetry?.(attempt - 1, maxAttempts - 1, errorMessage); hasRetried = true; try { await sleepWithAbort(computeStreamRetryBackoffMs(attempt - 1), signal); - source = factory(); - continue; } catch { // Stopped mid-backoff: the terminal must say "aborted", not replay // the prior attempt's transport error. Handing the consumer that @@ -160,16 +385,25 @@ export function withStreamRetry( // abort branches upstream never fire, so nothing records the // cancellation and the status row falls back to a spinner. if (signal?.aborted) { - const aborted = buildAbortedAssistantMessage( - terminalMessage(terminal) as AssistantMessage | undefined, - ); - output.push({ type: "error", reason: "aborted", error: aborted }); - output.end(aborted); + endAsAborted(terminalAssistantMessage(terminal)); return; } // The next attempt failed to start — surface the prior attempt's // real failure below instead of hanging the consumer on a retry // that will never happen. + break; + } + try { + source = factory(); + continue; + } catch (error) { + // A synchronous provider construction failure is still a + // transport attempt. Feed it through the same bounded retry loop. + source = createSyntheticErrorStream( + error, + terminalAssistantMessage(terminal), + ); + continue; } } } @@ -181,10 +415,20 @@ export function withStreamRetry( // done/error event through iteration and only expose the final message // via result(). output.end() is idempotent once a terminal event has // already been pushed above, so this also safety-nets that case. - output.end(await source.result()); + output.end(result ?? terminalAssistantMessage(terminal) ?? buildTransportErrorMessage("Provider stream ended without a result")); return; } - })(); + })().catch((error) => { + if (signal?.aborted) { + endAsAborted(); + return; + } + const failed = buildAbortedAssistantMessage(undefined); + failed.stopReason = "error"; + failed.errorMessage = error instanceof Error ? error.message : String(error); + output.push({ type: "error", reason: "error", error: failed }); + output.end(failed); + }); return output; } diff --git a/crates/agent-gui/src/lib/providers/runtime/textOnlyRuntime.ts b/crates/agent-gui/src/lib/providers/runtime/textOnlyRuntime.ts index a7b2de98f..0d2aaf395 100644 --- a/crates/agent-gui/src/lib/providers/runtime/textOnlyRuntime.ts +++ b/crates/agent-gui/src/lib/providers/runtime/textOnlyRuntime.ts @@ -5,7 +5,12 @@ import { type HostedSearchOrderedBlock, mergeHostedSearchBlocks, } from "@liveagent/ui/lib/chat/hostedSearch"; -import { buildStreamRequestDebugPayload, type StreamDebugLogger } from "../../debug/agentDebug"; +import { raceWithAbort } from "../../cancellation/abortRace"; +import { + buildStreamRequestDebugPayload, + flushDebugLoggerInBackground, + type StreamDebugLogger, +} from "../../debug/agentDebug"; import type { ProviderId } from "../../settings"; import { withPowerActivity } from "../../system/powerActivity"; import { @@ -442,6 +447,11 @@ export async function streamAssistantMessage(params: { enabled: shouldProbeHostedSearch, onRawEvent: hostedSearchAggregator.accept, }); + let hostedSearchProbeFinalization: Promise | undefined; + const finishHostedSearchProbe = () => + (hostedSearchProbeFinalization ??= hostedSearchProbe.finish()); + const finishHostedSearchProbeWithAbort = () => + raceWithAbort(finishHostedSearchProbe(), params.signal); try { let activeContext = callContext; for (let toolRecoveryTurn = 0; toolRecoveryTurn < 4; toolRecoveryTurn += 1) { @@ -468,7 +478,7 @@ export async function streamAssistantMessage(params: { } } - let final = await s.result(); + let final = await raceWithAbort(s.result(), params.signal); if (final.stopReason === "error" || final.stopReason === "aborted") { throw new Error( normalizeErrorMessage( @@ -495,27 +505,30 @@ export async function streamAssistantMessage(params: { continue; } - await hostedSearchProbe.finish(); + await finishHostedSearchProbeWithAbort(); final = appendHostedSearchBlocksToAssistant( final as AssistantMessage & { content: unknown[] }, hostedSearchAggregator.complete(), { orderedBlocks }, ) as AssistantMessage; params.debugLogger?.logResult(final); - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "text stream"); return final; } throw new Error("Too many text-mode tool-call recovery attempts"); } catch (error) { - await hostedSearchProbe.finish(); if (params.signal?.aborted) { + // finish() unregisters the probe before it waits for its clone reader. + // Do not make cancellation wait for a broken network stream to close. + void finishHostedSearchProbe().catch(() => undefined); hostedSearchAggregator.dispose(); } else { + await finishHostedSearchProbe(); hostedSearchAggregator.fail(); } params.debugLogger?.logError(error); - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "text stream"); throw error; } }); @@ -576,7 +589,7 @@ export async function completeAssistantMessage(params: { return withPowerActivity("assistant-complete", `${params.providerId}:${modelId}`, async () => { try { const s = streamSimpleByApi(m, callContext, options); - const final = await s.result(); + const final = await raceWithAbort(s.result(), params.signal); if (final.stopReason === "error" || final.stopReason === "aborted") { throw new Error( @@ -588,11 +601,11 @@ export async function completeAssistantMessage(params: { } params.debugLogger?.logResult(final); - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "text completion"); return final; } catch (error) { params.debugLogger?.logError(error); - await params.debugLogger?.flush(); + flushDebugLoggerInBackground(params.debugLogger, "text completion"); throw error; } }); diff --git a/crates/agent-gui/src/lib/trajectory/recorder.ts b/crates/agent-gui/src/lib/trajectory/recorder.ts index e77399b4c..076c95eaf 100644 --- a/crates/agent-gui/src/lib/trajectory/recorder.ts +++ b/crates/agent-gui/src/lib/trajectory/recorder.ts @@ -50,6 +50,8 @@ export type TrajectoryStepEndInfo = { }; export type TrajectoryRecorder = { + /** Internal hook used by conversation runs that share one recorder. */ + selectTurn?: (turn: number) => void; /** * 一轮开始（用户消息落定）。同时把 `turn` 记为当前轮，后续调用不再重复传—— * 让每个埋点点自己传 turn 号，多一个参数就多一处传错的机会。 @@ -134,6 +136,76 @@ export const NOOP_TRAJECTORY_RECORDER: TrajectoryRecorder = { discard: () => {}, }; +/** + * Bind a recorder to one conversation turn. A conversation can briefly have + * two runs alive while a force-stop finalizer is unwinding; every synchronous + * recorder call must select its own turn before emitting an event. + */ +export function scopeTrajectoryRecorder( + recorder: TrajectoryRecorder, + turn: number, + onSelect?: () => void, +): TrajectoryRecorder { + const select = () => { + onSelect?.(); + recorder.selectTurn?.(turn); + }; + return { + selectTurn: recorder.selectTurn, + beginTurn: (info) => { + select(); + recorder.beginTurn({ ...info, turn }); + }, + noteContext: (info) => { + select(); + recorder.noteContext(info); + }, + captureHeader: (input) => { + select(); + return recorder.captureHeader(input); + }, + stepStart: (step, headerId) => { + select(); + recorder.stepStart(step, headerId); + }, + firstToken: (step) => { + select(); + recorder.firstToken(step); + }, + stepEnd: (step, info) => { + select(); + recorder.stepEnd(step, info); + }, + noteRetry: (step, info) => { + select(); + recorder.noteRetry(step, info); + }, + toolStart: (step, toolCall) => { + select(); + recorder.toolStart(step, toolCall); + }, + toolEnd: (callId, info) => { + select(); + recorder.toolEnd(callId, info); + }, + compactionStart: (options) => { + select(); + recorder.compactionStart(options); + }, + compactionEnd: (info) => { + select(); + recorder.compactionEnd(info); + }, + endTurn: (info) => { + select(); + recorder.endTurn(info); + }, + flush: () => recorder.flush(), + dispose: () => recorder.dispose(), + discard: () => recorder.discard(), + }; +} + /** * 创建会话级 recorder。 * @@ -152,7 +224,7 @@ export function createTrajectoryRecorder(params: { let lastHeader: { headerId: string; refs: TrajectorySectionRefs } | undefined; // Events before beginTurn are retained as turn 1 rather than discarded. let currentTurn = 1; - let turnOpen = false; + const openTurns = new Set(); let timer: ReturnType | null = null; let disposed = false; const firstTokenSeen = new Set(); @@ -225,9 +297,12 @@ export function createTrajectoryRecorder(params: { }; return { + selectTurn: (turn) => { + if (Number.isFinite(turn)) currentTurn = Math.max(1, Math.trunc(turn)); + }, beginTurn: ({ turn, messageIndex, messageId, text }) => { currentTurn = turn; - turnOpen = true; + openTurns.add(turn); emit({ k: "user", t: turn, @@ -347,8 +422,8 @@ export function createTrajectoryRecorder(params: { }); }, endTurn: (info) => { - if (!turnOpen) return; - turnOpen = false; + if (!openTurns.has(currentTurn)) return; + openTurns.delete(currentTurn); const unfinishedSteps = [...openSteps] .map((key) => { const [turnText, stepText] = key.split(" "); diff --git a/crates/agent-gui/src/lib/trajectory/recorderRegistry.ts b/crates/agent-gui/src/lib/trajectory/recorderRegistry.ts index 76d3a5a62..8b7d26bc0 100644 --- a/crates/agent-gui/src/lib/trajectory/recorderRegistry.ts +++ b/crates/agent-gui/src/lib/trajectory/recorderRegistry.ts @@ -13,7 +13,11 @@ import { type PreparedSystemPromptSlots, } from "../../pages/chat/runtime/conversationContextBuilders"; import { appendDesktopLiveTrajectory, clearDesktopLiveTrajectory } from "./liveTrajectory"; -import { createTrajectoryRecorder, type TrajectoryRecorder } from "./recorder"; +import { + createTrajectoryRecorder, + scopeTrajectoryRecorder, + type TrajectoryRecorder, +} from "./recorder"; import { createTauriTrajectoryPorts, resolvePersistedTrajectoryTurnNumber, @@ -47,12 +51,23 @@ export function acquireTrajectoryRecorder( conversationId: string, segmentIndex: number, publish?: TrajectoryPublish, + turn?: number, ): { recorder: TrajectoryRecorder; readSlots: () => PreparedSystemPromptSlots } { const existing = entries.get(conversationId); if (existing !== undefined) { existing.segmentIndex = segmentIndex; existing.publish = publish; - return { recorder: existing.recorder, readSlots: existing.slots.read }; + return { + recorder: + turn === undefined + ? existing.recorder + : scopeTrajectoryRecorder(existing.recorder, turn, () => { + // Recorder calls are synchronous. Select the matching bridge immediately before + // each event so an old run cannot publish late events through a replacement run. + existing.publish = publish; + }), + readSlots: existing.slots.read, + }; } const slots = createPreparedSystemPromptSlotHolder(); const entry: Entry = { @@ -69,7 +84,15 @@ export function acquireTrajectoryRecorder( }), }; entries.set(conversationId, entry); - return { recorder: entry.recorder, readSlots: slots.read }; + return { + recorder: + turn === undefined + ? entry.recorder + : scopeTrajectoryRecorder(entry.recorder, turn, () => { + entry.publish = publish; + }), + readSlots: slots.read, + }; } /** 供上下文构建器写入分段原文。 */ diff --git a/crates/agent-gui/src/pages/ChatPage.tsx b/crates/agent-gui/src/pages/ChatPage.tsx index b55119fb0..4d1ffeb80 100644 --- a/crates/agent-gui/src/pages/ChatPage.tsx +++ b/crates/agent-gui/src/pages/ChatPage.tsx @@ -1461,6 +1461,7 @@ export function ChatPage(props: ChatPageProps) { buildRuntimeEntryFromVisibleState, updateConversationRuntimeEntry, setConversationAbortController, + getConversationAbortController, getConversationStopRequestVersion, isConversationStopRequested, consumeConversationStop, @@ -1543,6 +1544,7 @@ export function ChatPage(props: ChatPageProps) { t, currentConversationIdRef, isConversationRunning, + getConversationAbortController, setConversationRunningState, setConversationAbortController, setConversationStopHandler, diff --git a/crates/agent-gui/src/pages/chat/history/useConversationHistoryActions.ts b/crates/agent-gui/src/pages/chat/history/useConversationHistoryActions.ts index 004cc8042..70c8f6eb0 100644 --- a/crates/agent-gui/src/pages/chat/history/useConversationHistoryActions.ts +++ b/crates/agent-gui/src/pages/chat/history/useConversationHistoryActions.ts @@ -50,6 +50,8 @@ export type PersistConversationParams = { createdAt: number; titlePromise: Promise | null; titleLookahead?: boolean; + /** Reject a stale run after title lookahead but before touching durable history. */ + shouldPersist?: () => boolean; }; // 成功返回盖好 revision 的持久化状态（revision 是 replace/分页的 CAS 令牌， @@ -497,6 +499,7 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi createdAt, titlePromise, titleLookahead = true, + shouldPersist, } = params; const pendingConversationTitle = t("chat.pendingTitle"); @@ -512,6 +515,10 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi } } + // A force-stopped run may finish its title lookahead after a replacement run has + // installed a new controller. Do not let that stale snapshot enter the history queue. + if (shouldPersist && !shouldPersist()) return null; + const updatedAt = Date.now(); markLocalHistorySnapshotSynced(conversationId, updatedAt); const selectedModelToPersist = resolvePersistedConversationModelSelection({ @@ -537,6 +544,9 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi commitPersistenceCursor: (cursor) => conversationPersistenceCursorRef.current.set(conversationId, cursor), }); + // The replacement run may have taken ownership while the write was in flight. + // The durable write is already ordered, but its stale summary must not become visible. + if (shouldPersist && !shouldPersist()) return null; markLocalHistorySnapshotSynced(conversationId, summary.updatedAt); // The write landed, so the durable row now matches `state` exactly — // stamp the CAS revision the backend will derive for it. Callers that @@ -573,6 +583,7 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi })); sidebarStore.upsertLocal({ ...summary, isPending: undefined }); } catch (err) { + if (shouldPersist && !shouldPersist()) return null; markLocalHistorySnapshotSynced(conversationId, -1); const msg = err instanceof Error ? err.message : String(err); const persistFailedMessage = t("chat.history.persistFailed").replace( @@ -592,6 +603,7 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi void titlePromise .then(async (resolvedTitle) => { if (!resolvedTitle || resolvedTitle === initialStoredTitle) return; + if (shouldPersist && !shouldPersist()) return; const currentItem = sidebarStore.peek(conversationId); if (!currentItem || currentItem.title !== initialStoredTitle) return; @@ -605,17 +617,26 @@ export function useConversationHistoryActions(params: UseConversationHistoryActi return; } + if (shouldPersist && !shouldPersist()) return; markLocalHistorySnapshotSynced(conversationId, Number.MAX_SAFE_INTEGER); const summary = await renameChatHistory(conversationId, resolvedTitle); + // The replacement run may have taken ownership while the rename IPC + // was in flight. Do not publish the stale title into the new run's + // sidebar state. + if (shouldPersist && !shouldPersist()) return; markLocalHistorySnapshotSynced(summary.id, summary.updatedAt); sidebarStore.upsertLocal({ ...summary, isPending: undefined }); }) .catch(() => { + if (shouldPersist && !shouldPersist()) return; markLocalHistorySnapshotSynced(conversationId, -1); // ignore late title failures; fallback title is already stored }) .finally(() => { - if (titleJobRef.current?.conversationId === conversationId) { + if ( + titleJobRef.current?.conversationId === conversationId && + titleJobRef.current.promise === titlePromise + ) { titleJobRef.current = null; } }); diff --git a/crates/agent-gui/src/pages/chat/runtime/chatRunFinalization.ts b/crates/agent-gui/src/pages/chat/runtime/chatRunFinalization.ts index 0ff26fb39..70a04b2c0 100644 --- a/crates/agent-gui/src/pages/chat/runtime/chatRunFinalization.ts +++ b/crates/agent-gui/src/pages/chat/runtime/chatRunFinalization.ts @@ -1,3 +1,5 @@ +import { raceWithAbort } from "../../../lib/cancellation/abortRace"; + export const CHAT_RUN_FINALIZATION_TIMEOUT_MS = 2_000; /** Terminal history writes get a few short retries before the run is marked failed. */ @@ -14,6 +16,25 @@ export function releaseChatRunUi(params: { params.clearToolStatus(); } +/** + * Trajectory persistence is diagnostic. Start its final flush immediately, + * but never make chat-run ownership or Stop recovery wait for a slow IPC + * write. The recorder keeps its own serial queue, so it can finish safely + * after the run's visible state has been released. + */ +export function flushTrajectoryInBackground( + flush: () => Promise, + context: "chat turn" | "manual compaction", +): void { + try { + void Promise.resolve(flush()).catch((error) => { + console.warn(`trajectory ${context} flush failed`, error); + }); + } catch (error) { + console.warn(`trajectory ${context} flush failed`, error); + } +} + function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, Math.max(0, ms)); @@ -124,6 +145,99 @@ export async function trackTerminalHistoryPersist( } } +/** + * A completed assistant response is already visible before its terminal + * history checkpoint returns. Stop must release that foreground wait, while + * still observing the persistence promise so a late failure is not unhandled. + */ +export async function awaitTerminalHistoryPersistOrStop( + persistPromise: Promise, + signal: AbortSignal, +): Promise<{ persisted: boolean; stopped: boolean }> { + try { + return { + persisted: await raceWithAbort(persistPromise, signal), + stopped: false, + }; + } catch (error) { + // A persistence failure can race with Stop. Only consume the exact abort + // reason produced by raceWithAbort; otherwise a real history error would + // be incorrectly reported as a successful Stop recovery. + if ( + signal.aborted && + (error === signal.reason || + (signal.reason === undefined && + error instanceof DOMException && + error.name === "AbortError")) + ) { + return { persisted: false, stopped: true }; + } + throw error; + } +} + +/** + * Keep a terminal write attached to the run that produced it. A force-stopped + * run can finish after its replacement has installed a new controller; it + * must not enqueue its old snapshot or mark the replacement as persist-failed. + */ +export async function persistOwnedTerminalHistory(params: { + input: T; + ownsRun: () => boolean; + persist: (input: T & { shouldPersist: () => boolean }) => Promise; + markFailed: () => void; + options?: { + maxAttempts?: number; + retryDelayMs?: number; + sleep?: (ms: number) => Promise; + onRetry?: (error: unknown, attempt: number, maxAttempts: number) => void; + }; +}): Promise { + const { input, ownsRun, persist, markFailed, options } = params; + if (!ownsRun()) return false; + + let ownershipLost = false; + const persistWhileOwned = async () => { + if (!ownsRun()) { + ownershipLost = true; + return true; + } + const persisted = await persist({ ...input, shouldPersist: ownsRun }); + if (!ownsRun()) { + ownershipLost = true; + return true; + } + return persisted; + }; + + try { + const persisted = await persistTerminalHistoryWithRetry(persistWhileOwned, options); + if (!persisted && !ownershipLost) { + markFailed(); + } + return ownershipLost ? false : persisted; + } catch (error) { + if (!ownsRun()) return false; + markFailed(); + throw error; + } +} + +/** + * A force-stopped run can outlive its UI ownership while the next run begins + * on the same conversation. In that state the live transcript store belongs + * to the replacement run, so an old terminal mirror must fall back to its + * own persisted-state projection instead of reading shared live state. + */ +export function resolveGatewayTerminalProjectionSource(params: { + state: "running" | "completed" | "failed" | "cancelled"; + hasFrozenProjection: boolean; + ownsRun: boolean; +}): "frozen" | "live" | "history" { + if (params.hasFrozenProjection) return "frozen"; + return params.state === "cancelled" && params.ownsRun ? "live" : "history"; +} + /** * Ordered chat-run finalization: history persistence must land before the * gateway stream close / terminal runtime snapshot become observable remotely diff --git a/crates/agent-gui/src/pages/chat/runtime/useManualCompaction.ts b/crates/agent-gui/src/pages/chat/runtime/useManualCompaction.ts index 2acb88664..52312b08a 100644 --- a/crates/agent-gui/src/pages/chat/runtime/useManualCompaction.ts +++ b/crates/agent-gui/src/pages/chat/runtime/useManualCompaction.ts @@ -5,6 +5,7 @@ import { useCallback } from "react"; import { readMessageContextUsage } from "../../../lib/chat/compaction/contextUsageMetadata"; import type { CompactionController, + CompactionObserver, CompactionSinks, ManualCompactionOutcome, ManualContextUsageSnapshot, @@ -29,6 +30,7 @@ import type { } from "../gateway/useGatewayRunMirrorCoordinator"; import type { PersistConversationAction } from "../history/useConversationHistoryActions"; import type { ConversationRuntimeEntry } from "./chatPageRuntime"; +import { flushTrajectoryInBackground } from "./chatRunFinalization"; import { buildPreparedContext as buildPreparedConversationContext, buildResumeContext as buildResumeConversationContext, @@ -97,6 +99,7 @@ export function useManualCompaction(params: { t: (key: string) => string; currentConversationIdRef: MutableRefObject; isConversationRunning: (conversationId: string) => boolean; + getConversationAbortController: (conversationId: string) => AbortController | null; setConversationRunningState: (conversationId: string, value: boolean) => void; setConversationAbortController: ( conversationId: string, @@ -141,6 +144,7 @@ export function useManualCompaction(params: { t, currentConversationIdRef, isConversationRunning, + getConversationAbortController, setConversationRunningState, setConversationAbortController, setConversationStopHandler, @@ -198,6 +202,9 @@ export function useManualCompaction(params: { let stopHandlerRegistered = false; let stopRequestVersion: number | null = null; let flushTrajectory: (() => Promise) | null = null; + const ownsManualRun = () => + !stopHandlerRegistered || + getConversationAbortController(conversationId) === cancellation.userStop; // 停止处理器与发送链路 handleConversationStop 同款：记录版本号供 finally // 消费 stop intent；abort 使 compactManually 中止（controller 返回 aborted）。 const handleStop: ConversationStopHandler = (options) => { @@ -322,13 +329,17 @@ export function useManualCompaction(params: { let compactionFailureMessage = ""; const sinks: CompactionSinks = { - applyState: (state) => - updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, state })), + applyState: (state) => { + if (!ownsManualRun()) return; + updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, state })); + }, applyStateMidRun: (state) => { + if (!ownsManualRun()) return; updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, state })); resetLiveTranscript(transcriptStore); }, publishStatus: (status) => { + if (!ownsManualRun()) return; if (status.phase === "failed") compactionFailureMessage = status.message; updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, @@ -336,13 +347,17 @@ export function useManualCompaction(params: { })); }, setBridgeToolStatus: (status, isCompaction = false) => { + if (!ownsManualRun()) return; gatewayBridgeEvents.queueToolStatus(status, isCompaction); updateToolStatus(status, transcriptStore); }, - queueCheckpoint: (state, contextUsageTokens) => - gatewayBridgeEvents.queueCheckpoint(state, contextUsageTokens), - persist: (state) => - persistConversation({ + queueCheckpoint: (state, contextUsageTokens) => { + if (!ownsManualRun()) return; + gatewayBridgeEvents.queueCheckpoint(state, contextUsageTokens); + }, + persist: (state) => { + if (!ownsManualRun()) return Promise.resolve(false); + return persistConversation({ conversationId, sessionId: runtimeEntry.sessionId, providerId, @@ -353,10 +368,14 @@ export function useManualCompaction(params: { fallbackTitle: t("chat.pendingTitle"), createdAt: runtimeEntry.createdAt, titlePromise: null, - }), + shouldPersist: ownsManualRun, + }); + }, // 压缩把携带 memory 增量块的 user 消息移出 active segment;丢弃注入 // 状态后,下一轮发送的 getSystemText 回退到现读快照并重新冻结。 - onCompacted: () => memoryTurnInjection.invalidate(conversationId), + onCompacted: () => { + if (ownsManualRun()) memoryTurnInjection.invalidate(conversationId); + }, }; const compactionController = getCompactionController(conversationId); @@ -373,9 +392,10 @@ export function useManualCompaction(params: { }); } }, + 1, ); flushTrajectory = trajectoryRecording.recorder.flush; - compactionController.setObserver({ + const compactionObserver: CompactionObserver = { onStart: ({ trigger }) => { trajectoryRecording.recorder.compactionStart({ standalone: trigger === "manual" }); }, @@ -391,13 +411,14 @@ export function useManualCompaction(params: { updateTrajectoryRecorderSegment(conversationId, newSegmentIndex); } }, - }); + }; const outcome = await compactionController.compactManually( { providerId, model, runtime, cancellation, + observer: compactionObserver, sinks, buildPreparedContext: (state, tools, options) => buildPreparedConversationContext({ @@ -430,6 +451,7 @@ export function useManualCompaction(params: { { tools: runtimeEntry.state.meta.tools, onProceed: () => { + if (!ownsManualRun()) return; proceeded = true; if (hasRemoteGatewayTarget) { // 与 useSendChatTurn 同款注册镜像：userMessage 取最近一条用户消息 @@ -471,27 +493,36 @@ export function useManualCompaction(params: { try { result = await run(); - if (result.status === "failed" && result.message && isCurrentConversation()) { + if ( + ownsManualRun() && + result.status === "failed" && + result.message && + isCurrentConversation() + ) { setErrorMessage(result.message); } return result; } catch (error) { const message = error instanceof Error ? error.message : String(error); - if (isCurrentConversation()) { + if (ownsManualRun() && isCurrentConversation()) { setErrorMessage(message); } result = { status: "failed", message }; return result; } finally { + const ownsRunOnFinalization = + stopHandlerRegistered && getConversationAbortController(conversationId) === cancellation.userStop; const flushRecordedTrajectory = flushTrajectory as (() => Promise) | null; if (flushRecordedTrajectory !== null) { - await flushRecordedTrajectory(); + flushTrajectoryInBackground(flushRecordedTrajectory, "manual compaction"); } if (stopHandlerRegistered) { clearConversationStopHandler(conversationId, handleStop); - setConversationAbortController(conversationId, null); + if (ownsRunOnFinalization) { + setConversationAbortController(conversationId, null); + } } - if (runningStateClaimed) { + if (runningStateClaimed && ownsRunOnFinalization) { setConversationRunningState(conversationId, false); } // 停止意图必须消费，否则残留会吞掉该会话的下一条消息。版本号不匹配 @@ -546,6 +577,7 @@ export function useManualCompaction(params: { finishGatewayRunMirror, flushGatewayBridgeEventsForRequest, getCompactionController, + getConversationAbortController, getConversationLiveTranscriptStore, isConversationRunning, persistConversation, diff --git a/crates/agent-gui/src/pages/chat/runtime/useSendChatTurn.ts b/crates/agent-gui/src/pages/chat/runtime/useSendChatTurn.ts index 1170a3d53..3f7adc977 100644 --- a/crates/agent-gui/src/pages/chat/runtime/useSendChatTurn.ts +++ b/crates/agent-gui/src/pages/chat/runtime/useSendChatTurn.ts @@ -23,6 +23,8 @@ import { invoke } from "@tauri-apps/api/core"; import type { Dispatch, MutableRefObject, SetStateAction } from "react"; import { useCallback } from "react"; import { createHookRunScope } from "../../../lib/automation/hookRunner"; +import { raceWithAbort } from "../../../lib/cancellation/abortRace"; +import type { CompactionObserver } from "../../../lib/chat/compaction/controller"; import { buildPersistableMessagesFromSnapshot, type SuppressedToolTraceSnapshot, @@ -37,6 +39,7 @@ import { type HistoryMessageRef, setTaskListState, } from "../../../lib/chat/conversation/conversationState"; +import type { LiveTranscriptStore } from "../../../lib/chat/conversation/liveTranscriptStore"; import { createConversationHookLifecycle, createGatewayBridgeEventController, @@ -120,9 +123,11 @@ import { } from "./chatPageRuntime"; import { finalizeChatRunInOrder, + flushTrajectoryInBackground, + persistOwnedTerminalHistory, releaseChatRunUi, + resolveGatewayTerminalProjectionSource, settleChatRunFinalization, - trackTerminalHistoryPersist, } from "./chatRunFinalization"; import { buildPreparedContext as buildPreparedConversationContext, @@ -174,6 +179,7 @@ type UseSendChatTurnParams = { buildRuntimeEntryFromVisibleState: ChatPageRuntimeStore["buildRuntimeEntryFromVisibleState"]; updateConversationRuntimeEntry: ChatPageRuntimeStore["updateConversationRuntimeEntry"]; setConversationAbortController: ChatPageRuntimeStore["setConversationAbortController"]; + getConversationAbortController: ChatPageRuntimeStore["getConversationAbortController"]; getConversationStopRequestVersion: ChatPageRuntimeStore["getConversationStopRequestVersion"]; isConversationStopRequested: ChatPageRuntimeStore["isConversationStopRequested"]; consumeConversationStop: ChatPageRuntimeStore["consumeConversationStop"]; @@ -250,6 +256,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { buildRuntimeEntryFromVisibleState, updateConversationRuntimeEntry, setConversationAbortController, + getConversationAbortController, getConversationStopRequestVersion, isConversationStopRequested, consumeConversationStop, @@ -424,12 +431,14 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { gatewayBridgeRequest?.conversationId ?? currentConversationIdRef.current, }); const updateGatewayBridgeToolStatus = (status: string | null, isCompaction = false) => { + if (!ownsConversationRun()) return; gatewayBridgeEvents.queueToolStatus(status, isCompaction); updateToolStatus(status, transcriptStore); }; // Mirrors the live retry-attempt list to remote WebUI clients alongside // the local live-transcript update. const updateGatewayBridgeRetryAttempts: typeof updateRetryAttempts = (attempts, store) => { + if (!ownsConversationRun()) return; gatewayBridgeEvents.queueRetryAttempts(attempts); updateRetryAttempts(attempts, store); }; @@ -497,6 +506,10 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { gatewayBridgeRequest?.runtimeControlsOverride ?? overrides?.runtimeControlsOverride ?? settings.chatRuntimeControls; + // Runtime callbacks can outlive a force-stopped turn. Keep the controller + // identity available before constructing the failover callbacks so late + // callbacks cannot mutate the replacement turn's runtime entry. + let activeTurnController: AbortController | null = null; const providerConfig = createProviderRuntimeConfig(provider, model, runtimeControls); // cc-switch style auto-failover plan for this turn (shared by the agent // and text runtimes). The switch callback makes the winning fallback the @@ -513,6 +526,12 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { round: number; errorMessage: string; }) => { + if ( + activeTurnController === null || + getConversationAbortController(conversationId) !== activeTurnController + ) { + return; + } const nextSelectedModel = event.target?.selectedModel ?? failoverPlan.primary.selectedModel; updateConversationRuntimeEntry(conversationId, (prev) => @@ -538,6 +557,12 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { : undefined; const handleMemoryExtractionModelFailure = memoryExtractionModel ? (failedModel: { selectedModel?: SelectedModel }) => { + if ( + activeTurnController === null || + getConversationAbortController(conversationId) !== activeTurnController + ) { + return; + } const failedSelectedModel = failedModel.selectedModel; setSettings((prev) => { if (!selectedModelsMatch(prev.memory.summaryModel, failedSelectedModel)) { @@ -606,7 +631,10 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { setIsImportingPastedText(false); } } - if (isConversationStopRequested(conversationId)) { + const stopRequestedForActiveRun = + isConversationStopRequested(conversationId) && + getConversationAbortController(conversationId) !== null; + if (stopRequestedForActiveRun) { const stopRequestVersion = getConversationStopRequestVersion(conversationId); if (gatewayBridgeRequest) { void invoke("gateway_chat_cancel_request", { @@ -621,6 +649,12 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { void settleChatRunFinalization(gatewayBridgeEvents.close()); return false; } + // Force-stop clears the old controller before its provider finally + // unwinds. A new manual message after that point is a new run, so the old + // stop intent must not silently discard it. + if (isConversationStopRequested(conversationId)) { + consumeConversationStop(conversationId, getConversationStopRequestVersion(conversationId)); + } const userMessage = createUserMessageWithUploads(text, uploadedFiles, Date.now()); if (!userMessage) { @@ -651,6 +685,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { // 轮次级取消：会话 abort controller 只注册 userStop 一次；每个 LLM 请求 // （主请求/压缩摘要/标题任务）各自派生子 scope，杜绝 abort 换代丢停止的窗口。 const cancellation = createTurnCancellation(); + activeTurnController = cancellation.userStop; const conversationDebugLogger = createStreamDebugLogger({ enabled: effectiveIsAgentDevExecutionMode, conversationId, @@ -755,12 +790,14 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { let gatewayRuntimeErrorMessage = ""; let frozenGatewayFinalProjectionJson: string | null = null; let frozenGatewayContentComplete = false; + let terminalResponseCommitted = false; let terminalHistoryPersistFailed = false; let initialUserTurnPersisted = false; let initialPersistPromise: Promise | null = null; let terminalHistoryPersistPromise: Promise | null = null; let runCleanupPromise: Promise = Promise.resolve(); let compactionBound = false; + let compactionBindingGeneration: number | null = null; let runStopRequestVersion: number | null = null; function registerGatewayRuntimeRun(state: GatewayRuntimeSnapshotState) { @@ -808,12 +845,16 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { async function persistTerminalConversation( input: Parameters[0], ) { - return trackTerminalHistoryPersist( - () => persistConversationWithHistorySync(input), - () => { + const persistPromise = persistOwnedTerminalHistory({ + input, + ownsRun: ownsTerminalHistoryPersist, + persist: persistConversationWithHistorySync, + markFailed: () => { terminalHistoryPersistFailed = true; }, - ); + }); + terminalHistoryPersistPromise = persistPromise; + return persistPromise; } function acknowledgeGatewayRunStarted() { @@ -838,9 +879,9 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { return; } conversationRunStarted = true; + setConversationAbortController(conversationId, cancellation.userStop); applyConversationState(nextConversationState); resetLiveTranscript(transcriptStore); - setConversationAbortController(conversationId, cancellation.userStop); if (isConversationStopRequested(conversationId)) { cancellation.userStop.abort(); } @@ -854,9 +895,49 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { } } + function ownsConversationRun() { + return ( + conversationRunStarted && + getConversationAbortController(conversationId) === cancellation.userStop + ); + } + + // A completed reply may still be waiting to enter history when Stop + // releases the UI. Keep that terminal write eligible until another run + // actually claims this conversation, then reject the old snapshot. + function ownsTerminalHistoryPersist() { + if (!conversationRunStarted) return false; + const activeController = getConversationAbortController(conversationId); + return activeController === null || activeController === cancellation.userStop; + } + + // The provider runtime may finish callbacks after Stop has released this run's + // controller. Keep every live-transcript sink scoped to the controller that + // created it so a late old callback cannot overwrite a replacement run. + const runResetLiveTranscript = (store: LiveTranscriptStore) => { + if (ownsConversationRun()) resetLiveTranscript(store); + }; + const runSettleLiveTranscript = (store: LiveTranscriptStore) => { + if (ownsConversationRun()) settleLiveTranscript(store); + }; + const runAppendDraftAssistantText = (delta: string, store: LiveTranscriptStore) => { + if (ownsConversationRun()) appendDraftAssistantText(delta, store); + }; + const runBatchLiveRoundsUpdate = ( + updater: Parameters[0], + store: LiveTranscriptStore, + ) => { + if (ownsConversationRun()) batchLiveRoundsUpdate(updater, store); + }; + const runUpdateToolStatus = (status: string | null, store: LiveTranscriptStore) => { + if (ownsConversationRun()) updateToolStatus(status, store); + }; function releaseConversationRunUi() { if (!conversationRunStarted || conversationUiReleased) return; conversationUiReleased = true; + // A force-stopped run can finish after a new turn has installed its own + // controller. Only the current controller owner may clear shared UI. + if (!ownsConversationRun()) return; releaseChatRunUi({ clearAbortController: () => setConversationAbortController(conversationId, null), clearSendingState: () => setConversationSendingState(conversationId, false), @@ -864,6 +945,16 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { }); } + function releaseCompactionTurn() { + if (!compactionBound) return; + if (compactionBindingGeneration === null) { + compaction.unbindTurn(); + } else { + compaction.unbindTurn(compactionBindingGeneration); + } + compactionBound = false; + } + function requestRemoteGatewayCancellation() { if (remoteGatewayCancelRequested) return; remoteGatewayCancelRequested = true; @@ -890,11 +981,22 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { const handleConversationStop = (options: { force: boolean; requestVersion: number }) => { runStopRequestVersion = options.requestVersion; - gatewayRuntimeFinalState = "cancelled"; + if (!terminalResponseCommitted) { + gatewayRuntimeFinalState = "cancelled"; + } cancellation.userStop.abort(); - requestRemoteGatewayCancellation(); + if (!terminalResponseCommitted) { + requestRemoteGatewayCancellation(); + } if (!options.force) return; + // Capture this run's live tail before force-stop releases the shared + // transcript store for a replacement run. The terminal mirror below + // must never read a new run's live state through this old request id. + if (ownsConversationRun() && frozenGatewayFinalProjectionJson === null) { + freezeGatewayLiveProjection(); + } releaseConversationRunUi(); + if (terminalResponseCommitted) return; // Force stop is the escape hatch for a stuck run: it intentionally // skips the persist barrier (which may itself be hung) so the gateway // still learns the run is cancelled. The run's own finally block will @@ -912,9 +1014,19 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { ensureGatewayRunForTerminalState(state); } if (gatewayRunStarted) { - if (frozenGatewayFinalProjectionJson === null) { + const projectionSource = resolveGatewayTerminalProjectionSource({ + state, + hasFrozenProjection: frozenGatewayFinalProjectionJson !== null, + ownsRun: ownsConversationRun(), + }); + if (projectionSource === "live") { + freezeGatewayLiveProjection(); + } else if (projectionSource === "history") { + // A stale cancelled run cannot safely read the conversation's live + // transcript because a replacement may already be writing there. + // Its own state is still safe, but must remain incomplete. if (state === "cancelled") { - freezeGatewayLiveProjection(); + freezeGatewayFinalProjection(nextConversationState, false); } else { freezeGatewayFinalProjection(nextConversationState, true); } @@ -965,12 +1077,15 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { cancellation.userStop.abort(); requestRemoteGatewayCancellation(); gatewayBridgeEvents.emitError("Cancelled", conversationId); + const ownsRunOnStop = ownsConversationRun(); + if (ownsRunOnStop && frozenGatewayFinalProjectionJson === null) { + freezeGatewayLiveProjection(); + } releaseConversationRunUi(); - if (compactionBound) { - compaction.unbindTurn(); - compactionBound = false; + releaseCompactionTurn(); + if (ownsRunOnStop) { + clearAbortSnapshot(transcriptStore); } - clearAbortSnapshot(transcriptStore); await finalizeConversationRun("cancelled"); clearConversationStopHandler(conversationId, handleConversationStop); consumeConversationStop(conversationId, runStopRequestVersion); @@ -978,6 +1093,20 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { return true; } + async function awaitBeforeRuntime(operation: PromiseLike | T) { + try { + return { + cancelled: false as const, + value: await raceWithAbort(operation, cancellation.userStop.signal), + }; + } catch (error) { + if (cancellation.userStop.signal.aborted) { + return { cancelled: true as const }; + } + throw error; + } + } + async function markLocalGatewayRunStarted() { if (!mirrorsLocalRunToGateway || localGatewayRunStarted) { return; @@ -1082,9 +1211,40 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { setPendingUploadsForConversation(conversationId, clearedPendingUploads); } }; + async function finalizePreRuntimeFailure(message: string, errorCode: string) { + // This is outside the provider runtime's try/finally. Every failure here + // must explicitly release the UI and any compaction lease so the next + // user message can begin a fresh run. + if (await finishRequestedStopBeforeRuntime()) return; + if (!ownsConversationRun()) return; + gatewayRuntimeFinalState = "failed"; + gatewayRuntimeErrorCode = errorCode; + gatewayRuntimeErrorMessage = message; + setConversationErrorState(message); + gatewayBridgeEvents.emitError(message, conversationId); + // Restore before releasing ownership: finalization may wait long enough + // for a replacement run to start and clear its own composer. + restoreComposerOnStartFailure(); + releaseConversationRunUi(); + releaseCompactionTurn(); + clearConversationStopHandler(conversationId, handleConversationStop); + await finalizeConversationRun("failed"); + pruneIdleConversationCaches([conversationId]); + if (runStopRequestVersion !== null || cancellation.userStop.signal.aborted) { + if (runStopRequestVersion !== null) { + consumeConversationStop(conversationId, runStopRequestVersion); + } + } else { + requestQueuedChatTurnProcessing(conversationId); + } + } if (mirrorsLocalRunToGateway) { try { - await markLocalGatewayRunStarted(); + const result = await awaitBeforeRuntime(markLocalGatewayRunStarted()); + if (result.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } } catch (error) { console.warn("gateway_chat_mark_local_started failed", error); } @@ -1094,7 +1254,11 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { } if (overrides?.beforeRuntimeStart) { try { - await overrides.beforeRuntimeStart(); + const result = await awaitBeforeRuntime(overrides.beforeRuntimeStart()); + if (result.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } if (await finishRequestedStopBeforeRuntime()) { return true; } @@ -1103,24 +1267,35 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { return true; } const message = asErrorMessage(error, "启动远程对话运行失败"); - setConversationErrorState(message); - gatewayBridgeEvents.emitError(message, conversationId); - releaseConversationRunUi(); - await finalizeConversationRun("failed"); - clearConversationStopHandler(conversationId, handleConversationStop); - restoreComposerOnStartFailure(); + await finalizePreRuntimeFailure(message, "runtime_start_failed"); return false; } } if (!initialUserTurnPersisted) { - trajectoryTurn = await resolveTrajectoryTurnNumber({ - conversationId, - currentUserPersisted: false, - fallbackTurn: nextConversationState.meta.totalMessageCount, - }); - if (await finishRequestedStopBeforeRuntime()) { - return true; + try { + const result = await awaitBeforeRuntime( + resolveTrajectoryTurnNumber({ + conversationId, + currentUserPersisted: false, + fallbackTurn: nextConversationState.meta.totalMessageCount, + }), + ); + if (result.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + trajectoryTurn = result.value; + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + } catch (error) { + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + // Trajectory numbering is diagnostic metadata. Keep the safe local + // fallback instead of preventing a conversation from starting. + console.warn("Failed to resolve trajectory turn number", error); } } @@ -1140,43 +1315,48 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { createdAt, titlePromise, titleLookahead: true, + shouldPersist: ownsConversationRun, }); const initialPersist = initialPersistPromise; if (overrides?.afterInitialHistoryPersist && !overrides.beforeRuntimeStart) { - const persisted = await initialPersist; - if (await finishRequestedStopBeforeRuntime()) { - return true; - } - if (!persisted) { - const message = "历史记录保存失败，已取消发送。"; - setConversationErrorState(message); - gatewayRuntimeErrorCode = "history_persist_failed"; - gatewayRuntimeErrorMessage = message; - gatewayBridgeEvents.emitError(message, conversationId); - releaseConversationRunUi(); - await finalizeConversationRun("failed"); - clearConversationStopHandler(conversationId, handleConversationStop); - restoreComposerOnStartFailure(); - return true; - } try { - await overrides.afterInitialHistoryPersist(); + const initialPersistResult = await awaitBeforeRuntime(initialPersist); + if (initialPersistResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + const persisted = initialPersistResult.value; if (await finishRequestedStopBeforeRuntime()) { return true; } + if (!persisted) { + const message = "历史记录保存失败，已取消发送。"; + await finalizePreRuntimeFailure(message, "history_persist_failed"); + return true; + } + try { + const result = await awaitBeforeRuntime(overrides.afterInitialHistoryPersist()); + if (result.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + } catch (error) { + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + const message = asErrorMessage(error, "历史保存后的启动操作失败"); + await finalizePreRuntimeFailure(message, "post_history_start_failed"); + return true; + } } catch (error) { if (await finishRequestedStopBeforeRuntime()) { return true; } - const message = asErrorMessage(error, "历史保存后的启动操作失败"); - setConversationErrorState(message); - gatewayRuntimeErrorCode = "post_history_start_failed"; - gatewayRuntimeErrorMessage = message; - gatewayBridgeEvents.emitError(message, conversationId); - releaseConversationRunUi(); - await finalizeConversationRun("failed"); - clearConversationStopHandler(conversationId, handleConversationStop); - restoreComposerOnStartFailure(); + const message = asErrorMessage(error, "历史记录保存失败，已取消发送。"); + await finalizePreRuntimeFailure(message, "history_persist_failed"); return true; } } else { @@ -1200,10 +1380,17 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { void initialPersistConfirmation; } if (gatewayBridgeRequest || hasRemoteGatewayTarget) { - const persisted = await initialPersist.catch((error) => { - console.warn("initial conversation history persist before gateway stream failed", error); - return false; - }); + const initialPersistResult = await awaitBeforeRuntime( + initialPersist.catch((error) => { + console.warn("initial conversation history persist before gateway stream failed", error); + return false; + }), + ); + if (initialPersistResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + const persisted = initialPersistResult.value; if (!persisted) { console.warn("gateway stream started before initial user turn was persisted"); } @@ -1211,20 +1398,42 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { return true; } } - await gatewayBridgeEvents.queueUserMessage(text, uploadedFiles, { - messageId: pendingUserMessage.id, - baseMessageRef: overrides?.editResendBaseMessageRef, - // The new message's own stable identity: lets remote transcripts bind - // their user bubble's messageRef immediately, so a follow-up edit of - // this message can anchor its rebase without a history round-trip. - messageRef: findHistoryMessageRefByMessageId(nextConversationState, pendingUserMessage.id), - }); + let queueUserMessageResult: Awaited>; + try { + queueUserMessageResult = await awaitBeforeRuntime( + gatewayBridgeEvents.queueUserMessage(text, uploadedFiles, { + messageId: pendingUserMessage.id, + baseMessageRef: overrides?.editResendBaseMessageRef, + // The new message's own stable identity: lets remote transcripts bind + // their user bubble's messageRef immediately, so a follow-up edit of + // this message can anchor its rebase without a history round-trip. + messageRef: findHistoryMessageRefByMessageId(nextConversationState, pendingUserMessage.id), + }), + ); + } catch (error) { + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + const message = asErrorMessage(error, "无法将消息转发至网关。"); + await finalizePreRuntimeFailure(message, "gateway_user_message_failed"); + return false; + } + if (queueUserMessageResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } if (effectiveIsAgentMode) { try { - await invoke("checkpoint_begin_turn", { - conversation_id: conversationId, - turn_id: pendingUserMessage.id, - }); + const checkpointResult = await awaitBeforeRuntime( + invoke("checkpoint_begin_turn", { + conversation_id: conversationId, + turn_id: pendingUserMessage.id, + }), + ); + if (checkpointResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } } catch (error) { console.warn("checkpoint turn boundary failed", error); } @@ -1233,10 +1442,33 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { return true; } acknowledgeGatewayRunStarted(); - const [{ memoryTurnInjection }, { buildMemoryOverviewSection }] = await Promise.all([ - import("../../../lib/chat/memory/injectionController"), - import("../../../lib/memory/prompts/injection"), - ]); + let promptModuleLoadStopped = false; + const promptModulesResult = await (async () => { + try { + return await awaitBeforeRuntime( + Promise.all([ + import("../../../lib/chat/memory/injectionController"), + import("../../../lib/memory/prompts/injection"), + ]), + ); + } catch (error) { + if (await finishRequestedStopBeforeRuntime()) { + promptModuleLoadStopped = true; + return null; + } + const message = asErrorMessage(error, "无法加载对话运行模块。"); + await finalizePreRuntimeFailure(message, "runtime_module_load_failed"); + return null; + } + })(); + if (promptModulesResult === null) { + return promptModuleLoadStopped; + } + if (promptModulesResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + const [{ memoryTurnInjection }, { buildMemoryOverviewSection }] = promptModulesResult.value; let skillsPrompt = ""; let memoryPrompt = ""; /** 本轮 `/skill-name` 显式提及块;没有提及时恒为空串,不会挂出任何内容。 */ @@ -1268,10 +1500,11 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { }); } }, + trajectoryTurn, ); // 压缩有四条触发路径，逐个调用点埋点必漏；订阅控制器生命周期一次覆盖全部。 // manual 发生在两轮之间，不属于任何 turn。 - compaction.setObserver({ + const compactionObserver: CompactionObserver = { onStart: ({ trigger }) => { trajectoryRecording.recorder.compactionStart({ standalone: trigger === "manual" }); }, @@ -1287,7 +1520,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { updateTrajectoryRecorderSegment(conversationId, newSegmentIndex); } }, - }); + }; function buildPreparedContext( state: ConversationViewState, @@ -1345,12 +1578,13 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { }); } - compaction.bindTurn({ + compactionBindingGeneration = compaction.bindTurn({ providerId, model, runtime: providerConfig, cancellation, debugLogger: compactionDebugLogger, + observer: compactionObserver, buildPreparedContext, buildResumeContext, presend: { @@ -1363,16 +1597,24 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { sinks: { applyState: applyConversationState, applyStateMidRun: rebaseConversationStateDuringRun, - publishStatus: (status) => + publishStatus: (status) => { + if (!ownsConversationRun()) return; updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, compactionStatus: status, - })), - setBridgeToolStatus: updateGatewayBridgeToolStatus, - queueCheckpoint: (state, contextUsageTokens) => - gatewayBridgeEvents.queueCheckpoint(state, contextUsageTokens), - persist: (state) => - persistConversation({ + })); + }, + setBridgeToolStatus: (status, isCompaction) => { + if (!ownsConversationRun()) return; + updateGatewayBridgeToolStatus(status, isCompaction); + }, + queueCheckpoint: (state, contextUsageTokens) => { + if (!ownsConversationRun()) return; + gatewayBridgeEvents.queueCheckpoint(state, contextUsageTokens); + }, + persist: async (state) => { + if (!ownsConversationRun()) return false; + return persistConversation({ conversationId, sessionId, providerId, @@ -1383,8 +1625,11 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { fallbackTitle, createdAt, titlePromise, - }), + shouldPersist: ownsConversationRun, + }); + }, restoreComposer: (composerText, restoredUploads) => { + if (!ownsConversationRun()) return; if (isConversationVisible() && typeof composerText === "string") { composerRef.current?.setText(composerText); composerRef.current?.focus(); @@ -1392,8 +1637,9 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { setPendingUploadsForConversation(conversationId, restoredUploads); }, persistRollback: async (state) => { + if (!ownsConversationRun()) return false; abortedConversationCommitted = true; - await persistConversationWithHistorySync({ + return persistConversationWithHistorySync({ conversationId, sessionId, providerId, @@ -1404,12 +1650,17 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { fallbackTitle, createdAt, titlePromise, + shouldPersist: ownsConversationRun, }); }, // 压缩把携带 memory 增量块的 user 消息移出 active segment,增量对模型 // 永久不可见;丢弃注入状态,下一轮把 fresh 快照重冻结进 system 段 —— // 压缩本来就要重建前缀,这次重冻结免费。 - onCompacted: () => memoryTurnInjection.invalidate(conversationId), + onCompacted: () => { + if (ownsConversationRun()) { + memoryTurnInjection.invalidate(conversationId); + } + }, }, }); compactionBound = true; @@ -1423,28 +1674,35 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { let byName = new Map(skillsList.map((s) => [s.name, s])); let missing = selectedSkillNames.filter((n) => !byName.has(n)); if (missing.length > 0 && workspaceResources.mode !== "custom") { - const fresh = await refreshSkills(); - if (await finishRequestedStopBeforeRuntime()) { - return true; - } - if (fresh) { - skillsList = fresh.skills; - rootDir = fresh.rootDir; - byName = new Map(skillsList.map((s) => [s.name, s])); - missing = selectedSkillNames.filter((n) => !byName.has(n)); + try { + const freshResult = await awaitBeforeRuntime(refreshSkills()); + if (freshResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + const fresh = freshResult.value; + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + if (fresh) { + skillsList = fresh.skills; + rootDir = fresh.rootDir; + byName = new Map(skillsList.map((s) => [s.name, s])); + missing = selectedSkillNames.filter((n) => !byName.has(n)); + } + } catch (error) { + if (await finishRequestedStopBeforeRuntime()) { + return true; + } + // Refresh is only a best-effort lookup. Continue to the existing + // missing-Skill error path so this preflight failure always cleans up. + console.warn("Failed to refresh skills before starting chat", error); } } if (missing.length > 0) { const message = `找不到以下 Skills：${missing.join(", ")}（请先重新扫描固定 Skills 目录）`; - setConversationErrorState(message); - gatewayRuntimeErrorCode = "skills_missing"; - gatewayRuntimeErrorMessage = message; - gatewayBridgeEvents.emitError(message, conversationId); - releaseConversationRunUi(); - await finalizeConversationRun("failed"); - clearConversationStopHandler(conversationId, handleConversationStop); - restoreComposerOnStartFailure(); + await finalizePreRuntimeFailure(message, "skills_missing"); return true; } @@ -1492,7 +1750,14 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { // 消息上的那个断点,不额外占用 Anthropic 的 4 个 cache_control 名额。 let memoryOverview: string | null = null; try { - memoryOverview = await buildMemoryOverviewSection(effectiveWorkdir); + const memoryOverviewResult = await awaitBeforeRuntime( + buildMemoryOverviewSection(effectiveWorkdir), + ); + if (memoryOverviewResult.cancelled) { + await finishRequestedStopBeforeRuntime(); + return true; + } + memoryOverview = memoryOverviewResult.value; } catch (error) { console.warn("Failed to build memory overview prompt", error); // null 表示这轮没读到,基线维持原样;空串是「一条记忆都没有」,属于正常内容。 @@ -1524,6 +1789,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { conversationId, workdir: effectiveWorkdir, onWarning: (warning) => { + if (!ownsConversationRun()) return; updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, hookWarning: formatHookWarningMessage(settings.locale, t, warning), @@ -1544,6 +1810,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { suppressedToolTrace: [], }; const commitVisibleAbortedConversation = () => { + if (!ownsConversationRun()) return false; if (abortedConversationCommitted) return true; const snapshot = getAbortSnapshot(transcriptStore); @@ -1579,6 +1846,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { }; const commitErroredConversation = (rawMessage: string) => { + if (!ownsConversationRun()) return; const snapshot = getAbortSnapshot(transcriptStore); const partialMessages = buildPersistableMessagesFromSnapshot({ executionMode: effectiveExecutionMode, @@ -1621,6 +1889,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { function applyConversationState(nextState: ConversationViewState) { nextConversationState = nextState; + if (!ownsConversationRun()) return; updateConversationRuntimeEntry(conversationId, (prev) => ({ ...prev, state: nextState, @@ -1631,6 +1900,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { // Once a compaction/prune result is committed into visible history, the // corresponding live transcript becomes stale and must be cleared. applyConversationState(nextState); + if (!ownsConversationRun()) return; resetLiveTranscript(transcriptStore); } @@ -1642,6 +1912,9 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { runId: gatewayBridgeRequestId, getState: () => nextConversationState.meta.taskList, commitState: async (taskList) => { + if (!ownsConversationRun()) { + throw new Error("Stale conversation run cannot persist task state."); + } const persisted = await persistConversationWithHistorySync({ conversationId, sessionId, @@ -1653,10 +1926,14 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { fallbackTitle, createdAt, titlePromise, + shouldPersist: ownsConversationRun, }).catch(() => false); if (!persisted) { throw new Error("Failed to persist task state."); } + if (!ownsConversationRun()) { + throw new Error("Stale conversation run cannot apply task state."); + } applyConversationState(setTaskListState(nextConversationState, taskList)); }, }; @@ -1682,6 +1959,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { skillsRootDir: skillsRootDirForTools, skillAccessPolicy: skillAccessPolicyForTools, onManagedSkillsChanged: (change) => { + if (!ownsConversationRun()) return; if (change.action !== "delete") { enableManagedSkills(change.names); return; @@ -1701,6 +1979,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { commandSafetyMode: effectiveCommandSafetyMode, planModeEnabled: effectivePlanModeEnabled, applyMcpOps: (ops) => { + if (!ownsConversationRun()) return; const removedIds = ops.filter((op) => op.kind === "remove").map((op) => op.serverId); setSettings((prev) => removeWorkspaceResourceReferences(applyMcpOpsToAppSettings(prev, ops), { @@ -1715,11 +1994,13 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { sshManagerRemoteAllowed: !gatewayBridgeRequest || settings.remote.enableWebSshTerminal === true, onSshSessionsChanged: (change) => { + if (!ownsConversationRun()) return; if (change.action === "create") { ensureSshTunnelToolTab(change.projectPathKey); } }, onTunnelsChanged: (change) => { + if (!ownsConversationRun()) return; if (change.action === "create") { ensureTunnelToolTab(change.projectPathKey); } @@ -1742,10 +2023,10 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { buildPreparedContext, compaction, cancellation, - resetLiveTranscript, - settleLiveTranscript, - batchLiveRoundsUpdate, - updateToolStatus, + resetLiveTranscript: runResetLiveTranscript, + settleLiveTranscript: runSettleLiveTranscript, + batchLiveRoundsUpdate: runBatchLiveRoundsUpdate, + updateToolStatus: runUpdateToolStatus, updateRetryAttempts: updateGatewayBridgeRetryAttempts, updatePersistableAgentProgress: (progress) => { persistableAgentProgress = progress; @@ -1753,6 +2034,9 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { commitVisibleAbortedConversation, persistConversationWithHistorySync: persistTerminalConversation, freezeGatewayFinalProjection, + onTerminalResponseCommitted: () => { + terminalResponseCommitted = true; + }, trajectory: trajectoryRecording.recorder, trajectoryTurn, trajectoryMessageIndex, @@ -1790,15 +2074,18 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { buildPreparedContext, compaction, cancellation, - resetLiveTranscript, - settleLiveTranscript, - appendDraftAssistantText, - batchLiveRoundsUpdate, + resetLiveTranscript: runResetLiveTranscript, + settleLiveTranscript: runSettleLiveTranscript, + appendDraftAssistantText: runAppendDraftAssistantText, + batchLiveRoundsUpdate: runBatchLiveRoundsUpdate, updateGatewayBridgeToolStatus, updateRetryAttempts: updateGatewayBridgeRetryAttempts, commitVisibleAbortedConversation, persistConversationWithHistorySync: persistTerminalConversation, freezeGatewayFinalProjection, + onTerminalResponseCommitted: () => { + terminalResponseCommitted = true; + }, trajectory: trajectoryRecording.recorder, trajectoryTurn, trajectoryMessageIndex, @@ -1809,43 +2096,66 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { } } catch (err) { const aborted = cancellation.userStop.signal.aborted || isAbortLikeError(err); - gatewayRuntimeFinalState = aborted ? "cancelled" : "failed"; + const stoppedAfterTerminalResponse = + cancellation.userStop.signal.aborted && terminalResponseCommitted; + gatewayRuntimeFinalState = stoppedAfterTerminalResponse + ? "completed" + : aborted + ? "cancelled" + : "failed"; const remoteErrorMessage = aborted ? "Cancelled" : (err instanceof Error ? err.message : String(err)) || "Request failed"; - gatewayRuntimeErrorCode = aborted ? "cancelled" : "provider_error"; - gatewayRuntimeErrorMessage = remoteErrorMessage; - if (aborted) { + gatewayRuntimeErrorCode = stoppedAfterTerminalResponse + ? "" + : aborted + ? "cancelled" + : "provider_error"; + gatewayRuntimeErrorMessage = stoppedAfterTerminalResponse ? "" : remoteErrorMessage; + if (aborted && !stoppedAfterTerminalResponse) { hookScope.cancel(); requestRemoteGatewayCancellation(); runCleanupPromise = (async () => { - const rolledBack = await compaction.handleTurnAbort(); + const rolledBack = + compactionBindingGeneration !== null && + compaction.isTurnBound(compactionBindingGeneration) + ? await compaction.handleTurnAbort(compactionBindingGeneration) + : false; if (!rolledBack) { commitVisibleAbortedConversation(); } - if (shouldCreatePendingHistoryItem && !abortedConversationCommitted) { + if ( + ownsConversationRun() && + shouldCreatePendingHistoryItem && + !abortedConversationCommitted + ) { sidebarStore.removeLocal(conversationId); } })(); - } else { + } else if (!aborted) { const msg = err instanceof Error ? err.message : String(err); commitErroredConversation(msg || "Request failed"); } - gatewayBridgeEvents.emitError(remoteErrorMessage, conversationId); - if (titleJobRef.current?.conversationId === conversationId) { + if (!stoppedAfterTerminalResponse) { + gatewayBridgeEvents.emitError(remoteErrorMessage, conversationId); + } + if (ownsConversationRun() && titleJobRef.current?.conversationId === conversationId) { titleJobRef.current = null; } } finally { - releaseConversationRunUi(); - if (compactionBound) { - compaction.unbindTurn(); - compactionBound = false; + const ownsRunOnFinalization = ownsConversationRun(); + const stopped = runStopRequestVersion !== null || cancellation.userStop.signal.aborted; + if (stopped && ownsRunOnFinalization && frozenGatewayFinalProjectionJson === null) { + freezeGatewayLiveProjection(); } + releaseConversationRunUi(); + releaseCompactionTurn(); hookLifecycle.endAgent(); hookScope.close(); - clearAbortSnapshot(transcriptStore); - const stopped = runStopRequestVersion !== null || cancellation.userStop.signal.aborted; - if (stopped) { + if (ownsRunOnFinalization) { + clearAbortSnapshot(transcriptStore); + } + if (stopped && !terminalResponseCommitted) { gatewayRuntimeFinalState = "cancelled"; requestRemoteGatewayCancellation(); } @@ -1859,7 +2169,7 @@ export function useSendChatTurn(params: UseSendChatTurnParams) { status: trajectoryStatus, ...(gatewayRuntimeErrorMessage ? { error: gatewayRuntimeErrorMessage } : {}), }); - await trajectoryRecording.recorder.flush(); + flushTrajectoryInBackground(trajectoryRecording.recorder.flush, "chat turn"); await finalizeConversationRun(gatewayRuntimeFinalState); clearConversationStopHandler(conversationId, handleConversationStop); pruneIdleConversationCaches([conversationId]); diff --git a/crates/agent-gui/src/pages/chat/turns/runAgentConversationTurn.ts b/crates/agent-gui/src/pages/chat/turns/runAgentConversationTurn.ts index c691d5e00..ca3723dab 100644 --- a/crates/agent-gui/src/pages/chat/turns/runAgentConversationTurn.ts +++ b/crates/agent-gui/src/pages/chat/turns/runAgentConversationTurn.ts @@ -12,6 +12,7 @@ import { serializeToolCatalog, } from "@liveagent/ui/lib/trajectory/sections"; import type { TrajectoryUsage } from "@liveagent/ui/lib/trajectory/types"; +import { raceWithAbort } from "../../../lib/cancellation/abortRace"; import type { CompactionController } from "../../../lib/chat/compaction/controller"; import { estimateTextTokenUnits } from "../../../lib/chat/compaction/tokenLedger"; import type { ProviderRuntimeConfig } from "../../../lib/chat/compaction/types"; @@ -108,6 +109,10 @@ import { buildPartialAssistantMessage, createEmptyAssistantUsage, } from "../runtime/chatPageRuntime"; +import { + awaitTerminalHistoryPersistOrStop, + flushTrajectoryInBackground, +} from "../runtime/chatRunFinalization"; import { buildGatewayToolCallPreviewArguments, summarizeToolCallForApproval, @@ -351,6 +356,8 @@ export type RunAgentConversationTurnParams = { }) => void; commitVisibleAbortedConversation: () => boolean; freezeGatewayFinalProjection: (state: ConversationViewState, contentComplete?: boolean) => void; + /** The complete assistant reply is visible even while its history checkpoint is pending. */ + onTerminalResponseCommitted?: () => void; persistConversationWithHistorySync: (params: PersistConversationParams) => Promise; memoryExtractionModel?: MemoryExtractionModelConfig; onMemoryExtractionModelFailure?: (model: MemoryExtractionModelConfig) => void; @@ -425,6 +432,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP updatePersistableAgentProgress, commitVisibleAbortedConversation, freezeGatewayFinalProjection, + onTerminalResponseCommitted, persistConversationWithHistorySync, memoryExtractionModel, onMemoryExtractionModelFailure, @@ -1581,22 +1589,29 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP applyConversationState(finalState); freezeGatewayFinalProjection(finalState, true); settleLiveTranscript(transcriptStore); - const historyPersisted = await persistCompletedState(finalState); + onTerminalResponseCommitted?.(); + const terminalHistory = await awaitTerminalHistoryPersistOrStop( + persistCompletedState(finalState), + cancellation.userStop.signal, + ); + if (terminalHistory.stopped) return; + const historyPersisted = terminalHistory.persisted; trajectory.endTurn( pendingTerminalAssistantMeta === null ? { status: "complete" } : trajectoryTerminalInfo(pendingTerminalAssistantMeta.assistant), ); // 落盘与历史写入对齐：turn 边界是账本的一致点，之后的记忆提取不属于本轮轨迹。 - await trajectory.flush(); + flushTrajectoryInBackground(trajectory.flush, "chat turn"); // Memory extraction reads the in-memory final state. Only run it after the // durable history write succeeds so we never keep "memory has the answer, // chat history only has the user prompt" after a failed final persist. if (historyPersisted && showSilentMemoryExtraction && shouldRunMemoryExtraction) { - const extraction = await runPostTurnMemoryExtraction({ + const extractionPromise = runPostTurnMemoryExtraction({ roundOffset: memoryRoundOffset, onTurnStart: (round) => { + if (cancellation.userStop.signal.aborted) return; gatewayBridgeEvents.queueToken("", { round, contextRelevant: false }); batchLiveRoundsUpdate( (prev) => [ @@ -1614,6 +1629,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP ); }, onTextDelta: (delta, round) => { + if (cancellation.userStop.signal.aborted) return; gatewayBridgeEvents.queueToken(delta, { round }); batchLiveRoundsUpdate( (prev) => @@ -1624,6 +1640,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP ); }, onThinkingDelta: (delta, round) => { + if (cancellation.userStop.signal.aborted) return; gatewayBridgeEvents.queueEvent({ type: "thinking", text: delta, @@ -1640,6 +1657,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP ); }, onToolCall: (toolCall, round) => { + if (cancellation.userStop.signal.aborted) return; if (!shouldShowToolEvent(toolCall)) return; gatewayBridgeEvents.queueEvent({ type: "tool_call", @@ -1659,6 +1677,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP ); }, onToolExecutionStart: (toolCall, round) => { + if (cancellation.userStop.signal.aborted) return; if (!shouldShowToolEvent(toolCall)) return; gatewayBridgeEvents.queueEvent({ type: "tool_call", @@ -1678,6 +1697,7 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP ); }, onToolResult: (toolCall, toolResult, round) => { + if (cancellation.userStop.signal.aborted) return; if (!shouldShowToolEvent(toolCall)) return; gatewayBridgeEvents.queueEvent({ type: "tool_result", @@ -1709,13 +1729,24 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP transcriptStore, ); }, - onAssistantMessage: (assistant, round) => - commitAssistantRoundMeta(assistant, round, { contextRelevant: false }), + onAssistantMessage: (assistant, round) => { + if (cancellation.userStop.signal.aborted) return; + commitAssistantRoundMeta(assistant, round, { contextRelevant: false }); + }, onToolStatus: (s) => { + if (cancellation.userStop.signal.aborted) return; gatewayBridgeEvents.queueToolStatus(s, false); updateToolStatus(s, transcriptStore); }, }); + const extraction = await raceWithAbort( + extractionPromise, + cancellation.userStop.signal, + ).catch((error) => { + if (cancellation.userStop.signal.aborted) return null; + throw error; + }); + if (!extraction || cancellation.userStop.signal.aborted) return; if (extraction.emittedMessages.length > 0) { completedState = appendRenderOnlyMessagesToConversation( finalState, @@ -1727,7 +1758,11 @@ export async function runAgentConversationTurn(params: RunAgentConversationTurnP applyConversationState(completedState); freezeGatewayFinalProjection(completedState, true); settleLiveTranscript(transcriptStore); - await persistCompletedState(completedState); + const completedHistory = await awaitTerminalHistoryPersistOrStop( + persistCompletedState(completedState), + cancellation.userStop.signal, + ); + if (completedHistory.stopped) return; } if (historyPersisted && !showSilentMemoryExtraction && shouldRunMemoryExtraction) { void runPostTurnMemoryExtraction(); diff --git a/crates/agent-gui/src/pages/chat/turns/runTextConversationTurn.ts b/crates/agent-gui/src/pages/chat/turns/runTextConversationTurn.ts index 9d49f1bca..39d902aba 100644 --- a/crates/agent-gui/src/pages/chat/turns/runTextConversationTurn.ts +++ b/crates/agent-gui/src/pages/chat/turns/runTextConversationTurn.ts @@ -48,6 +48,10 @@ import { type TrajectoryRecorder, } from "../../../lib/trajectory/recorder"; import { buildPartialAssistantMessage } from "../runtime/chatPageRuntime"; +import { + awaitTerminalHistoryPersistOrStop, + flushTrajectoryInBackground, +} from "../runtime/chatRunFinalization"; export type RuntimeModel = { api: AssistantMessage["api"]; @@ -129,6 +133,8 @@ export type RunTextConversationTurnParams = { updateRetryAttempts: (attempts: RetryAttemptRecord[], store: LiveTranscriptStore) => void; commitVisibleAbortedConversation: () => boolean; freezeGatewayFinalProjection: (state: ConversationViewState, contentComplete?: boolean) => void; + /** The complete assistant reply is visible even while its history checkpoint is pending. */ + onTerminalResponseCommitted?: () => void; persistConversationWithHistorySync: (params: PersistConversationParams) => Promise; memoryExtractionModel?: MemoryExtractionModelConfig; onMemoryExtractionModelFailure?: (model: MemoryExtractionModelConfig) => void; @@ -179,6 +185,7 @@ export async function runTextConversationTurn(params: RunTextConversationTurnPar updateRetryAttempts, commitVisibleAbortedConversation, freezeGatewayFinalProjection, + onTerminalResponseCommitted, persistConversationWithHistorySync, memoryExtractionModel, onMemoryExtractionModelFailure, @@ -550,21 +557,27 @@ export async function runTextConversationTurn(params: RunTextConversationTurnPar applyConversationState(finalState); freezeGatewayFinalProjection(finalState, true); settleLiveTranscript(transcriptStore); + onTerminalResponseCommitted?.(); hookLifecycle.ensureMessageEnded(); hookLifecycle.endAgent(); - const historyPersisted = await persistConversationWithHistorySync({ - conversationId, - sessionId, - providerId, - model, - cwd: historyCwd, - state: finalState, - fallbackTitle, - createdAt, - titlePromise, - }); + const terminalHistory = await awaitTerminalHistoryPersistOrStop( + persistConversationWithHistorySync({ + conversationId, + sessionId, + providerId, + model, + cwd: historyCwd, + state: finalState, + fallbackTitle, + createdAt, + titlePromise, + }), + cancellation.userStop.signal, + ); + if (terminalHistory.stopped) return; + const historyPersisted = terminalHistory.persisted; trajectory.endTurn(trajectoryTerminalInfo(finalAssistant)); - await trajectory.flush(); + flushTrajectoryInBackground(trajectory.flush, "chat turn"); // Only extract memory after durable history lands; otherwise memory can // retain the answer while a failed final persist leaves chat history on the // user-only snapshot. diff --git a/crates/agent-gui/test/chat/agent-runner.test.mjs b/crates/agent-gui/test/chat/agent-runner.test.mjs index 1e1d0e203..589f84dd4 100644 --- a/crates/agent-gui/test/chat/agent-runner.test.mjs +++ b/crates/agent-gui/test/chat/agent-runner.test.mjs @@ -472,6 +472,28 @@ function createBaseParams(overrides = {}) { }; } +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function settlesPromptly(promise) { + let timeoutId = null; + const timedOut = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(false), 100); + }); + try { + return await Promise.race([promise.then(() => true), timedOut]); + } finally { + if (timeoutId !== null) clearTimeout(timeoutId); + } +} + test("runAssistantWithTools returns terminal stop messages without scheduling a next-turn override", async () => { resetFakeStreams(createTextAssistant("done")); let beforeNextTurnCalls = 0; @@ -493,6 +515,37 @@ test("runAssistantWithTools returns terminal stop messages without scheduling a assert.equal(result.messages[1].role, "assistant"); }); +test("runAssistantWithTools returns without waiting for diagnostic debug persistence", async () => { + resetFakeStreams(createTextAssistant("done")); + const flushGate = deferred(); + let flushCalls = 0; + const originalWarn = console.warn; + console.warn = () => {}; + try { + const { params } = createBaseParams({ + debugLogger: { + enabled: true, + logRequest() {}, + logResponse() {}, + logResult() {}, + logError() {}, + flush() { + flushCalls += 1; + return flushGate.promise; + }, + }, + }); + + assert.equal(await settlesPromptly(runAssistantWithTools(params)), true); + assert.equal(flushCalls, 1); + + flushGate.reject(new Error("late debug persistence failure")); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + console.warn = originalWarn; + } +}); + test("runAssistantWithTools sends tracked deletion rules with a non-empty base prompt", async () => { resetFakeStreams(createTextAssistant("done")); const tools = [ @@ -578,6 +631,112 @@ test("runAssistantWithTools waits for delayed hosted search probe finalization", } }); +test("runAssistantWithTools cancellation does not wait for a stalled hosted-search probe", async () => { + const originalFetch = globalThis.fetch; + const controller = new AbortController(); + const parserStarted = deferred(); + let closeResponse; + + globalThis.fetch = async () => { + const response = new Response( + new ReadableStream({ + start(streamController) { + closeResponse = () => streamController.close(); + }, + }), + { headers: { "content-type": "text/event-stream; charset=utf-8" } }, + ); + parserStarted.resolve(); + return response; + }; + + try { + resetFakeStreams(createTextAssistant("done")); + queueStreamSideEffect((options) => + fetch("http://127.0.0.1:18080/proxy/codex/v1/responses", { + method: "POST", + headers: options.headers, + body: JSON.stringify({ prompt_cache_key: "session-1" }), + }), + ); + const { params } = createBaseParams({ + nativeWebSearch: true, + signal: controller.signal, + }); + const outcome = runAssistantWithTools(params).then( + () => "resolved", + () => "rejected", + ); + + await parserStarted.promise; + controller.abort(); + + assert.equal(await settlesPromptly(outcome), true); + assert.equal(await outcome, "rejected"); + } finally { + closeResponse?.(); + globalThis.fetch = originalFetch; + await new Promise((resolve) => setImmediate(resolve)); + } +}); + +test("runAssistantWithTools cancellation interrupts stalled hosted-search bridge finalization", async () => { + const originalFetch = globalThis.fetch; + const controller = new AbortController(); + const parserStarted = deferred(); + let closeResponse; + + globalThis.fetch = async () => { + const response = new Response( + new ReadableStream({ + start(streamController) { + closeResponse = () => streamController.close(); + }, + }), + { headers: { "content-type": "text/event-stream; charset=utf-8" } }, + ); + parserStarted.resolve(); + return response; + }; + + try { + const webSearchCall = createToolCall("stalled-bridge-search", "web_search", { + query: "stalled hosted search", + }); + resetFakeStreams( + createAssistant( + [{ type: "text", text: "Searching" }, webSearchCall], + "stop", + ), + ); + queueStreamSideEffect((options) => + fetch("http://127.0.0.1:18080/proxy/codex/v1/responses", { + method: "POST", + headers: options.headers, + body: JSON.stringify({ prompt_cache_key: "session-1" }), + }), + ); + const { params } = createBaseParams({ + nativeWebSearch: true, + signal: controller.signal, + }); + const outcome = runAssistantWithTools(params).then( + () => "resolved", + () => "rejected", + ); + + await parserStarted.promise; + controller.abort(); + + assert.equal(await settlesPromptly(outcome), true); + assert.equal(await outcome, "rejected"); + } finally { + closeResponse?.(); + globalThis.fetch = originalFetch; + await new Promise((resolve) => setImmediate(resolve)); + } +}); + test("runAssistantWithTools calls onBeforeNextTurn only for toolUse turns with tool results", async () => { const toolCall = { type: "toolCall", diff --git a/crates/agent-gui/test/chat/agent-turn-cancelled-history.test.mjs b/crates/agent-gui/test/chat/agent-turn-cancelled-history.test.mjs index 1333f227a..27e3ada65 100644 --- a/crates/agent-gui/test/chat/agent-turn-cancelled-history.test.mjs +++ b/crates/agent-gui/test/chat/agent-turn-cancelled-history.test.mjs @@ -171,6 +171,28 @@ const { composeTrajectorySystemPrompt } = loader.loadModule( function noOp() {} +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function settlesPromptly(promise) { + let timeoutId = null; + const timedOut = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(false), 100); + }); + try { + return await Promise.race([promise.then(() => true), timedOut]); + } finally { + if (timeoutId !== null) clearTimeout(timeoutId); + } +} + function createHookLifecycle() { return { startAgent: noOp, @@ -433,6 +455,227 @@ test("agent dev skips memory extraction when final history persistence fails", a } }); +test("agent mode releases a completed response when its terminal history write stalls", async () => { + const finalAssistant = { + ...abortedAssistant, + content: [{ type: "text", text: "durable answer" }], + stopReason: "stop", + }; + const state = conversationState.createConversationStateFromContext({ + systemPrompt: "", + messages: [], + }); + const persistStarted = deferred(); + const persistGate = deferred(); + const userStop = new AbortController(); + let committed = 0; + runAssistantWithToolsScenario = async (params) => { + params.onTurnStart?.(1); + params.onAssistantMessage?.(finalAssistant, 1); + return { + assistant: finalAssistant, + messages: [finalAssistant], + emittedMessages: [finalAssistant], + }; + }; + + try { + const run = runAgentConversationTurn( + createCompletedAgentDevTurnParams({ + state, + persistConversationWithHistorySync: () => { + persistStarted.resolve(); + return persistGate.promise; + }, + extra: { + showSilentMemoryExtraction: false, + cancellation: { + userStop, + deriveScope() { + return { controller: new AbortController(), release: noOp }; + }, + }, + onTerminalResponseCommitted() { + committed += 1; + }, + }, + }), + ); + await persistStarted.promise; + assert.equal(committed, 1); + + userStop.abort(); + assert.equal(await settlesPromptly(run), true); + + persistGate.resolve(true); + await run; + } finally { + runAssistantWithToolsScenario = replayCancelledHistoryScenario; + } +}); + +test("agent mode releases the follow-up memory history write after Stop", async () => { + const finalAssistant = { + ...abortedAssistant, + content: [{ type: "text", text: "durable answer" }], + stopReason: "stop", + }; + const memoryAssistant = { + ...finalAssistant, + provider: "liveagent", + api: "liveagent-memory", + content: [{ type: "text", text: "Memory updated" }], + timestamp: 5, + }; + const state = conversationState.createConversationStateFromContext({ + systemPrompt: "", + messages: [], + }); + const secondPersistStarted = deferred(); + const secondPersistGate = deferred(); + const userStop = new AbortController(); + let persistCalls = 0; + runAssistantWithToolsScenario = async (params) => { + params.onTurnStart?.(1); + params.onAssistantMessage?.(finalAssistant, 1); + return { + assistant: finalAssistant, + messages: [finalAssistant], + emittedMessages: [finalAssistant], + }; + }; + memoryExtractionRequestScenario = async () => ({ + ok: true, + acceptedCount: 1, + rejectedCount: 0, + writtenSlugs: ["preference"], + emittedMessages: [memoryAssistant], + }); + + try { + const run = runAgentConversationTurn( + createCompletedAgentDevTurnParams({ + state, + persistConversationWithHistorySync: () => { + persistCalls += 1; + if (persistCalls === 1) return Promise.resolve(true); + secondPersistStarted.resolve(); + return secondPersistGate.promise; + }, + extra: { + cancellation: { + userStop, + deriveScope() { + return { controller: new AbortController(), release: noOp }; + }, + }, + }, + }), + ); + await secondPersistStarted.promise; + + userStop.abort(); + assert.equal(await settlesPromptly(run), true); + + secondPersistGate.resolve(true); + await run; + } finally { + runAssistantWithToolsScenario = replayCancelledHistoryScenario; + memoryExtractionRequestScenario = async () => ({ + ok: true, + acceptedCount: 0, + rejectedCount: 0, + writtenSlugs: [], + emittedMessages: [], + }); + } +}); + +test("agent mode ignores a late silent-memory result after Stop", async () => { + const finalAssistant = { + ...abortedAssistant, + content: [{ type: "text", text: "durable answer" }], + stopReason: "stop", + }; + const memoryAssistant = { + ...finalAssistant, + provider: "liveagent", + api: "liveagent-memory", + content: [{ type: "text", text: "Memory updated" }], + timestamp: 5, + }; + const state = conversationState.createConversationStateFromContext({ + systemPrompt: "", + messages: [], + }); + const extractionStarted = deferred(); + const extractionGate = deferred(); + const userStop = new AbortController(); + const appliedStates = []; + let persistCalls = 0; + runAssistantWithToolsScenario = async (params) => { + params.onTurnStart?.(1); + params.onAssistantMessage?.(finalAssistant, 1); + return { + assistant: finalAssistant, + messages: [finalAssistant], + emittedMessages: [finalAssistant], + }; + }; + memoryExtractionRequestScenario = () => { + extractionStarted.resolve(); + return extractionGate.promise; + }; + + try { + const run = runAgentConversationTurn( + createCompletedAgentDevTurnParams({ + state, + applyConversationState(nextState) { + appliedStates.push(nextState); + }, + persistConversationWithHistorySync() { + persistCalls += 1; + return Promise.resolve(true); + }, + extra: { + cancellation: { + userStop, + deriveScope() { + return { controller: new AbortController(), release: noOp }; + }, + }, + }, + }), + ); + await extractionStarted.promise; + + userStop.abort(); + assert.equal(await settlesPromptly(run), true); + + extractionGate.resolve({ + ok: true, + acceptedCount: 1, + rejectedCount: 0, + writtenSlugs: ["preference"], + emittedMessages: [memoryAssistant], + }); + await new Promise((resolve) => setImmediate(resolve)); + + assert.equal(persistCalls, 1); + assert.equal(appliedStates.length, 1); + } finally { + runAssistantWithToolsScenario = replayCancelledHistoryScenario; + memoryExtractionRequestScenario = async () => ({ + ok: true, + acceptedCount: 0, + rejectedCount: 0, + writtenSlugs: [], + emittedMessages: [], + }); + } +}); + test("agent dev persists the answer before memory extraction and its visible status", async () => { const finalAssistant = { ...abortedAssistant, diff --git a/crates/agent-gui/test/chat/chat-stop-timing.test.mjs b/crates/agent-gui/test/chat/chat-stop-timing.test.mjs index 9b8364e79..08718caa5 100644 --- a/crates/agent-gui/test/chat/chat-stop-timing.test.mjs +++ b/crates/agent-gui/test/chat/chat-stop-timing.test.mjs @@ -1,8 +1,42 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import test from "node:test"; import { createTsModuleLoader } from "../helpers/load-ts-module.mjs"; +const chatComposerBarSource = readFileSync( + new URL("../../../agent-ui/src/pages/chat/ChatComposerBar.tsx", import.meta.url), + "utf8", +); +const conversationHistoryActionsSource = readFileSync( + new URL("../../src/pages/chat/history/useConversationHistoryActions.ts", import.meta.url), + "utf8", +); +const sendChatTurnSource = readFileSync( + new URL("../../src/pages/chat/runtime/useSendChatTurn.ts", import.meta.url), + "utf8", +); +const manualCompactionSource = readFileSync( + new URL("../../src/pages/chat/runtime/useManualCompaction.ts", import.meta.url), + "utf8", +); +const agentConversationTurnSource = readFileSync( + new URL("../../src/pages/chat/turns/runAgentConversationTurn.ts", import.meta.url), + "utf8", +); +const textConversationTurnSource = readFileSync( + new URL("../../src/pages/chat/turns/runTextConversationTurn.ts", import.meta.url), + "utf8", +); +const agentRunnerSource = readFileSync( + new URL("../../src/lib/chat/runner/agentRunner.ts", import.meta.url), + "utf8", +); +const textOnlyRuntimeSource = readFileSync( + new URL("../../src/lib/providers/runtime/textOnlyRuntime.ts", import.meta.url), + "utf8", +); + function createHookHarness() { const refs = []; const states = []; @@ -80,6 +114,100 @@ async function flushPromises() { await new Promise((resolve) => setImmediate(resolve)); } +test("an aborted pre-runtime wait releases without waiting for local work", async () => { + const loader = createTsModuleLoader(); + const { raceWithAbort } = loader.loadModule("src/lib/cancellation/abortRace.ts"); + const localWork = deferred(); + const controller = new AbortController(); + const waiting = raceWithAbort(localWork.promise, controller.signal); + + controller.abort(new Error("cancelled by user")); + + await assert.rejects(waiting, /cancelled by user/); + // Finish the underlying local work after cancellation. Its settlement must + // remain observed and must not turn into an unhandled rejection. + localWork.resolve(); + await flushPromises(); +}); + +test("a pre-aborted wait still observes local work that rejects later", async () => { + const loader = createTsModuleLoader(); + const { raceWithAbort } = loader.loadModule("src/lib/cancellation/abortRace.ts"); + const localWork = deferred(); + const controller = new AbortController(); + controller.abort(new Error("already cancelled")); + + await assert.rejects(raceWithAbort(localWork.promise, controller.signal), /already cancelled/); + localWork.reject(new Error("late local failure")); + await flushPromises(); +}); + +test("a completed response Stop releases a stalled history checkpoint", async () => { + const loader = createTsModuleLoader(); + const { awaitTerminalHistoryPersistOrStop } = loader.loadModule( + "src/pages/chat/runtime/chatRunFinalization.ts", + ); + const persistGate = deferred(); + const controller = new AbortController(); + const waiting = awaitTerminalHistoryPersistOrStop(persistGate.promise, controller.signal); + + controller.abort(); + assert.deepEqual(await waiting, { persisted: false, stopped: true }); + + // The underlying IPC may report an error after the foreground Stop has + // already recovered. The helper must keep that rejection observed. + persistGate.reject(new Error("late history failure")); + await flushPromises(); +}); + +test("a completed response keeps its history write eligible after UI release", () => { + assert.match( + sendChatTurnSource, + /function ownsTerminalHistoryPersist\(\) \{[\s\S]*?activeController === null \|\| activeController === cancellation\.userStop;/, + ); + assert.match( + sendChatTurnSource, + /persistOwnedTerminalHistory\(\{\s*input,\s*ownsRun: ownsTerminalHistoryPersist,/, + ); +}); + +test("a queued draft keeps a direct Stop control available", () => { + assert.match( + chatComposerBarSource, + /\{canQueueDraftWhileSending \? \(\s*/); +}); + +test("a stale title task cannot publish a failure or clear its replacement", () => { + assert.match( + conversationHistoryActionsSource, + /\.catch\(\(\) => \{\s*if \(shouldPersist && !shouldPersist\(\)\) return;\s*markLocalHistorySnapshotSynced\(conversationId, -1\);/, + ); + assert.match( + conversationHistoryActionsSource, + /titleJobRef\.current\?\.conversationId === conversationId &&\s*titleJobRef\.current\.promise === titlePromise/, + ); +}); + +test("a pre-runtime failure releases UI and the compaction turn before it exits", () => { + assert.match( + sendChatTurnSource, + /async function finalizePreRuntimeFailure\(message: string, errorCode: string\)[\s\S]*?await finishRequestedStopBeforeRuntime\(\)[\s\S]*?restoreComposerOnStartFailure\(\);\s*releaseConversationRunUi\(\);\s*releaseCompactionTurn\(\);\s*clearConversationStopHandler\(conversationId, handleConversationStop\);\s*await finalizeConversationRun\("failed"\);[\s\S]*?pruneIdleConversationCaches\(\[conversationId\]\);[\s\S]*?requestQueuedChatTurnProcessing\(conversationId\);/, + ); + const skillsFailureStart = sendChatTurnSource.indexOf('gatewayRuntimeErrorCode = "skills_missing"'); + assert.equal(skillsFailureStart, -1, "the Skill path delegates cleanup to the shared helper"); + assert.match(sendChatTurnSource, /gateway_user_message_failed/); + assert.match(sendChatTurnSource, /runtime_module_load_failed/); + assert.match(sendChatTurnSource, /Failed to resolve trajectory turn number/); + assert.match(sendChatTurnSource, /Failed to refresh skills before starting chat/); + assert.match( + sendChatTurnSource, + /if \(overrides\?\.afterInitialHistoryPersist && !overrides\.beforeRuntimeStart\) \{\s*try \{\s*const initialPersistResult = await awaitBeforeRuntime\(initialPersist\);[\s\S]*?const message = asErrorMessage\(error, "历史记录保存失败，已取消发送。"\);\s*await finalizePreRuntimeFailure\(message, "history_persist_failed"\);/, + ); +}); + /** * Per-conversation live transcript stores, matching * useLiveTranscriptController: every conversation owns its own store, so a @@ -597,6 +725,125 @@ test("slow chat finalization cannot delay synchronous UI release", async () => { gate.resolve(); }); +test("trajectory persistence is detached from chat and manual-compaction cleanup", async () => { + const loader = createTsModuleLoader(); + const { flushTrajectoryInBackground } = loader.loadModule( + "src/pages/chat/runtime/chatRunFinalization.ts", + ); + const gate = deferred(); + let flushCalls = 0; + + const result = flushTrajectoryInBackground(() => { + flushCalls += 1; + return gate.promise; + }, "chat turn"); + + assert.equal(result, undefined); + assert.equal(flushCalls, 1, "the diagnostic flush begins immediately"); + assert.match( + sendChatTurnSource, + /flushTrajectoryInBackground\(trajectoryRecording\.recorder\.flush, "chat turn"\);/, + ); + assert.doesNotMatch(sendChatTurnSource, /await trajectoryRecording\.recorder\.flush\(\)/); + assert.match( + manualCompactionSource, + /flushTrajectoryInBackground\(flushRecordedTrajectory, "manual compaction"\);/, + ); + assert.doesNotMatch(manualCompactionSource, /await flushRecordedTrajectory\(\)/); + assert.match( + agentConversationTurnSource, + /flushTrajectoryInBackground\(trajectory\.flush, "chat turn"\);/, + ); + assert.doesNotMatch(agentConversationTurnSource, /await trajectory\.flush\(\)/); + assert.match( + textConversationTurnSource, + /flushTrajectoryInBackground\(trajectory\.flush, "chat turn"\);/, + ); + assert.doesNotMatch(textConversationTurnSource, /await trajectory\.flush\(\)/); + + gate.resolve(); + await flushPromises(); +}); + +test("debug persistence is detached from provider completion and cancellation paths", () => { + assert.equal( + agentRunnerSource.match( + /flushDebugLoggerInBackground\(params\.debugLogger, "agent runner"\);/g, + )?.length, + 2, + ); + assert.doesNotMatch(agentRunnerSource, /await params\.debugLogger\?\.flush\(\)/); + + assert.equal( + textOnlyRuntimeSource.match(/flushDebugLoggerInBackground\(params\.debugLogger, "text /g)?.length, + 4, + ); + assert.doesNotMatch(textOnlyRuntimeSource, /await params\.debugLogger\?\.flush\(\)/); +}); + +test("hosted-search probe cleanup is abortable while normal completion still waits for sources", () => { + assert.match( + textOnlyRuntimeSource, + /raceWithAbort\(finishHostedSearchProbe\(\), params\.signal\)/, + ); + assert.equal( + textOnlyRuntimeSource.match(/raceWithAbort\(s\.result\(\), params\.signal\)/g)?.length, + 2, + ); + assert.match( + agentRunnerSource, + /await raceWithAbort\(pending, params\.signal\)/, + ); + assert.match( + agentRunnerSource, + /await raceWithAbort\(\s*finishHostedSearchRound\(currentRound, "completed"\),\s*params\.signal,\s*\)/, + ); + assert.match( + agentRunnerSource, + /if \(!assistantRef \|\| params\.signal\?\.aborted\) return;/, + ); +}); + +test("a stale cancelled run cannot read a replacement live transcript for its gateway terminal", () => { + const loader = createTsModuleLoader(); + const { resolveGatewayTerminalProjectionSource } = loader.loadModule( + "src/pages/chat/runtime/chatRunFinalization.ts", + ); + + assert.equal( + resolveGatewayTerminalProjectionSource({ + state: "cancelled", + hasFrozenProjection: false, + ownsRun: true, + }), + "live", + ); + assert.equal( + resolveGatewayTerminalProjectionSource({ + state: "cancelled", + hasFrozenProjection: false, + ownsRun: false, + }), + "history", + ); + assert.equal( + resolveGatewayTerminalProjectionSource({ + state: "completed", + hasFrozenProjection: false, + ownsRun: true, + }), + "history", + ); + assert.equal( + resolveGatewayTerminalProjectionSource({ + state: "cancelled", + hasFrozenProjection: true, + ownsRun: false, + }), + "frozen", + ); +}); + test("finalization flushes the gateway stream only after history persists", async () => { const loader = createTsModuleLoader(); const { finalizeChatRunInOrder } = loader.loadModule( @@ -695,6 +942,53 @@ test("terminal history persistence marks both false results and thrown errors", assert.equal(failures, 2); }); +test("a stale terminal run cannot enqueue or publish a history snapshot", async () => { + const loader = createTsModuleLoader(); + const { persistOwnedTerminalHistory } = loader.loadModule( + "src/pages/chat/runtime/chatRunFinalization.ts", + ); + let ownsRun = false; + let persistCalls = 0; + let markedFailed = 0; + + const skipped = await persistOwnedTerminalHistory({ + input: { state: "old terminal snapshot" }, + ownsRun: () => ownsRun, + persist: async () => { + persistCalls += 1; + return true; + }, + markFailed: () => { + markedFailed += 1; + }, + }); + + assert.equal(skipped, false); + assert.equal(persistCalls, 0); + assert.equal(markedFailed, 0); + + ownsRun = true; + let persistenceGuard; + const persisted = await persistOwnedTerminalHistory({ + input: { state: "current terminal snapshot" }, + ownsRun: () => ownsRun, + persist: async (input) => { + persistCalls += 1; + persistenceGuard = input.shouldPersist; + return true; + }, + markFailed: () => { + markedFailed += 1; + }, + }); + + assert.equal(persisted, true); + assert.equal(persistCalls, 1); + assert.equal(typeof persistenceGuard, "function"); + assert.equal(persistenceGuard(), true); + assert.equal(markedFailed, 0); +}); + test("terminal history persistence retries transient failures before succeeding", async () => { const loader = createTsModuleLoader(); const { persistTerminalHistoryWithRetry, trackTerminalHistoryPersist } = loader.loadModule( diff --git a/crates/agent-gui/test/chat/compaction-controller.test.mjs b/crates/agent-gui/test/chat/compaction-controller.test.mjs index 3ffdd4b82..8a8ef0c48 100644 --- a/crates/agent-gui/test/chat/compaction-controller.test.mjs +++ b/crates/agent-gui/test/chat/compaction-controller.test.mjs @@ -73,6 +73,16 @@ function summaryResponse() { }; } +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + // 3 个用户消息绕开 MIN_COMPACTION_USER_MESSAGES 冷却窗，方便连续压缩场景。 function bigState(extraMessages = []) { return conversationState.createConversationStateFromContext({ @@ -140,6 +150,17 @@ function bindController(controller, overrides = {}) { return { cancellation, recorder }; } +test("a stale turn lease cannot unbind a replacement compaction binding", () => { + const controller = new CompactionController(); + const oldLease = controller.bindTurn({ sinks: {} }); + const replacementLease = controller.bindTurn({ sinks: {} }); + + assert.equal(controller.unbindTurn(oldLease), false); + assert.equal(controller.isTurnBound(replacementLease), true); + assert.equal(controller.unbindTurn(replacementLease), true); + assert.equal(controller.isTurnBound(replacementLease), false); +}); + test("pre-send compaction: checkpoint, persist, re-appended user message, paired status", async () => { const controller = new CompactionController(); const baseState = bigState(); @@ -362,6 +383,9 @@ test("a late result cannot settle a newer compaction with the same trigger", asy releaseOld(); await assert.rejects(oldPending, /abort/i); assert.equal(oldBinding.recorder.byKind("persist").length, 0); + // The old task has now unwound. Its finally block must not clear the + // replacement compaction's in-flight guard. + assert.equal(controller.shouldProtectMidStream(1_000_000), false); assert.equal(observed.at(-1)[0], "start"); releaseNew(); @@ -379,6 +403,68 @@ test("a late result cannot settle a newer compaction with the same trigger", asy ); }); +test("a replacement binding keeps a late old terminal on the old observer", async () => { + const controller = new CompactionController(); + const oldObserved = []; + const newObserved = []; + let releaseOld; + const oldGate = new Promise((resolve) => { + releaseOld = resolve; + }); + + const oldBinding = bindController(controller, { + observer: { + onStart: (info) => oldObserved.push(["start", info]), + onEnd: (info) => oldObserved.push(["end", info]), + }, + complete: async () => { + await oldGate; + return summaryResponse(); + }, + }); + const oldPending = controller.compactDuringRun({ trigger: "post-tool", state: bigState() }); + await new Promise((resolve) => setImmediate(resolve)); + + controller.unbindTurn(); + let releaseNew; + const newGate = new Promise((resolve) => { + releaseNew = resolve; + }); + const newBinding = bindController(controller, { + observer: { + onStart: (info) => newObserved.push(["start", info]), + onEnd: (info) => newObserved.push(["end", info]), + }, + complete: async () => { + await newGate; + return summaryResponse(); + }, + }); + const newPending = controller.compactDuringRun({ trigger: "post-tool", state: bigState() }); + await new Promise((resolve) => setImmediate(resolve)); + + releaseOld(); + await assert.rejects(oldPending, /abort/i); + assert.deepEqual( + oldObserved.map(([kind, info]) => [kind, info.status ?? info.trigger]), + [ + ["start", "post-tool"], + ["end", "aborted"], + ], + ); + assert.deepEqual( + newObserved.map(([kind, info]) => [kind, info.status ?? info.trigger]), + [["start", "post-tool"]], + ); + + releaseNew(); + const result = await newPending; + assert.equal(result.outcome, "compacted"); + assert.equal(newObserved.at(-1)[1].status, "complete"); + assert.equal(oldObserved.at(-1)[1].status, "aborted"); + assert.notEqual(oldBinding.cancellation, newBinding.cancellation); +}); + test("summarizer failure degrades to prune and still returns a usable context", async () => { const controller = new CompactionController(); // 大工具输出（200k 字符 ≈ 50k tokens > 40k 保护额度）必须在"最近 2 个用户轮次"之前才可被裁剪。 @@ -618,6 +704,51 @@ test("a rejected checkpoint persist never switches runtime state to the unpersis ); }); +test("a stopped checkpoint write cannot hold compaction open or apply its stale state", async () => { + const controller = new CompactionController(); + const checkpointWrite = deferred(); + const checkpointWriteStarted = deferred(); + const baseState = bigState(); + const pendingUserMessage = user("next question", 9); + const { cancellation, recorder } = bindController(controller, { + complete: async () => summaryResponse(), + presend: { + baseState, + pendingUserText: "next question", + composerText: "next question", + uploadedFiles: [], + composeAppliedState: (state) => + conversationState.appendMessagesToConversation(state, [pendingUserMessage]), + }, + }); + recorder.sinks.persist = async (state) => { + recorder.events.push(["persist", state]); + checkpointWriteStarted.resolve(); + return checkpointWrite.promise; + }; + + const pending = controller.maybeCompactPreSend({ + budgetContext: conversationState.buildRequestContext(baseState), + }); + await checkpointWriteStarted.promise; + + cancellation.userStop.abort(); + await assert.rejects(pending, /abort/i); + + assert.equal(recorder.byKind("applyState").length, 0); + assert.equal(recorder.byKind("queueCheckpoint").length, 0); + assert.equal(await controller.handleTurnAbort(), true); + assert.equal(recorder.byKind("applyStateMidRun").length, 1); + assert.equal(recorder.byKind("applyStateMidRun")[0][1], baseState); + + // The cancelled race keeps the original write observed. A late write failure + // must not revive the old checkpoint or surface as an unhandled rejection. + checkpointWrite.reject(new Error("late checkpoint persistence failure")); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(recorder.byKind("applyState").length, 0); + assert.equal(recorder.byKind("queueCheckpoint").length, 0); +}); + test("registry hands out one controller per conversation and disposes cleanly", () => { const registry = createCompactionControllerRegistry(); const a = registry.get("conv-a"); diff --git a/crates/agent-gui/test/chat/edit-resend-atomic.test.mjs b/crates/agent-gui/test/chat/edit-resend-atomic.test.mjs index 8c0f81064..fa1003865 100644 --- a/crates/agent-gui/test/chat/edit-resend-atomic.test.mjs +++ b/crates/agent-gui/test/chat/edit-resend-atomic.test.mjs @@ -32,7 +32,7 @@ const messageRef = { const sendSource = fs.readFileSync( new URL("../../src/pages/chat/runtime/useSendChatTurn.ts", import.meta.url), "utf8", -); +).replace(/\r\n/g, "\n"); test("edit-resend delegates the replacement anchor to the send preflight", async () => { const calls = []; diff --git a/crates/agent-gui/test/debug/agent-debug.test.mjs b/crates/agent-gui/test/debug/agent-debug.test.mjs index 7f3ed09f0..c67aa7bcf 100644 --- a/crates/agent-gui/test/debug/agent-debug.test.mjs +++ b/crates/agent-gui/test/debug/agent-debug.test.mjs @@ -5,6 +5,16 @@ import { createTsModuleLoader } from "../helpers/load-ts-module.mjs"; const loader = createTsModuleLoader(); const agentDebug = loader.loadModule("src/lib/debug/agentDebug.ts"); +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + test("debug sanitizer redacts base64 data URLs", () => { const payload = { input: [ @@ -166,3 +176,32 @@ test("stream request debug payload omits prefix cache when not provided", () => assert.equal(payload.prefixCache, undefined); }); + +test("background debug flush does not block and observes a late failure", async () => { + const gate = deferred(); + const warnings = []; + const originalWarn = console.warn; + let flushCalls = 0; + console.warn = (...args) => warnings.push(args); + try { + const result = agentDebug.flushDebugLoggerInBackground( + { + flush() { + flushCalls += 1; + return gate.promise; + }, + }, + "test", + ); + + assert.equal(result, undefined); + assert.equal(flushCalls, 1); + + gate.reject(new Error("late debug write failure")); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(warnings.length, 1); + assert.match(String(warnings[0][0]), /Agent dev debug test flush failed/); + } finally { + console.warn = originalWarn; + } +}); diff --git a/crates/agent-gui/test/providers/provider-failover.test.mjs b/crates/agent-gui/test/providers/provider-failover.test.mjs index 6587b3925..c285e9503 100644 --- a/crates/agent-gui/test/providers/provider-failover.test.mjs +++ b/crates/agent-gui/test/providers/provider-failover.test.mjs @@ -80,6 +80,15 @@ async function collectEvents(stream) { return events; } +function resolvesWithin(promise, timeoutMs = 200) { + return Promise.race([ + promise, + new Promise((_, reject) => { + setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs); + }), + ]); +} + function makeCandidate(key, events, extra = {}) { return { key, @@ -246,6 +255,32 @@ test("errors after content committed do not switch providers", async () => { assert.equal(events.at(-1)?.type, "error"); }); +test("abort ends a source that finishes iteration without resolving result", async () => { + const controller = new AbortController(); + const source = piAiEventStream.createAssistantMessageEventStream(); + source.end(); + const stream = withProviderFailover( + [ + { + key: "primary", + label: "primary", + model: { api: "anthropic-messages", provider: "anthropic", id: "primary" }, + start: () => source, + }, + ], + { config: BREAKER_CONFIG, signal: controller.signal }, + ); + + controller.abort(new Error("cancelled by user")); + + const result = await resolvesWithin(stream.result()); + assert.equal(result.stopReason, "aborted"); + assert.equal(result.errorMessage, "Cancelled"); + const events = await collectEvents(stream); + assert.equal(events.at(-1)?.type, "error"); + assert.equal(events.at(-1)?.reason, "aborted"); +}); + test("maxSwitches caps how many candidates are attempted", async () => { let thirdStarted = false; const stream = withProviderFailover( diff --git a/crates/agent-gui/test/providers/stream-retry.test.mjs b/crates/agent-gui/test/providers/stream-retry.test.mjs index bab095560..c463b7b91 100644 --- a/crates/agent-gui/test/providers/stream-retry.test.mjs +++ b/crates/agent-gui/test/providers/stream-retry.test.mjs @@ -42,6 +42,18 @@ function createErrorStream(errorMessage) { }; } +function createErrorStreamWithHangingResult(errorMessage) { + return { + async *[Symbol.asyncIterator]() { + yield { type: "error", error: createAssistant(undefined, "error", { errorMessage }) }; + await new Promise(() => {}); + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + function createSuccessStream(text) { const assistant = createAssistant(text, "stop"); return { @@ -92,12 +104,177 @@ function createAbortedDoneStream() { }; } +function createNeverYieldingStream() { + return { + async *[Symbol.asyncIterator]() { + await new Promise(() => {}); + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + +function createNeverResolvingIteratorStream() { + return { + [Symbol.asyncIterator]() { + return { + next() { + return new Promise(() => {}); + }, + async return() { + return { done: true }; + }, + }; + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + +function createCountingAbortSignal() { + const listeners = new Set(); + return { + signal: { + aborted: false, + reason: undefined, + addEventListener(type, listener) { + if (type === "abort") listeners.add(listener); + }, + removeEventListener(type, listener) { + if (type === "abort") listeners.delete(listener); + }, + }, + get listenerCount() { + return listeners.size; + }, + }; +} + +function createCommittedIdleStream(text) { + const assistant = createAssistant(text, "stop"); + return { + async *[Symbol.asyncIterator]() { + yield { type: "start", partial: { ...assistant, content: [] } }; + yield { + type: "text_delta", + contentIndex: 0, + delta: text, + partial: { ...assistant, content: [{ type: "text", text }] }, + }; + await new Promise(() => {}); + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + +function createNeverResolvingResultStream() { + return { + async *[Symbol.asyncIterator]() { + return; + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + +function createRejectingIteratorStream(errorMessage) { + return { + [Symbol.asyncIterator]() { + return { + async next() { + throw new Error(errorMessage); + }, + async return() { + return { done: true }; + }, + }; + }, + async result() { + throw new Error(errorMessage); + }, + }; +} + +function createRejectingIteratorWithHangingResultStream(errorMessage) { + return { + [Symbol.asyncIterator]() { + return { + async next() { + throw new Error(errorMessage); + }, + async return() { + return { done: true }; + }, + }; + }, + async result() { + return await new Promise(() => {}); + }, + }; +} + +function createRejectingResultStream(errorMessage) { + return { + async *[Symbol.asyncIterator]() {}, + async result() { + throw new Error(errorMessage); + }, + }; +} + +function createResultOnlyErrorStream(errorMessage) { + return { + async *[Symbol.asyncIterator]() {}, + async result() { + return createAssistant(undefined, "error", { errorMessage }); + }, + }; +} + +function createBufferedEventStream(events) { + return { + [Symbol.asyncIterator]() { + let index = 0; + return { + async next() { + return index < events.length + ? { value: events[index++], done: false } + : { done: true }; + }, + async return() { + return { done: true }; + }, + }; + }, + async result() { + return createAssistant(undefined, "aborted"); + }, + }; +} + async function collectEvents(eventStream) { const events = []; for await (const event of eventStream) events.push(event); return events; } +async function resolveWithin(promise, timeoutMs = 250) { + let timeoutId; + const timeout = new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timeoutId); + } +} + test("withStreamRetry succeeds after N retryable errors without leaking failed-attempt events", async () => { let calls = 0; const wrapped = withStreamRetry( @@ -120,6 +297,157 @@ test("withStreamRetry succeeds after N retryable errors without leaking failed-a assert.equal(final.content[0].text, "final answer"); }); +test("withStreamRetry retries a provider iterator rejection before content commits", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 ? createRejectingIteratorStream("fetch failed") : createSuccessStream("recovered"); + }, + { maxAttempts: 3 }, + ); + + const events = await collectEvents(wrapped); + assert.equal(calls, 2); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "done"]); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry does not await result after an iterator rejection", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 + ? createRejectingIteratorWithHangingResultStream("network connection reset") + : createSuccessStream("recovered"); + }, + { maxAttempts: 2, idleTimeoutMs: 5_000 }, + ); + + const events = await resolveWithin(collectEvents(wrapped), 1_500); + assert.equal(calls, 2); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "done"]); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry retries an explicit terminal error without awaiting a hanging result", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 + ? createErrorStreamWithHangingResult("network connection reset") + : createSuccessStream("recovered"); + }, + { maxAttempts: 2, idleTimeoutMs: 5_000 }, + ); + + const events = await resolveWithin(collectEvents(wrapped), 1_500); + assert.equal(calls, 2); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "done"]); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry retries a synchronous provider construction failure", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + if (calls === 1) throw new Error("network unavailable"); + return createSuccessStream("recovered"); + }, + { maxAttempts: 3 }, + ); + + await collectEvents(wrapped); + assert.equal(calls, 2); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry retries a provider result rejection before content commits", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 ? createRejectingResultStream("network timeout") : createSuccessStream("recovered"); + }, + { maxAttempts: 3 }, + ); + + await collectEvents(wrapped); + assert.equal(calls, 2); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry retries a result-only provider error before content commits", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 + ? createResultOnlyErrorStream("503 service unavailable") + : createSuccessStream("recovered"); + }, + { maxAttempts: 3 }, + ); + + const events = await collectEvents(wrapped); + assert.equal(calls, 2); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "done"]); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry retries an uncommitted idle provider stream", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 ? createNeverYieldingStream() : createSuccessStream("recovered"); + }, + { maxAttempts: 2, idleTimeoutMs: 5 }, + ); + + await collectEvents(wrapped); + assert.equal(calls, 2); + assert.equal((await wrapped.result()).content[0].text, "recovered"); +}); + +test("withStreamRetry removes the abort listener after a timed-out read retries", async () => { + let calls = 0; + const abort = createCountingAbortSignal(); + const wrapped = withStreamRetry( + () => { + calls += 1; + return calls === 1 ? createNeverResolvingIteratorStream() : createSuccessStream("recovered"); + }, + { maxAttempts: 2, idleTimeoutMs: 5, signal: abort.signal }, + ); + + const events = await resolveWithin(collectEvents(wrapped), 1_500); + assert.equal(calls, 2); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "done"]); + assert.equal(abort.listenerCount, 0); +}); + +test("withStreamRetry drops buffered ordinary events after an already-aborted stop", async () => { + const controller = new AbortController(); + controller.abort(new Error("cancelled")); + const assistant = createAssistant("late", "stop"); + const wrapped = withStreamRetry( + () => + createBufferedEventStream([ + { type: "text_delta", contentIndex: 0, delta: "late", partial: assistant }, + { type: "done", message: assistant }, + ]), + { signal: controller.signal }, + ); + + const events = await collectEvents(wrapped); + assert.deepEqual(events.map((event) => event.type), ["error"]); + assert.equal(events[0].reason, "aborted"); +}); + test("withStreamRetry invokes onRetry per attempt and onRetryRecovered once content commits", async () => { let calls = 0; const retryCalls = []; @@ -198,6 +526,24 @@ test("withStreamRetry does not retry once content has been committed", async () assert.equal(final.stopReason, "error"); }); +test("withStreamRetry ends a committed stream that becomes idle without retrying it", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return createCommittedIdleStream("partial"); + }, + { maxAttempts: 2, idleTimeoutMs: 5 }, + ); + + const events = await resolveWithin(collectEvents(wrapped), 1_500); + assert.equal(calls, 1); + assert.deepEqual(events.map((event) => event.type), ["start", "text_delta", "error"]); + const final = await wrapped.result(); + assert.equal(final.stopReason, "error"); + assert.match(final.errorMessage, /idle timeout/i); +}); + test("withStreamRetry never retries an aborted stream", async () => { let calls = 0; const wrapped = withStreamRetry(() => { @@ -215,6 +561,44 @@ test("withStreamRetry never retries an aborted stream", async () => { assert.equal(final.stopReason, "aborted"); }); +test("withStreamRetry aborts a provider iterator that never yields", async () => { + const controller = new AbortController(); + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return createNeverYieldingStream(); + }, + { signal: controller.signal }, + ); + const eventsPromise = collectEvents(wrapped); + + controller.abort(new Error("cancelled by user")); + + const events = await resolveWithin(eventsPromise); + assert.equal(calls, 1); + assert.deepEqual(events.map((event) => event.type), ["error"]); + assert.equal(events[0].reason, "aborted"); + assert.equal((await resolveWithin(wrapped.result())).stopReason, "aborted"); +}); + +test("withStreamRetry aborts a provider result that never resolves", async () => { + const controller = new AbortController(); + const wrapped = withStreamRetry(() => createNeverResolvingResultStream(), { + signal: controller.signal, + }); + const eventsPromise = collectEvents(wrapped); + + // Let the eager pump reach source.result() before simulating Stop. + await Promise.resolve(); + controller.abort(new Error("cancelled by user")); + + const events = await resolveWithin(eventsPromise); + assert.deepEqual(events.map((event) => event.type), ["error"]); + assert.equal(events[0].reason, "aborted"); + assert.equal((await resolveWithin(wrapped.result())).stopReason, "aborted"); +}); + test("withStreamRetry respects maxAttempts and surfaces the last failure", async () => { let calls = 0; const wrapped = withStreamRetry( diff --git a/crates/agent-gui/test/providers/text-only-failover.test.mjs b/crates/agent-gui/test/providers/text-only-failover.test.mjs index c6dac12f1..b1b0c591b 100644 --- a/crates/agent-gui/test/providers/text-only-failover.test.mjs +++ b/crates/agent-gui/test/providers/text-only-failover.test.mjs @@ -20,6 +20,7 @@ const streamCalls = []; let streamImpl = () => { throw new Error("streamImpl was not configured for this test"); }; +let hostedSearchProbeFinish = async () => {}; const loader = createTsModuleLoader({ mocks: { @@ -41,7 +42,7 @@ const loader = createTsModuleLoader({ [abs("src/lib/providers/runtime/modelFactory.ts")]: { // Deterministic identity carrying baseUrl so tests can tell targets apart. createModelFromConfig: (providerId, modelId, baseUrl) => ({ - api: "anthropic-messages", + api: providerId === "codex" ? "openai-responses" : "anthropic-messages", provider: providerId, id: modelId, baseUrl, @@ -53,13 +54,12 @@ const loader = createTsModuleLoader({ [abs("src/lib/system/powerActivity.ts")]: { withPowerActivity: (_scope, _reason, run) => run(), }, - [abs("src/lib/debug/agentDebug.ts")]: { - buildStreamRequestDebugPayload: () => ({}), - }, [abs("src/lib/providers/hostedSearchEvents.ts")]: { createHostedSearchProbeId: () => undefined, withHostedSearchProbeHeader: (headers) => headers ?? {}, - startHostedSearchFetchProbe: () => ({ finish: async () => {} }), + startHostedSearchFetchProbe: () => ({ + finish: () => hostedSearchProbeFinish(), + }), createHostedSearchEventAggregator: () => ({ accept: () => {}, complete: () => [], @@ -71,7 +71,7 @@ const loader = createTsModuleLoader({ }, }); -const { streamAssistantMessage } = loader.loadModule( +const { streamAssistantMessage, completeAssistantMessage } = loader.loadModule( "src/lib/providers/runtime/textOnlyRuntime.ts", ); const { resetFailoverBreakers } = loader.loadModule( @@ -174,9 +174,157 @@ function baseParams(overrides = {}) { }; } +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function settlesPromptly(promise) { + let timeoutId = null; + const timedOut = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(false), 100); + }); + try { + return await Promise.race([promise.then(() => true), timedOut]); + } finally { + if (timeoutId !== null) clearTimeout(timeoutId); + } +} + test.beforeEach(() => { resetFailoverBreakers(); streamCalls.length = 0; + hostedSearchProbeFinish = async () => {}; +}); + +test("text mode returns without waiting for diagnostic debug persistence", async () => { + streamImpl = () => successStream("answer"); + const flushGate = deferred(); + let flushCalls = 0; + const originalWarn = console.warn; + console.warn = () => {}; + try { + const run = streamAssistantMessage( + baseParams({ + debugLogger: { + enabled: true, + logRequest() {}, + logResponse() {}, + logResult() {}, + logError() {}, + flush() { + flushCalls += 1; + return flushGate.promise; + }, + }, + }), + ); + + assert.equal(await settlesPromptly(run), true); + assert.equal(flushCalls, 1); + + flushGate.reject(new Error("late debug persistence failure")); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + console.warn = originalWarn; + } +}); + +test("text mode cancellation does not wait for a stalled hosted-search probe", async () => { + streamImpl = () => successStream("answer"); + const controller = new AbortController(); + const finishStarted = deferred(); + const finishGate = deferred(); + let finishCalls = 0; + + hostedSearchProbeFinish = () => { + finishCalls += 1; + finishStarted.resolve(); + return finishGate.promise; + }; + + const outcome = streamAssistantMessage( + baseParams({ + providerId: "codex", + nativeWebSearch: true, + signal: controller.signal, + }), + ).then( + () => "resolved", + () => "rejected", + ); + + await finishStarted.promise; + controller.abort(); + + assert.equal(await settlesPromptly(outcome), true); + assert.equal(await outcome, "rejected"); + assert.equal(finishCalls, 1, "abort cleanup starts the probe finish exactly once"); + + finishGate.resolve(); + await new Promise((resolve) => setImmediate(resolve)); +}); + +test("text mode cancellation does not wait for a stalled provider result", async () => { + const controller = new AbortController(); + const resultStarted = deferred(); + const resultGate = deferred(); + streamImpl = () => ({ + async *[Symbol.asyncIterator]() {}, + result() { + resultStarted.resolve(); + return resultGate.promise; + }, + }); + + const outcome = streamAssistantMessage( + baseParams({ signal: controller.signal }), + ).then( + () => "resolved", + () => "rejected", + ); + + await resultStarted.promise; + controller.abort(); + + assert.equal(await settlesPromptly(outcome), true); + assert.equal(await outcome, "rejected"); + + resultGate.resolve(makeAssistantMessage()); + await new Promise((resolve) => setImmediate(resolve)); +}); + +test("completion cancellation does not wait for a stalled provider result", async () => { + const controller = new AbortController(); + const resultStarted = deferred(); + const resultGate = deferred(); + streamImpl = () => ({ + result() { + resultStarted.resolve(); + return resultGate.promise; + }, + }); + + const outcome = completeAssistantMessage( + baseParams({ signal: controller.signal }), + ).then( + () => "resolved", + () => "rejected", + ); + + await resultStarted.promise; + controller.abort(); + + assert.equal(await settlesPromptly(outcome), true); + assert.equal(await outcome, "rejected"); + + resultGate.resolve(makeAssistantMessage()); + await new Promise((resolve) => setImmediate(resolve)); }); test("text mode fails over to the queued provider before content commits", async () => { diff --git a/crates/agent-gui/test/trajectory/desktop-live.test.mjs b/crates/agent-gui/test/trajectory/desktop-live.test.mjs index f70577f8b..04509e41c 100644 --- a/crates/agent-gui/test/trajectory/desktop-live.test.mjs +++ b/crates/agent-gui/test/trajectory/desktop-live.test.mjs @@ -90,3 +90,38 @@ test("the recorder registry exclusively owns desktop live trajectory writes", () assert.doesNotMatch(source, /appendDesktopLiveTrajectory/); } }); + +test("late scoped trajectory events publish through their owning run", () => { + const conversationId = "desktop-live-interleaved-runs"; + const publishedByRun = { old: [], replacement: [] }; + clearDesktopLiveTrajectory(conversationId); + + const oldRun = acquireTrajectoryRecorder( + conversationId, + 0, + (events) => publishedByRun.old.push(...events), + 7, + ); + const replacementRun = acquireTrajectoryRecorder( + conversationId, + 0, + (events) => publishedByRun.replacement.push(...events), + 8, + ); + + try { + oldRun.recorder.beginTurn({ turn: 7, messageIndex: 0, text: "old" }); + replacementRun.recorder.beginTurn({ turn: 8, messageIndex: 1, text: "replacement" }); + oldRun.recorder.stepStart(1); + replacementRun.recorder.stepStart(1); + oldRun.recorder.endTurn({ status: "aborted" }); + replacementRun.recorder.endTurn({ status: "complete" }); + + assert.equal(publishedByRun.old.every((event) => event.t === 7), true); + assert.equal(publishedByRun.replacement.every((event) => event.t === 8), true); + assert.equal(publishedByRun.old.some((event) => event.k === "turn_end"), true); + assert.equal(publishedByRun.replacement.some((event) => event.k === "turn_end"), true); + } finally { + discardTrajectoryRecorder(conversationId); + } +}); diff --git a/crates/agent-gui/test/trajectory/recorder.test.mjs b/crates/agent-gui/test/trajectory/recorder.test.mjs index d42f65464..4382586e6 100644 --- a/crates/agent-gui/test/trajectory/recorder.test.mjs +++ b/crates/agent-gui/test/trajectory/recorder.test.mjs @@ -4,7 +4,7 @@ import test from "node:test"; import { createTsModuleLoader } from "../helpers/load-ts-module.mjs"; const loader = createTsModuleLoader(); -const { createTrajectoryRecorder, NOOP_TRAJECTORY_RECORDER } = loader.loadModule( +const { createTrajectoryRecorder, scopeTrajectoryRecorder, NOOP_TRAJECTORY_RECORDER } = loader.loadModule( "src/lib/trajectory/recorder.ts", ); @@ -366,6 +366,27 @@ test("turn end is idempotent across happy-path and finalizer calls", () => { assert.equal(published.find((event) => event.k === "turn_end").st, "complete"); }); +test("interleaved scoped runs close only their own trajectory turn", () => { + const { recorder, published } = harness(); + const first = scopeTrajectoryRecorder(recorder, 1); + const second = scopeTrajectoryRecorder(recorder, 2); + + first.beginTurn({ turn: 1 }); + first.stepStart(1); + second.beginTurn({ turn: 2 }); + second.stepStart(1); + first.endTurn({ status: "aborted" }); + second.endTurn({ status: "complete" }); + + assert.deepEqual( + published.filter((event) => event.k === "turn_end").map((event) => [event.t, event.st]), + [ + [1, "aborted"], + [2, "complete"], + ], + ); +}); + test("new request headers declare the seven-slot runtime layout version", () => { const { recorder, published } = harness(); const headerId = recorder.captureHeader({ diff --git a/crates/agent-gui/test/trajectory/text-mode.test.mjs b/crates/agent-gui/test/trajectory/text-mode.test.mjs index b885f20e5..2ff6677e1 100644 --- a/crates/agent-gui/test/trajectory/text-mode.test.mjs +++ b/crates/agent-gui/test/trajectory/text-mode.test.mjs @@ -89,6 +89,28 @@ function recorderHarness() { return { recorder, calls }; } +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function settlesPromptly(promise) { + let timeoutId = null; + const timedOut = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(false), 100); + }); + try { + return await Promise.race([promise.then(() => true), timedOut]); + } finally { + if (timeoutId !== null) clearTimeout(timeoutId); + } +} + function baseParams(recorder) { let state = { messages: [] }; const stop = new AbortController(); @@ -197,6 +219,64 @@ test("text mode records the exact request boundary, TTFT, terminal model and tur assert.equal(calls.at(-1)[0], "flush"); }); +test("text mode releases a completed response when its terminal history write stalls", async () => { + const final = assistant(); + const { runTextConversationTurn } = loadTurn(async (params) => { + params.onRequestStart?.({ context: params.context }); + return final; + }); + const { recorder } = recorderHarness(); + const params = baseParams(recorder); + const persistStarted = deferred(); + const persistGate = deferred(); + let committed = 0; + params.onTerminalResponseCommitted = () => { + committed += 1; + }; + params.persistConversationWithHistorySync = () => { + persistStarted.resolve(); + return persistGate.promise; + }; + + const run = runTextConversationTurn(params); + await persistStarted.promise; + assert.equal(committed, 1); + + params.cancellation.userStop.abort(); + assert.equal(await settlesPromptly(run), true); + + persistGate.resolve(true); + await run; +}); + +test("text mode does not wait for diagnostic trajectory persistence", async () => { + const final = assistant(); + const { runTextConversationTurn } = loadTurn(async (params) => { + params.onRequestStart?.({ context: params.context }); + return final; + }); + const { recorder } = recorderHarness(); + const flushGate = deferred(); + let flushes = 0; + recorder.flush = () => { + flushes += 1; + return flushGate.promise; + }; + + const originalWarn = console.warn; + console.warn = () => {}; + try { + const run = runTextConversationTurn(baseParams(recorder)); + assert.equal(await settlesPromptly(run), true); + assert.equal(flushes, 1); + + flushGate.reject(new Error("late trajectory persistence failure")); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + console.warn = originalWarn; + } +}); + test("text mode preserves error and aborted assistant outcomes at both terminal levels", async () => { for (const [stopReason, expectedStatus] of [ ["error", "error"], diff --git a/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx b/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx index 9449c9a34..e6f5a9ed3 100644 --- a/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx +++ b/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx @@ -1185,6 +1185,22 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer
+ {canQueueDraftWhileSending ? ( + + ) : null}