From 79e01d397ef6f17a4b9106110d16ff68d79e8e79 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Mon, 21 Sep 2026 09:43:09 +0900 Subject: [PATCH 01/12] fix(combos): retry single targets only after cooldown --- src/server/responses/core-combo.ts | 2 ++ tests/server/server-combo-failover-e2e.test.ts | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 7f285a89e7b..e455b30a4b1 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -767,6 +767,7 @@ export async function executeComboResponses( code: failure.upstreamCode, message: failure.classificationText, }); + const failedTargetCooled = isComboTargetInCooldown(comboId, pick.target, failureNow); // Same target selector as the exclusionary pick below, minus `exclude`: the only // difference is deliberate and is the whole point of the single-target retry. const retryAfterCooldown = () => @@ -797,6 +798,7 @@ export async function executeComboResponses( && combo.targets.length === 1 && combo.waitForCooldownMs > 0 && comboTargetsDispatched <= 1 + && failedTargetCooled && !options.abortSignal?.aborted ) { pick = await retryAfterCooldown(); diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index 1fd65f10304..fb4f8aa1afb 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -2256,6 +2256,19 @@ describe("server combo failover 030 activation matrix", () => { expect(hits).toBe(1); }); + test("single-target wait does not retry a request-local refusal", async () => { + let hits = 0; + const upstream = serve(() => { + hits += 1; + return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); + }); + const response = await post(comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ + { provider: "a", model: "m1" }, + ], { cooldownMs: 50, waitForCooldownMs: 500 }), { user: "synthetic-client" }); + expect(response.status).toBe(400); + expect(hits).toBe(1); + }); + test("a past Retry-After date remains immediate through response consumption", async () => { const now = Date.parse("2026-07-18T00:00:00.000Z"); const failure = await consumeComboFailure(Response.json({ error: { message: "rate limited" } }, { From b7fcd27868f9b2d4158b14e37b59eb8752bb5800 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:44:11 +0000 Subject: [PATCH 02/12] fix(combos): gate single-target retry on this failure's cooldown scope A concurrent request can cool the shared target while this request's own failure recorded no cooldown (scope "none"), so isComboTargetInCooldown alone was enough to arm the exclusion-free retry and replay a refused request. Require a cooldown-producing scope alongside the shared-state check, and cover the interleaving with a synchronized two-request test. Co-Authored-By: Epinephrine --- src/server/responses/core-combo.ts | 14 ++++-- .../server/server-combo-failover-e2e.test.ts | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index e455b30a4b1..1569a137697 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -754,20 +754,26 @@ export async function executeComboResponses( ); const failureNow = Date.now(); const attemptedTargets = pick.attempted; + const failureCooldownScope = comboFailureCooldownScope(failure.response.status, failure.classificationText, { + code: failure.upstreamCode, + }); const nextPick = advanceComboAfterFailure(config, pick, { retryAfter: failure.retryAfter, resetAt: failure.resetAt, cooldownMs: combo.cooldownMs, now: failureNow, - cooldownScope: comboFailureCooldownScope(failure.response.status, failure.classificationText, { - code: failure.upstreamCode, - }), + cooldownScope: failureCooldownScope, eligible: targetEligible, status: failure.response.status, code: failure.upstreamCode, message: failure.classificationText, }); - const failedTargetCooled = isComboTargetInCooldown(comboId, pick.target, failureNow); + // Cooldown state is shared by every request using this target, so a concurrent failure + // can put it in cooldown while THIS failure recorded none. Scope "none" means the + // refusal described this request's shape rather than the target's health — only a + // cooldown this failure produced itself may arm the single-target retry below. + const failedTargetCooled = failureCooldownScope !== "none" + && isComboTargetInCooldown(comboId, pick.target, failureNow); // Same target selector as the exclusionary pick below, minus `exclude`: the only // difference is deliberate and is the whole point of the single-target retry. const retryAfterCooldown = () => diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index fb4f8aa1afb..3801b1ab2a3 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -2269,6 +2269,51 @@ describe("server combo failover 030 activation matrix", () => { expect(hits).toBe(1); }); + test("a concurrent cooldown does not retry a request-local refusal", async () => { + // Two requests share one target: the first stays in flight on a gate while the second + // fails hot and writes the SHARED cooldown. The first request's own failure records no + // cooldown (scope "none"), so the foreign entry alone must not arm the retry gate — + // it would wait out the sibling's cooldown and replay the refused request. + let markHeld!: () => void; + let releaseHeld!: () => void; + const heldRequest = new Promise(resolve => { markHeld = resolve; }); + const gate = new Promise(resolve => { releaseHeld = resolve; }); + let hits = 0; + const upstream = serve(async () => { + hits += 1; + if (hits === 1) { + markHeld(); + await gate; + return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); + } + // 429 rather than 5xx so the failure reaches the combo layer directly: + // fetchWithTransientRetry would absorb a 503 before it could cool the target. + return hits === 2 + ? Response.json({ error: { message: "rate limited" } }, { status: 429 }) + : chatSuccess("single target recovered", "m1"); + }); + const config = comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ + { provider: "a", model: "m1" }, + ], { cooldownMs: 500, waitForCooldownMs: 2_000 }); + + const refused = post(config, { user: "synthetic-client" }); + await heldRequest; + const cooling = post(config); + const target = { provider: "a", model: "m1" }; + const deadline = Date.now() + 5_000; + while (!isComboTargetInCooldown("free", target)) { + if (Date.now() > deadline) throw new Error("sibling request never cooled the target"); + await Bun.sleep(5); + } + releaseHeld(); + const [refusal, cooled] = await Promise.all([refused, cooling]); + expect(refusal.status).toBe(400); + expect(cooled.status).toBe(200); + // The cooling request hits twice (failure, then its own post-cooldown retry); the + // refused request must hit exactly once. + expect(hits).toBe(3); + }); + test("a past Retry-After date remains immediate through response consumption", async () => { const now = Date.parse("2026-07-18T00:00:00.000Z"); const failure = await consumeComboFailure(Response.json({ error: { message: "rate limited" } }, { From f16ca52185af0ff32cad25601ad8db2dfc6525cd Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:39:56 +0900 Subject: [PATCH 03/12] docs(combos): cover the single-target post-cooldown retry --- docs-site/src/content/docs/guides/combos.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 82cb31fecfd..1bc94093233 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -277,7 +277,10 @@ dispatches normally. by default, so existing combos are unaffected. Only the exact string `before-last-resort` opts in. -The current request never retries the same attempted target. Later requests skip a cooled target until its +The current request never retries the same attempted target — with one exception: a single-target combo +that sets `waitForCooldownMs` may retry its only target once that target's cooldown expires inside the +same request, since there is no alternate to fail over to. Request-local compatibility rejections still +return without any retry. Later requests skip a cooled target until its cooldown expires; request-local compatibility rejections do not cool the target. A `Retry-After` HTTP-date that is already in the past is also preserved as an immediate upstream directive, just like `Retry-After: 0`. Set `waitForCooldownMs` to allow a later request to wait for the earliest eligible target cooldown, up to that cap on each selection attempt, From fe45a02f06f8d654d17e56ad2cee07356c3d8f1a Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:56:24 +0900 Subject: [PATCH 04/12] docs(combos): translate the single-target retry exception --- docs-site/src/content/docs/ja/guides/combos.md | 2 +- docs-site/src/content/docs/ko/guides/combos.md | 2 +- docs-site/src/content/docs/ru/guides/combos.md | 2 +- docs-site/src/content/docs/zh-cn/guides/combos.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs-site/src/content/docs/ja/guides/combos.md b/docs-site/src/content/docs/ja/guides/combos.md index 217636f4672..8f3aa996687 100644 --- a/docs-site/src/content/docs/ja/guides/combos.md +++ b/docs-site/src/content/docs/ja/guides/combos.md @@ -131,7 +131,7 @@ ocx combo set balanced \ `cooldownMs` が未設定の場合、ホップされたターゲットはアップストリームのフォールバックを使用します。アップストリームコード `1302` または `1305` を伴うリクエストレート 429 では 5 秒、それ以外では 60 秒です。設定されている場合、使用可能なアップストリームの `Retry-After` または Codex リセットシグナルが存在しないときは、これらのリクエストレート 429 を含め、`cooldownMs` が適用されます。数値の `Retry-After` 秒数と HTTP-date 値が受け入れられ、明示的な上流の `Retry-After` は最大 24 時間、リセット由来・設定済み・フォールバックのクールダウンは最大 10 分です。優先順位は強い順に、明示的な `Retry-After` → Codex リセットヘッダー(`x-codex-primary-reset-at`、`x-codex-secondary-reset-at`、または `x-codex-tertiary-reset-at`)→ コンボの `cooldownMs`(設定時)→ アップストリームのレート制限コード `1302`/`1305` に対する 5 秒のリクエストレート フォールバック → 60 秒のデフォルトです。有効な即時指定 `Retry-After: 0` は、設定されたクールダウンで置き換えられず、即時のアップストリーム指示として維持されます。 -現在のリクエストは、同じ試行ターゲットを再試行しません。後続のリクエストでは、クールダウンが期限切れになるまでそのターゲットをスキップします。すでに過去の時刻である `Retry-After` の HTTP-date も、`Retry-After: 0` と同様に即時のアップストリーム指示として維持されます。`waitForCooldownMs` を設定すると、後続のリクエストは、最も早く利用可能になるターゲットのクールダウンを、その選択試行ごとにこの上限まで待ってから、新たに 1 回選択できます。したがって、複数のフェイルオーバー ホップをまたぐリクエストは合計で `hops × waitForCooldownMs` まで待つ場合があります。デフォルトは `0` です。適格なターゲットがすべて冷却中の場合、HTTP 503 ですぐにフェイルクローズし、その `combo_unavailable` 503 には最も早く終了する残りのクールダウンと等しい `Retry-After` ヘッダーが含まれ、秒単位に切り上げられ、最小値は 1 秒です。待機にはジッターがないため、同期したウェイクアップが発生する可能性があります。中止されたリクエストはこの待機をキャンセルし、通常の `client_cancelled` 応答を返します。キャンセル後にバックアップ ターゲットをディスパッチすることはありません。コンボ ターゲットのクールダウンはプロセス ローカルなコンボごとの状態であり、ネイティブ アカウント ルーティングで使用されるアカウントレベルの Codex クォータ クールダウンとは別です。 +現在のリクエストは、同じ試行ターゲットを再試行しません。ただし例外が1つあります。`waitForCooldownMs` を設定した単一ターゲット コンボは、フェイルオーバー先の代替がないため、同じリクエスト内でそのターゲットのクールダウンが期限切れになれば、唯一のターゲットを再試行できます。リクエストローカルの互換性拒否は、引き続き再試行せずに返されます。後続のリクエストでは、クールダウンが期限切れになるまでそのターゲットをスキップします。すでに過去の時刻である `Retry-After` の HTTP-date も、`Retry-After: 0` と同様に即時のアップストリーム指示として維持されます。`waitForCooldownMs` を設定すると、後続のリクエストは、最も早く利用可能になるターゲットのクールダウンを、その選択試行ごとにこの上限まで待ってから、新たに 1 回選択できます。したがって、複数のフェイルオーバー ホップをまたぐリクエストは合計で `hops × waitForCooldownMs` まで待つ場合があります。デフォルトは `0` です。適格なターゲットがすべて冷却中の場合、HTTP 503 ですぐにフェイルクローズし、その `combo_unavailable` 503 には最も早く終了する残りのクールダウンと等しい `Retry-After` ヘッダーが含まれ、秒単位に切り上げられ、最小値は 1 秒です。待機にはジッターがないため、同期したウェイクアップが発生する可能性があります。中止されたリクエストはこの待機をキャンセルし、通常の `client_cancelled` 応答を返します。キャンセル後にバックアップ ターゲットをディスパッチすることはありません。コンボ ターゲットのクールダウンはプロセス ローカルなコンボごとの状態であり、ネイティブ アカウント ルーティングで使用されるアカウントレベルの Codex クォータ クールダウンとは別です。 :::note フェイルオーバーは意図的に制限されています。これは、ターゲット固有の可用性、認証、クォータ、および過負荷の障害に役立ちます。呼び出し元のエラーやポリシーの拒否は隠蔽されません。 diff --git a/docs-site/src/content/docs/ko/guides/combos.md b/docs-site/src/content/docs/ko/guides/combos.md index 614eea8873a..1b6fda9726d 100644 --- a/docs-site/src/content/docs/ko/guides/combos.md +++ b/docs-site/src/content/docs/ko/guides/combos.md @@ -141,7 +141,7 @@ ocx combo set balanced \ `cooldownMs`가 설정되지 않으면 홉된 대상은 업스트림 폴백을 사용합니다. 업스트림 코드 `1302` 또는 `1305`인 요청 속도 제한 429는 5초, 그 외에는 60초입니다. 설정하면 사용 가능한 업스트림 `Retry-After` 또는 Codex 재설정 신호가 없을 때, 해당 요청 속도 제한 429를 포함해 `cooldownMs`가 적용됩니다. 숫자로 된 `Retry-After` 초와 HTTP-date 값을 허용합니다. 명시적 서버 지연은 최대 24시간, 재설정 신호·설정값·폴백 쿨다운은 최대 10분으로 제한됩니다. 우선순위는 강한 순서대로 명시적 `Retry-After` → Codex 재설정 헤더(`x-codex-primary-reset-at`, `x-codex-secondary-reset-at`, 또는 `x-codex-tertiary-reset-at`) → 콤보의 `cooldownMs`(설정된 경우) → 업스트림 속도 제한 코드 `1302`/`1305`의 5초 요청 속도 제한 폴백 → 60초 기본값입니다. 유효한 즉시 지시인 `Retry-After: 0`은 설정된 쿨다운으로 대체하지 않고 업스트림의 즉시 지시로 유지합니다. -현재 요청은 이미 시도한 대상을 다시 시도하지 않습니다. 이후 요청은 쿨다운이 끝날 때까지 해당 대상을 건너뜁니다. 이미 지난 시각을 가리키는 `Retry-After` HTTP-date도 `Retry-After: 0`과 마찬가지로 업스트림의 즉시 지시로 유지됩니다. `waitForCooldownMs`를 설정하면 이후 요청은 가장 먼저 적합해지는 대상의 쿨다운을 선택 시도마다 이 한도까지 기다린 뒤 새로 한 번 선택합니다. 따라서 여러 failover 홉을 거치는 요청은 총 `hops × waitForCooldownMs`까지 기다릴 수 있습니다. 기본값은 `0`입니다. 모든 적합한 대상이 쿨다운 중이고 대기 한도가 0이거나 가장 이른 만료 시각이 대기 한도를 넘으면 요청은 즉시 HTTP 503으로 종료됩니다. 이 `combo_unavailable` 503에는 가장 이른 잔여 쿨다운과 같은 `Retry-After` 헤더가 포함되며, 값은 올림해 정수 초로 표시되고 최소 1초입니다. 대기에 지터를 적용하지 않으므로 동시에 깨어날 수 있습니다. 요청이 중단되면 이 대기가 취소되고 정상 `client_cancelled` 응답이 반환됩니다. 취소 후 백업 대상을 디스패치하지 않습니다. 콤보 대상 쿨다운은 프로세스 로컬 콤보별 상태입니다. 네이티브 계정 라우팅에서 사용하는 계정 수준 Codex 쿼터 쿨다운과는 별개입니다. +현재 요청은 이미 시도한 대상을 다시 시도하지 않습니다. 단 한 가지 예외가 있습니다. `waitForCooldownMs`를 설정한 단일 대상 콤보는 대체할 다른 대상이 없으므로, 같은 요청 안에서 그 대상의 쿨다운이 끝나면 유일한 대상을 다시 시도할 수 있습니다. 요청 로컬 호환성 거부는 여전히 재시도 없이 반환됩니다. 이후 요청은 쿨다운이 끝날 때까지 해당 대상을 건너뜁니다. 이미 지난 시각을 가리키는 `Retry-After` HTTP-date도 `Retry-After: 0`과 마찬가지로 업스트림의 즉시 지시로 유지됩니다. `waitForCooldownMs`를 설정하면 이후 요청은 가장 먼저 적합해지는 대상의 쿨다운을 선택 시도마다 이 한도까지 기다린 뒤 새로 한 번 선택합니다. 따라서 여러 failover 홉을 거치는 요청은 총 `hops × waitForCooldownMs`까지 기다릴 수 있습니다. 기본값은 `0`입니다. 모든 적합한 대상이 쿨다운 중이고 대기 한도가 0이거나 가장 이른 만료 시각이 대기 한도를 넘으면 요청은 즉시 HTTP 503으로 종료됩니다. 이 `combo_unavailable` 503에는 가장 이른 잔여 쿨다운과 같은 `Retry-After` 헤더가 포함되며, 값은 올림해 정수 초로 표시되고 최소 1초입니다. 대기에 지터를 적용하지 않으므로 동시에 깨어날 수 있습니다. 요청이 중단되면 이 대기가 취소되고 정상 `client_cancelled` 응답이 반환됩니다. 취소 후 백업 대상을 디스패치하지 않습니다. 콤보 대상 쿨다운은 프로세스 로컬 콤보별 상태입니다. 네이티브 계정 라우팅에서 사용하는 계정 수준 Codex 쿼터 쿨다운과는 별개입니다. :::note 페일오버는 의도적으로 범위를 제한합니다. 대상별 가용성, 인증, 쿼터, 과부하 실패에는 도움이 되지만, 호출자 오류나 정책 거부를 숨기지는 않습니다. diff --git a/docs-site/src/content/docs/ru/guides/combos.md b/docs-site/src/content/docs/ru/guides/combos.md index cb2ca35806f..77a320cc449 100644 --- a/docs-site/src/content/docs/ru/guides/combos.md +++ b/docs-site/src/content/docs/ru/guides/combos.md @@ -168,7 +168,7 @@ ocx combo set balanced \ Если `cooldownMs` не задан, цель после hop использует upstream fallback: 5 секунд для 429, ограничивающих частоту запросов, с кодом upstream `1302` или `1305`, и 60 секунд в остальных случаях. Если он задан, `cooldownMs` применяется, когда нет пригодного сигнала upstream `Retry-After` или сигнала сброса Codex, включая такие 429, ограничивающие частоту запросов. Принимаются числовые секунды в `Retry-After` и значения HTTP-date; явный `Retry-After` ограничен 24 часами, а cooldown по сбросу, настройке или fallback — 10 минутами. Приоритет от сильного к слабому: явный `Retry-After` → заголовки сброса Codex (`x-codex-primary-reset-at`, `x-codex-secondary-reset-at` или `x-codex-tertiary-reset-at`) → `cooldownMs` этой combo (если задан) → 5-секундный fallback для rate-limit-кодов upstream `1302`/`1305` → стандартные 60 секунд. Корректный немедленный `Retry-After: 0` сохраняется как немедленная директива upstream, а не заменяется настроенным cooldown. -Текущий запрос никогда не повторяет уже опробованную цель. Более поздние запросы пропускают её, пока не истечёт cooldown. HTTP-date в `Retry-After`, указывающий на уже прошедшее время, также сохраняется как немедленная директива upstream, как и `Retry-After: 0`. Задайте `waitForCooldownMs`, чтобы следующий запрос мог подождать cooldown цели, которая станет подходящей раньше всех, до этого ограничения при каждой попытке выбора, а затем выполнить один новый выбор. Поэтому запрос с несколькими hop в failover может ждать в общей сложности до `hops × waitForCooldownMs`. По умолчанию это `0`: если все подходящие цели находятся в cooldown, запрос немедленно завершается HTTP 503; этот ответ `combo_unavailable` содержит заголовок `Retry-After`, равный оставшемуся cooldown цели с самым ранним окончанием, округлённый вверх до целых секунд, минимум до 1 секунды. К ожиданиям не добавляется джиттер, поэтому возможны синхронные пробуждения. Отмена запроса отменяет это ожидание и возвращает обычный ответ `client_cancelled`; после отмены резервная цель не запускается. Cooldown цели combo — состояние процесса, отдельное для каждой combo; он не связан с cooldown квоты Codex на уровне аккаунта, который используется нативной маршрутизацией аккаунта. +Текущий запрос никогда не повторяет уже опробованную цель — с одним исключением: combo с единственной целью и заданным `waitForCooldownMs` может повторить свою единственную цель, когда её cooldown истекает внутри того же запроса, поскольку альтернативы для failover нет. Отказы совместимости, локальные для запроса, по-прежнему возвращаются без повторной попытки. Более поздние запросы пропускают её, пока не истечёт cooldown. HTTP-date в `Retry-After`, указывающий на уже прошедшее время, также сохраняется как немедленная директива upstream, как и `Retry-After: 0`. Задайте `waitForCooldownMs`, чтобы следующий запрос мог подождать cooldown цели, которая станет подходящей раньше всех, до этого ограничения при каждой попытке выбора, а затем выполнить один новый выбор. Поэтому запрос с несколькими hop в failover может ждать в общей сложности до `hops × waitForCooldownMs`. По умолчанию это `0`: если все подходящие цели находятся в cooldown, запрос немедленно завершается HTTP 503; этот ответ `combo_unavailable` содержит заголовок `Retry-After`, равный оставшемуся cooldown цели с самым ранним окончанием, округлённый вверх до целых секунд, минимум до 1 секунды. К ожиданиям не добавляется джиттер, поэтому возможны синхронные пробуждения. Отмена запроса отменяет это ожидание и возвращает обычный ответ `client_cancelled`; после отмены резервная цель не запускается. Cooldown цели combo — состояние процесса, отдельное для каждой combo; он не связан с cooldown квоты Codex на уровне аккаунта, который используется нативной маршрутизацией аккаунта. :::note Failover намеренно ограничен. Он помогает при проблемах доступности конкретной цели, diff --git a/docs-site/src/content/docs/zh-cn/guides/combos.md b/docs-site/src/content/docs/zh-cn/guides/combos.md index 31a5764aa7a..b5702b3a8c7 100644 --- a/docs-site/src/content/docs/zh-cn/guides/combos.md +++ b/docs-site/src/content/docs/zh-cn/guides/combos.md @@ -157,7 +157,7 @@ combo 失败分为 **跳转** 失败和 **终止** 失败。 未设置 `cooldownMs` 时,发生跳转的目标使用上游回退值:对于上游代码为 `1302` 或 `1305` 的请求速率限制 429,等待 5 秒;其他情况等待 60 秒。设置后,只要不存在可用的上游 `Retry-After` 或 Codex 重置信号,就会应用 `cooldownMs`,包括这些请求速率限制 429。接受数字形式的 `Retry-After` 秒数和 HTTP-date 值,显式上游 `Retry-After` 最多 24 小时;重置推导、配置和回退冷却最多 10 分钟。优先级从强到弱依次为:显式 `Retry-After` → Codex 重置标头(`x-codex-primary-reset-at`、`x-codex-secondary-reset-at` 或 `x-codex-tertiary-reset-at`)→ combo 的 `cooldownMs`(已设置时)→ 上游速率限制代码 `1302`/`1305` 的 5 秒请求速率限制回退值 → 60 秒默认值。有效的即时指令 `Retry-After: 0` 会保留为上游即时指令,不会被配置的冷却替换。 -当前请求不会再次重试同一个已经尝试过的目标。后续请求会跳过它,直到冷却结束。已过去的 HTTP-date `Retry-After` 同样会像 `Retry-After: 0` 一样保留为上游即时指令。设置 `waitForCooldownMs` 后,后续请求可以等待最早恢复资格的目标的冷却,单次选择尝试最多等待该上限,然后重新选择一次。因此,多次故障切换跳转的请求总共最多等待 `hops × waitForCooldownMs`。默认值为 `0`;当所有合格目标都处于冷却中时,请求会立即失败并返回 HTTP 503;该 `combo_unavailable` 503 会带有 `Retry-After` 标头,其值等于剩余冷却时间最短的目标,向上取整为整秒,最小值为 1 秒。等待不加入抖动,因此可能同时唤醒。请求中止会取消这次等待并返回正常的 `client_cancelled` 响应;取消后不会调度备用目标。combo 目标冷却是进程本地、按 combo 区分的状态,与原生账户路由使用的账户级 Codex 配额冷却彼此独立。 +当前请求不会再次重试同一个已经尝试过的目标——但有一个例外:设置了 `waitForCooldownMs` 的单目标 combo 由于没有可故障转移的替代目标,可以在同一请求内该目标的冷却结束后重试其唯一目标。请求本地的兼容性拒绝仍然直接返回而不重试。后续请求会跳过它,直到冷却结束。已过去的 HTTP-date `Retry-After` 同样会像 `Retry-After: 0` 一样保留为上游即时指令。设置 `waitForCooldownMs` 后,后续请求可以等待最早恢复资格的目标的冷却,单次选择尝试最多等待该上限,然后重新选择一次。因此,多次故障切换跳转的请求总共最多等待 `hops × waitForCooldownMs`。默认值为 `0`;当所有合格目标都处于冷却中时,请求会立即失败并返回 HTTP 503;该 `combo_unavailable` 503 会带有 `Retry-After` 标头,其值等于剩余冷却时间最短的目标,向上取整为整秒,最小值为 1 秒。等待不加入抖动,因此可能同时唤醒。请求中止会取消这次等待并返回正常的 `client_cancelled` 响应;取消后不会调度备用目标。combo 目标冷却是进程本地、按 combo 区分的状态,与原生账户路由使用的账户级 Codex 配额冷却彼此独立。 :::note 故障切换是有边界的。它有助于处理特定目标的可用性、认证、配额和过载失败;它不会掩盖调用方错误或策略拒绝。 From fbc3f1d85e7ba5a8c2c3c495c2a9fd5c7997627f Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 25 Sep 2026 07:26:33 +0900 Subject: [PATCH 05/12] docs(server): align single-target cooldown retry comment with behavior --- src/server/responses/core-combo.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 1569a137697..4aba19f20a2 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -770,8 +770,9 @@ export async function executeComboResponses( }); // Cooldown state is shared by every request using this target, so a concurrent failure // can put it in cooldown while THIS failure recorded none. Scope "none" means the - // refusal described this request's shape rather than the target's health — only a - // cooldown this failure produced itself may arm the single-target retry below. + // refusal described this request's shape rather than the target's health. The retry + // below arms whenever this failure is cooldown-scoped and the target is currently + // cooling down, regardless of which request recorded that cooldown. const failedTargetCooled = failureCooldownScope !== "none" && isComboTargetInCooldown(comboId, pick.target, failureNow); // Same target selector as the exclusionary pick below, minus `exclude`: the only From 3d47f5371cb64f7af09d540bb49b38ae97d1c13d Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 25 Sep 2026 07:41:33 +0900 Subject: [PATCH 06/12] docs(server): shorten cooldown retry comment to stay under size ratchet --- src/server/responses/core-combo.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 4aba19f20a2..168e1f6207a 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -770,9 +770,8 @@ export async function executeComboResponses( }); // Cooldown state is shared by every request using this target, so a concurrent failure // can put it in cooldown while THIS failure recorded none. Scope "none" means the - // refusal described this request's shape rather than the target's health. The retry - // below arms whenever this failure is cooldown-scoped and the target is currently - // cooling down, regardless of which request recorded that cooldown. + // refusal described this request's shape rather than the target's health — the retry + // arms on any live cooldown for this target, whoever recorded it. const failedTargetCooled = failureCooldownScope !== "none" && isComboTargetInCooldown(comboId, pick.target, failureNow); // Same target selector as the exclusionary pick below, minus `exclude`: the only From 1e64a046e87406e4f8f1813a68f58c65d17de52e Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 25 Sep 2026 07:49:52 +0900 Subject: [PATCH 07/12] test(server): raise combo-failover e2e size cap for the new refusal tests --- tests/fixtures/file-size-baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/file-size-baseline.json b/tests/fixtures/file-size-baseline.json index 716b586605b..d13b2adec78 100644 --- a/tests/fixtures/file-size-baseline.json +++ b/tests/fixtures/file-size-baseline.json @@ -57,7 +57,7 @@ "tests/server/config.test.ts": 3828, "tests/server/management-provider-validation.test.ts": 5506, "tests/server/server-auth.test.ts": 4589, - "tests/server/server-combo-failover-e2e.test.ts": 4166, + "tests/server/server-combo-failover-e2e.test.ts": 4218, "tests/server/server-images.test.ts": 2755, "tests/server/server-live.test.ts": 2253, "tests/service/service.test.ts": 4106, From d0e08ae7cc83cf10213578b566ff1ed5ca7fb639 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:21:00 +0000 Subject: [PATCH 08/12] fix(combos): require this failure to record the retried cooldown --- scripts/test-layout/layout.json | 5 +- src/combos/failover.ts | 5 +- src/combos/resolve.ts | 4 +- src/server/responses/core-combo.ts | 12 ++- structure/transports/responses-failover.md | 7 ++ tests/fixtures/test-layout-expected.json | 3 +- .../server-combo-cooldown-recording.test.ts | 99 +++++++++++++++++++ 7 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 tests/server/server-combo-cooldown-recording.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b1dbc20861f..91118999b90 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1688,7 +1688,7 @@ "url-normalization.test.ts": "config", "usage-aggregate-cache.test.ts": "usage", "usage-anthropic-fast-pricing.test.ts": "usage", - "usage-cursor-fast-pricing.test.ts": "usage", + "usage-cursor-fast-pricing.test.ts": "usage", "usage-attempt-delivery.test.ts": "usage", "usage-cost.test.ts": "usage", "usage-debug.test.ts": "usage", @@ -1796,7 +1796,8 @@ "zhipu-bigmodel-responses-quota.test.ts": "providers", "zz-ci-api-usage-isolation.test.ts": "ci-workflows", "zz-ci-storage-policy-isolation.test.ts": "ci-workflows", - "zz-pr-coderabbit-readiness-revalidation.test.ts": "ci-workflows" + "zz-pr-coderabbit-readiness-revalidation.test.ts": "ci-workflows", + "server-combo-cooldown-recording.test.ts": "server" }, "migrated": [ "adapters", diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 5e7ca9553f3..88343ae8dac 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -215,11 +215,11 @@ export function coolComboTarget( code?: string | null; message?: string; }, -): void { +): boolean { const now = options?.now ?? Date.now(); const writerGeneration = options?.writerGeneration ?? captureConfigGeneration(); const ownerKey = `${comboId}::${targetKey(target)}`; - if (writerGeneration < lastReconciledGeneration && !liveComboTargets.has(ownerKey)) return; + if (writerGeneration < lastReconciledGeneration && !liveComboTargets.has(ownerKey)) return false; // A server-provided Retry-After is authoritative, including an immediate `0` directive. // A quota reset is the next-most-specific signal (#3256); configured and default cooldowns // are only fallbacks when upstream supplied neither usable value. @@ -240,6 +240,7 @@ export function coolComboTarget( cooldownUntil: now + (serverDelayMs ?? Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS)), }); sweepExpiredOnWrite(now); + return true; } export function earliestComboCooldown( diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 8043644c5d1..67bc2579e0d 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -338,6 +338,7 @@ export function advanceComboAfterFailure( cooldownMs?: number; eligible?: (target: Required) => boolean; cooldownScope?: ComboFailureCooldownScope; + onCooldownRecorded?: (target: Pick) => void; status?: number; code?: string | null; message?: string; @@ -352,11 +353,12 @@ export function advanceComboAfterFailure( ? combo.targets.filter(target => target.provider === pick.target.provider) : [pick.target]; for (const target of cooldownTargets) { - coolComboTarget(pick.comboId, target, { + const recorded = coolComboTarget(pick.comboId, target, { ...options, cooldownMs: options.cooldownMs ?? combo?.cooldownMs, writerGeneration: pick.writerGeneration, }); + if (recorded) options.onCooldownRecorded?.(target); } } // #5691: under `cooldownWaitPolicy: "before-last-resort"` this final synchronous pick diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 168e1f6207a..b46957d39b8 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -757,6 +757,8 @@ export async function executeComboResponses( const failureCooldownScope = comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, }); + const failedTargetKey = targetKey(pick.target); + let failedTargetCooldownRecorded = false; const nextPick = advanceComboAfterFailure(config, pick, { retryAfter: failure.retryAfter, resetAt: failure.resetAt, @@ -767,12 +769,14 @@ export async function executeComboResponses( status: failure.response.status, code: failure.upstreamCode, message: failure.classificationText, + onCooldownRecorded: target => { + failedTargetCooldownRecorded ||= targetKey(target) === failedTargetKey; + }, }); - // Cooldown state is shared by every request using this target, so a concurrent failure - // can put it in cooldown while THIS failure recorded none. Scope "none" means the - // refusal described this request's shape rather than the target's health — the retry - // arms on any live cooldown for this target, whoever recorded it. + // A sibling cooldown is not this failure's write: stale-generation removal can + // refuse recording even for a cooldown-producing classification. Require both. const failedTargetCooled = failureCooldownScope !== "none" + && failedTargetCooldownRecorded && isComboTargetInCooldown(comboId, pick.target, failureNow); // Same target selector as the exclusionary pick below, minus `exclude`: the only // difference is deliberate and is the whole point of the single-target retry. diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index b16045ac125..f42b4e6f4c8 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -100,6 +100,13 @@ combo whose remaining eligible targets use other providers. > Decision record: [ADR-0070](../decisions/ADR-0070-same-provider-combo-quota-fallback.md) +## Single-target cooldown retry ownership + +`src/combos/failover.ts` reports whether the current failure actually records a cooldown. +`src/combos/resolve.ts` forwards that result for each target; `src/server/responses/core-combo.ts` +permits its bounded same-target retry only when this failure records the failed target and its +cooldown is live. A stale-generation refusal cannot borrow a sibling request's shared entry. + ## Combo per-target reasoning controls `src/server/responses/core.ts` passes the combo's `reasoningEffortMode` and the final target's diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 10b04d90c0c..8a4d689e029 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1627,5 +1627,6 @@ "mimo-token-plan-capacity.test.ts": "providers", "command-code-tool-text-prose-split.test.ts": "providers", "cli-effort-slug.test.ts": "cli", - "grok-47-build-fast-metadata.test.ts": "providers/xai" + "grok-47-build-fast-metadata.test.ts": "providers/xai", + "server-combo-cooldown-recording.test.ts": "server" } diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts new file mode 100644 index 00000000000..7004c3ac474 --- /dev/null +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -0,0 +1,99 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearComboSelectionState } from "../../src/combos/resolve"; +import { advanceComboAfterFailure, pickComboTarget } from "../../src/combos/resolve"; +import { clearComboTargetCooldowns, coolComboTarget, isComboTargetInCooldown, reconcileComboTargetCooldowns } from "../../src/combos/failover"; +import { captureConfigGeneration, type GenerationContext } from "../../src/lib/state-store-sweeper"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { chatSuccess } from "../helpers/combo-failover-upstream"; + +let home = ""; +let previous: string | undefined; +let codex: IsolatedCodexHome | undefined; +let release: (() => void) | undefined; +let upstream: ReturnType | undefined; +const target = { provider: "a", model: "m1" }; +function removalContext(): GenerationContext { + return { generation: captureConfigGeneration() + 1, providerNames: new Set(["a"]), + comboIds: new Set(["free"]), comboTargets: new Set(), codexAccountIds: new Set(), + oauthAccountKeys: new Set(), configRoots: new Set([home]) }; +} +function config(baseUrl = "http://127.0.0.1:1/v1"): OcxConfig { + return { port: 0, defaultProvider: "a", + providers: { a: { adapter: "openai-chat", baseUrl, apiKey: "synthetic-key" } }, + combos: { free: { strategy: "failover", targets: [target], cooldownMs: 100, waitForCooldownMs: 1000 } } }; +} +beforeEach(() => { + previous = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-combo-recording-")); + process.env.OPENCODEX_HOME = home; + codex = installIsolatedCodexHome("ocx-combo-recording-codex-"); + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearResponseStateForTests(); +}); +afterEach(async () => { + try { + await upstream?.stop(true); + upstream = undefined; + await flushResponseState(); + clearResponseStateForTests(); + release?.(); + release = undefined; + clearComboSelectionState(); + clearComboTargetCooldowns(); + } finally { + codex?.restore(); + codex = undefined; + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); +test("cooldown recording distinguishes a successful write from a stale removed writer", () => { + expect(coolComboTarget("free", target, { cooldownMs: 1000 })).toBe(true); + reconcileComboTargetCooldowns(removalContext()); + expect(isComboTargetInCooldown("free", target)).toBe(true); + expect(coolComboTarget("free", target, { cooldownMs: 1000 })).toBe(false); +}); +test("advance reports only the current failure's committed cooldown", () => { + const cfg = config(); + const pick = pickComboTarget(cfg, "free")!; + expect(pick).not.toBeNull(); + const recorded: string[] = []; + advanceComboAfterFailure(cfg, pick, { cooldownScope: "target", onCooldownRecorded: t => recorded.push(t.model) }); + expect(recorded).toEqual(["m1"]); + recorded.length = 0; + reconcileComboTargetCooldowns(removalContext()); + advanceComboAfterFailure(cfg, pick, { cooldownScope: "target", onCooldownRecorded: t => recorded.push(t.model) }); + expect(recorded).toEqual([]); + expect(isComboTargetInCooldown("free", target)).toBe(true); +}); +test("a stale in-flight single-target request does not replay a reconciled-away target", async () => { + let hits = 0; + upstream = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { + hits += 1; + if (hits > 1) return chatSuccess("unexpected replay", "m1"); + // The request already captured its generation. A sibling records cooldown, + // then management removes the target before this response reaches failover. + expect(coolComboTarget("free", target, { cooldownMs: 100 })).toBe(true); + reconcileComboTargetCooldowns(removalContext()); + return Response.json({ error: { message: "rate limited" } }, { status: 429 }); + } }); + release = acquireOwnedSpendHome(); + const cfg = config(`${upstream.url.toString().replace(/\/$/, "")}/v1`); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/free", input: "hello", stream: false }), + }), cfg, { model: "", provider: "" }); + await response.text(); + expect(response.status).toBe(429); + expect(hits).toBe(1); +}, 15000); From 69ccf5d9f1531eebf087699d47d50be493939fc7 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 25 Sep 2026 04:23:49 +0000 Subject: [PATCH 09/12] test(combos): explicitly admit the isolated loopback fixture --- tests/server/server-combo-cooldown-recording.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index 7004c3ac474..00b31d67e95 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -27,7 +27,7 @@ function removalContext(): GenerationContext { } function config(baseUrl = "http://127.0.0.1:1/v1"): OcxConfig { return { port: 0, defaultProvider: "a", - providers: { a: { adapter: "openai-chat", baseUrl, apiKey: "synthetic-key" } }, + providers: { a: { adapter: "openai-chat", baseUrl, apiKey: "synthetic-key", allowPrivateNetwork: true } }, combos: { free: { strategy: "failover", targets: [target], cooldownMs: 100, waitForCooldownMs: 1000 } } }; } beforeEach(() => { From b06f5892bde1571c6bff7d9a826527c7608c8e88 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 25 Sep 2026 20:44:45 +0900 Subject: [PATCH 10/12] test(combos): move single-target cooldown cases to the recording suite PR #5778 added two single-target cooldown/recording tests to server-combo-failover-e2e.test.ts, pushing it past its file-size cap. Move them byte-for-byte into the already-registered server-combo-cooldown-recording.test.ts and restore the cap to the origin/dev value (4166) without raising it. --- tests/fixtures/file-size-baseline.json | 2 +- .../server-combo-cooldown-recording.test.ts | 128 ++++++++++++++++++ .../server/server-combo-failover-e2e.test.ts | 58 -------- 3 files changed, 129 insertions(+), 59 deletions(-) diff --git a/tests/fixtures/file-size-baseline.json b/tests/fixtures/file-size-baseline.json index d13b2adec78..716b586605b 100644 --- a/tests/fixtures/file-size-baseline.json +++ b/tests/fixtures/file-size-baseline.json @@ -57,7 +57,7 @@ "tests/server/config.test.ts": 3828, "tests/server/management-provider-validation.test.ts": 5506, "tests/server/server-auth.test.ts": 4589, - "tests/server/server-combo-failover-e2e.test.ts": 4218, + "tests/server/server-combo-failover-e2e.test.ts": 4166, "tests/server/server-images.test.ts": 2755, "tests/server/server-live.test.ts": 2253, "tests/service/service.test.ts": 4106, diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index 00b31d67e95..bdd716a2543 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -14,6 +14,67 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; import { chatSuccess } from "../helpers/combo-failover-upstream"; +type HandleOptions = NonNullable[3]>; +const pendingTurns: Response[] = []; +function trackTurn(response: Response): Response { pendingTurns.push(response); return response; } +function takeSpendHome(): void { release ??= acquireOwnedSpendHome(); } +function provider(adapter: string, url: string, apiKey: string): OcxConfig["providers"][string] { + return { adapter, baseUrl: url, apiKey, allowPrivateNetwork: true }; +} +function serve(handler: (request: Request) => Response | Promise) { + upstream = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: handler }); + return upstream; +} +function baseUrl(server: ReturnType): string { + return `${server.url.toString().replace(/\/$/, "")}/v1`; +} +function comboConfig( + providers: OcxConfig["providers"], + targets = Object.keys(providers).map((name, index) => ({ provider: name, model: `m${index + 1}` })), + extra: Partial[string]> = {}, +): OcxConfig { + return { port: 0, defaultProvider: Object.keys(providers)[0]!, providers, combos: { free: { strategy: "failover", targets, ...extra } } }; +} +async function post( + config: OcxConfig, + raw: Record = {}, + options: HandleOptions = {}, + headers: Record = {}, +): Promise { + takeSpendHome(); + return trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), + }), config, { model: "", provider: "" }, options)); +} + +function serve(handler: (request: Request) => Response | Promise) { + upstream = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: handler }); + return upstream; +} +function baseUrl(server: ReturnType): string { + return `${server.url.toString().replace(/\/$/, "")}/v1`; +} +function comboConfig( + providers: OcxConfig["providers"], + targets = Object.keys(providers).map((name, index) => ({ provider: name, model: `m${index + 1}` })), + extra: Partial[string]> = {}, +): OcxConfig { + return { port: 0, defaultProvider: Object.keys(providers)[0]!, providers, combos: { free: { strategy: "failover", targets, ...extra } } }; +} +async function post( + config: OcxConfig, + raw: Record = {}, + options: HandleOptions = {}, + headers: Record = {}, +): Promise { + takeSpendHome(); + return trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), + }), config, { model: "", provider: "" }, options)); +} + let home = ""; let previous: string | undefined; let codex: IsolatedCodexHome | undefined; @@ -41,6 +102,10 @@ beforeEach(() => { }); afterEach(async () => { try { + for (const turn of pendingTurns.splice(0)) { + if (turn.bodyUsed || !turn.body || turn.body.locked) continue; + try { await turn.body.cancel(); } catch {} + } await upstream?.stop(true); upstream = undefined; await flushResponseState(); @@ -97,3 +162,66 @@ test("a stale in-flight single-target request does not replay a reconciled-away expect(response.status).toBe(429); expect(hits).toBe(1); }, 15000); + + + +// Moved from tests/server/server-combo-failover-e2e.test.ts to keep that file under its file-size cap. + test("single-target wait does not retry a request-local refusal", async () => { + let hits = 0; + const upstream = serve(() => { + hits += 1; + return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); + }); + const response = await post(comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ + { provider: "a", model: "m1" }, + ], { cooldownMs: 50, waitForCooldownMs: 500 }), { user: "synthetic-client" }); + expect(response.status).toBe(400); + expect(hits).toBe(1); + }); + + test("a concurrent cooldown does not retry a request-local refusal", async () => { + // Two requests share one target: the first stays in flight on a gate while the second + // fails hot and writes the SHARED cooldown. The first request's own failure records no + // cooldown (scope "none"), so the foreign entry alone must not arm the retry gate — + // it would wait out the sibling's cooldown and replay the refused request. + let markHeld!: () => void; + let releaseHeld!: () => void; + const heldRequest = new Promise(resolve => { markHeld = resolve; }); + const gate = new Promise(resolve => { releaseHeld = resolve; }); + let hits = 0; + const upstream = serve(async () => { + hits += 1; + if (hits === 1) { + markHeld(); + await gate; + return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); + } + // 429 rather than 5xx so the failure reaches the combo layer directly: + // fetchWithTransientRetry would absorb a 503 before it could cool the target. + return hits === 2 + ? Response.json({ error: { message: "rate limited" } }, { status: 429 }) + : chatSuccess("single target recovered", "m1"); + }); + const config = comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ + { provider: "a", model: "m1" }, + ], { cooldownMs: 500, waitForCooldownMs: 2_000 }); + + const refused = post(config, { user: "synthetic-client" }); + await heldRequest; + const cooling = post(config); + const target = { provider: "a", model: "m1" }; + const deadline = Date.now() + 5_000; + while (!isComboTargetInCooldown("free", target)) { + if (Date.now() > deadline) throw new Error("sibling request never cooled the target"); + await Bun.sleep(5); + } + releaseHeld(); + const [refusal, cooled] = await Promise.all([refused, cooling]); + expect(refusal.status).toBe(400); + expect(cooled.status).toBe(200); + // The cooling request hits twice (failure, then its own post-cooldown retry); the + // refused request must hit exactly once. + expect(hits).toBe(3); + }); + + diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index 3801b1ab2a3..1fd65f10304 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -2256,64 +2256,6 @@ describe("server combo failover 030 activation matrix", () => { expect(hits).toBe(1); }); - test("single-target wait does not retry a request-local refusal", async () => { - let hits = 0; - const upstream = serve(() => { - hits += 1; - return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); - }); - const response = await post(comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ - { provider: "a", model: "m1" }, - ], { cooldownMs: 50, waitForCooldownMs: 500 }), { user: "synthetic-client" }); - expect(response.status).toBe(400); - expect(hits).toBe(1); - }); - - test("a concurrent cooldown does not retry a request-local refusal", async () => { - // Two requests share one target: the first stays in flight on a gate while the second - // fails hot and writes the SHARED cooldown. The first request's own failure records no - // cooldown (scope "none"), so the foreign entry alone must not arm the retry gate — - // it would wait out the sibling's cooldown and replay the refused request. - let markHeld!: () => void; - let releaseHeld!: () => void; - const heldRequest = new Promise(resolve => { markHeld = resolve; }); - const gate = new Promise(resolve => { releaseHeld = resolve; }); - let hits = 0; - const upstream = serve(async () => { - hits += 1; - if (hits === 1) { - markHeld(); - await gate; - return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 }); - } - // 429 rather than 5xx so the failure reaches the combo layer directly: - // fetchWithTransientRetry would absorb a 503 before it could cool the target. - return hits === 2 - ? Response.json({ error: { message: "rate limited" } }, { status: 429 }) - : chatSuccess("single target recovered", "m1"); - }); - const config = comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [ - { provider: "a", model: "m1" }, - ], { cooldownMs: 500, waitForCooldownMs: 2_000 }); - - const refused = post(config, { user: "synthetic-client" }); - await heldRequest; - const cooling = post(config); - const target = { provider: "a", model: "m1" }; - const deadline = Date.now() + 5_000; - while (!isComboTargetInCooldown("free", target)) { - if (Date.now() > deadline) throw new Error("sibling request never cooled the target"); - await Bun.sleep(5); - } - releaseHeld(); - const [refusal, cooled] = await Promise.all([refused, cooling]); - expect(refusal.status).toBe(400); - expect(cooled.status).toBe(200); - // The cooling request hits twice (failure, then its own post-cooldown retry); the - // refused request must hit exactly once. - expect(hits).toBe(3); - }); - test("a past Retry-After date remains immediate through response consumption", async () => { const now = Date.parse("2026-07-18T00:00:00.000Z"); const failure = await consumeComboFailure(Response.json({ error: { message: "rate limited" } }, { From 09186a3417c20d0a15c51dbb70dce1aa6da9785e Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 25 Sep 2026 21:44:57 +0900 Subject: [PATCH 11/12] test(combos): explain the tolerated cancel failure in cleanup The moved helper carried an empty catch, which the PR hygiene gate rejects. The cleanup deliberately tolerates cancel on an already drained turn; say so. --- tests/server/server-combo-cooldown-recording.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index bdd716a2543..547cfe7c298 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -104,7 +104,7 @@ afterEach(async () => { try { for (const turn of pendingTurns.splice(0)) { if (turn.bodyUsed || !turn.body || turn.body.locked) continue; - try { await turn.body.cancel(); } catch {} + try { await turn.body.cancel(); } catch { /* a turn the test already drained may refuse cancel; cleanup continues */ } } await upstream?.stop(true); upstream = undefined; @@ -224,4 +224,3 @@ test("a stale in-flight single-target request does not replay a reconciled-away expect(hits).toBe(3); }); - From 2e7e4e53e54d68047bd56430528aeae9a10263ad Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 25 Sep 2026 22:17:26 +0900 Subject: [PATCH 12/12] test(combos): drop the duplicated helper block in the recording suite The move in b06f5892bd copied serve, baseUrl, comboConfig and post twice; the second, identical declarations silently replaced the first. Keep one copy. --- .../server-combo-cooldown-recording.test.ts | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index 547cfe7c298..64760a404e1 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -48,33 +48,6 @@ async function post( }), config, { model: "", provider: "" }, options)); } -function serve(handler: (request: Request) => Response | Promise) { - upstream = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: handler }); - return upstream; -} -function baseUrl(server: ReturnType): string { - return `${server.url.toString().replace(/\/$/, "")}/v1`; -} -function comboConfig( - providers: OcxConfig["providers"], - targets = Object.keys(providers).map((name, index) => ({ provider: name, model: `m${index + 1}` })), - extra: Partial[string]> = {}, -): OcxConfig { - return { port: 0, defaultProvider: Object.keys(providers)[0]!, providers, combos: { free: { strategy: "failover", targets, ...extra } } }; -} -async function post( - config: OcxConfig, - raw: Record = {}, - options: HandleOptions = {}, - headers: Record = {}, -): Promise { - takeSpendHome(); - return trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", headers: { "content-type": "application/json", ...headers }, - body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), - }), config, { model: "", provider: "" }, options)); -} - let home = ""; let previous: string | undefined; let codex: IsolatedCodexHome | undefined;