From bc45bd7f1d418c1ca89a3b849a3ffecf151df1c1 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 01:16:44 +0800 Subject: [PATCH 1/2] feat: add native Codex gateway borrowing and pelican comparisons --- .downstream/gateway-borrow-notices.md | 35 + .downstream/native-domains.md | 41 + .github/workflows/downstream-verify.yml | 8 + backend/cmd/server/wire.go | 15 + backend/cmd/server/wire_gen.go | 28 +- backend/cmd/server/wire_gen_test.go | 3 + .../admin/codex_gateway_borrow_handler.go | 270 ++++++ .../codex_gateway_borrow_handler_test.go | 141 +++ .../codex_gateway_borrow_failover_test.go | 27 + .../codex_gateway_borrow_ws_failure_test.go | 23 + backend/internal/handler/handler.go | 1 + .../internal/handler/openai_failover_retry.go | 5 + .../handler/openai_gateway_handler.go | 3 + backend/internal/handler/wire.go | 3 + .../codex_gateway_borrow_test_repo.go | 272 ++++++ ...teway_borrow_test_repo_integration_test.go | 214 +++++ .../codex_gateway_borrow_test_repo_test.go | 142 +++ .../codex_gateway_borrow_upstream.go | 171 ++++ .../codex_gateway_borrow_upstream_test.go | 380 ++++++++ backend/internal/repository/http_upstream.go | 31 +- backend/internal/repository/wire.go | 2 + backend/internal/server/routes/admin.go | 14 + .../internal/service/codex_gateway_borrow.go | 815 ++++++++++++++++++ .../service/codex_gateway_borrow_core_test.go | 476 ++++++++++ .../service/codex_gateway_borrow_http.go | 189 ++++ .../service/codex_gateway_borrow_http_test.go | 333 +++++++ .../service/codex_gateway_borrow_pelican.go | 123 +++ .../service/codex_gateway_borrow_preview.go | 239 +++++ .../codex_gateway_borrow_preview_test.go | 105 +++ .../service/codex_gateway_borrow_probe.go | 504 +++++++++++ .../codex_gateway_borrow_probe_stream_test.go | 160 ++++ .../codex_gateway_borrow_test_runner.go | 525 +++++++++++ .../codex_gateway_borrow_test_runner_test.go | 317 +++++++ .../codex_gateway_borrow_token_test.go | 65 ++ .../service/codex_gateway_borrow_ws_anchor.go | 356 ++++++++ ...odex_gateway_borrow_ws_passthrough_test.go | 104 +++ .../codex_gateway_borrow_ws_phase_test.go | 101 +++ .../service/codex_gateway_borrow_ws_test.go | 327 +++++++ .../internal/service/http_upstream_profile.go | 7 +- .../internal/service/openai_alpha_search.go | 4 +- .../openai_codex_request_compression.go | 11 +- .../openai_gateway_chat_completions.go | 2 +- .../service/openai_gateway_forward.go | 50 +- .../service/openai_gateway_messages.go | 2 +- .../service/openai_gateway_passthrough.go | 2 +- .../openai_gateway_response_handling.go | 4 +- .../service/openai_gateway_service.go | 1 + .../service/openai_images_responses.go | 2 +- .../service/openai_reasoning_recovery.go | 3 + .../internal/service/openai_token_provider.go | 4 +- .../openai_upstream_transport_error.go | 4 + .../service/openai_ws_forwarder_ingress.go | 51 ++ .../service/openai_ws_forwarder_v2.go | 44 +- .../internal/service/openai_ws_http_bridge.go | 4 +- backend/internal/service/openai_ws_pool.go | 42 +- .../openai_ws_v2_passthrough_adapter.go | 32 + backend/internal/service/wire.go | 3 + .../275_codex_gateway_borrow_tests.sql | 49 ++ .../__tests__/codexGatewayBorrow.spec.ts | 76 ++ frontend/src/api/admin/codexGatewayBorrow.ts | 218 +++++ .../admin/codex/BorrowPelicanPreview.vue | 70 ++ .../__tests__/BorrowPelicanPreview.spec.ts | 44 + frontend/src/components/layout/AppSidebar.vue | 5 + .../locales/en/admin/codexGatewayBorrow.ts | 99 +++ frontend/src/i18n/locales/en/admin/index.ts | 2 + frontend/src/i18n/locales/en/common.ts | 1 + .../locales/zh/admin/codexGatewayBorrow.ts | 99 +++ frontend/src/i18n/locales/zh/admin/index.ts | 2 + frontend/src/i18n/locales/zh/common.ts | 1 + frontend/src/router/index.ts | 6 + .../views/admin/CodexGatewayBorrowView.vue | 458 ++++++++++ .../__tests__/CodexGatewayBorrowView.spec.ts | 183 ++++ 72 files changed, 8110 insertions(+), 43 deletions(-) create mode 100644 .downstream/gateway-borrow-notices.md create mode 100644 backend/internal/handler/admin/codex_gateway_borrow_handler.go create mode 100644 backend/internal/handler/admin/codex_gateway_borrow_handler_test.go create mode 100644 backend/internal/handler/codex_gateway_borrow_failover_test.go create mode 100644 backend/internal/handler/codex_gateway_borrow_ws_failure_test.go create mode 100644 backend/internal/repository/codex_gateway_borrow_test_repo.go create mode 100644 backend/internal/repository/codex_gateway_borrow_test_repo_integration_test.go create mode 100644 backend/internal/repository/codex_gateway_borrow_test_repo_test.go create mode 100644 backend/internal/repository/codex_gateway_borrow_upstream.go create mode 100644 backend/internal/repository/codex_gateway_borrow_upstream_test.go create mode 100644 backend/internal/service/codex_gateway_borrow.go create mode 100644 backend/internal/service/codex_gateway_borrow_core_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_http.go create mode 100644 backend/internal/service/codex_gateway_borrow_http_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_pelican.go create mode 100644 backend/internal/service/codex_gateway_borrow_preview.go create mode 100644 backend/internal/service/codex_gateway_borrow_preview_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_probe.go create mode 100644 backend/internal/service/codex_gateway_borrow_probe_stream_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_test_runner.go create mode 100644 backend/internal/service/codex_gateway_borrow_test_runner_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_token_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_ws_anchor.go create mode 100644 backend/internal/service/codex_gateway_borrow_ws_passthrough_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_ws_phase_test.go create mode 100644 backend/internal/service/codex_gateway_borrow_ws_test.go create mode 100644 backend/migrations/275_codex_gateway_borrow_tests.sql create mode 100644 frontend/src/api/admin/__tests__/codexGatewayBorrow.spec.ts create mode 100644 frontend/src/api/admin/codexGatewayBorrow.ts create mode 100644 frontend/src/components/admin/codex/BorrowPelicanPreview.vue create mode 100644 frontend/src/components/admin/codex/__tests__/BorrowPelicanPreview.spec.ts create mode 100644 frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts create mode 100644 frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts create mode 100644 frontend/src/views/admin/CodexGatewayBorrowView.vue create mode 100644 frontend/src/views/admin/__tests__/CodexGatewayBorrowView.spec.ts diff --git a/.downstream/gateway-borrow-notices.md b/.downstream/gateway-borrow-notices.md new file mode 100644 index 000000000000..a67a53d140bb --- /dev/null +++ b/.downstream/gateway-borrow-notices.md @@ -0,0 +1,35 @@ +# Gateway borrowing implementation sources + +The native Cookie lease, two-shot STATE qualification and WebSocket anchor +behavior were adapted from [ranxi2001/sub2api](https://github.com/ranxi2001/sub2api), +commit `5ca3cca21eeaf4ca8a694a7f2f8f0ecd9575c549`, under LGPL-3.0. The source +files are `codex_gateway_pin.go`, `astra_routing_upstream.go`, +`astra_target_validation.go`, `openai_codex_state_probe.go`, +`openai_codex_ws_anchor.go`, `pelicanHtml.ts` and the pelican preview components. +The repository [LICENSE](../LICENSE) retains the LGPL-3.0 terms. + +HTML fence selection also references `web/src/lib/render.ts` from +[iwyxdxl/gpttesticu](https://github.com/iwyxdxl/gpttesticu), commit +`1b3e237c15dcdf89c4aa66d3521f8d6485d6da4a`. Its MIT notice follows. + +MIT License + +Copyright (c) 2026 iwyxdxl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.downstream/native-domains.md b/.downstream/native-domains.md index 6df368a4be80..db11d2a7f125 100644 --- a/.downstream/native-domains.md +++ b/.downstream/native-domains.md @@ -17,6 +17,47 @@ other key or a non-boolean value (including `null`), returns HTTP 400 and stores nothing. While no Codex runtime row is stored, `gateway.openai_codex_request_zstd` decides the compression switch. +## Codex gateway borrowing + +`GET/PUT /admin/codex-gateway-borrow/config` owns one JSON setting, +`codex_gateway_borrow_config`: `enabled`, `source_account_ids`, +`target_account_ids` and `models`. A missing row is off with empty account +lists and the two models `gpt-6-astra` and `gpt-6.1-sol`. Only existing OpenAI +OAuth-like accounts can be selected; the source and target lists are disjoint. +The extension never rewrites account model mappings, proxies, WS switches, +groups, quota or scheduling state. Normal credential refresh remains available. + +Source acquisition shares one process-local `__oailb` candidate. Its maximum +lease is 230 seconds, capped by an earlier upstream expiry; receiving the same +value during a live lease does not extend it. Source preparation is sequential +and has a 90-second budget. Each target/model is validated with two completed +HTTP 200 streams, at most 45 seconds each: the first must return STATE, and +the second may omit STATE or return the same value. Qualification includes +the target identity, actual model, exit, client headers, STATE and TLS profile. +Source and target probes use separate HTTP/TLS pools. Business WS connections +stay in the ordinary account pool, with fixed one-hour continuation anchors. + +Status/config/history reads do not call models. Saving an enabled config starts +one finite preparation; business cache misses prepare synchronously. There is +no renewal timer. `/prepare` and `/verify` explicitly prepare or revalidate. +All probes only record observations, including failures before business dispatch. + +`POST /admin/codex-gateway-borrow/tests` accepts a timestamped UUIDv7 and explicit +account/model pairs. It sends the fixed pelican prompt once per pair, only with +matching unexpired qualification, using at most three shared generation slots +and serializing each account. It does not prepare missing routes. An expired +or future UUID is rejected; replaying a saved UUID cannot dispatch again. +Migration 275 adds only this feature's task/result tables. Records expire 24 +hours after task creation, disappear from reads immediately, and are removed +by scoped minute cleanup. Output/errors retain their original bytes. + +Result previews use short random capability URLs under +`/codex-gateway-borrow/preview/`, an opaque script-enabled sandbox and their own +response CSP. Inline animation and external scripts/styles/fonts/images are +allowed. Preview documents receive no administrator token or UI bridge. +Adaptation sources and retained notices are in +[gateway borrowing notices](gateway-borrow-notices.md). + ## Startup and stored data Migrations run first. Migration 264 stops startup, changing nothing, on a diff --git a/.github/workflows/downstream-verify.yml b/.github/workflows/downstream-verify.yml index e1105f24b1b5..a6f718262d8d 100644 --- a/.github/workflows/downstream-verify.yml +++ b/.github/workflows/downstream-verify.yml @@ -41,6 +41,11 @@ jobs: - name: Service tests working-directory: backend run: go test -p=1 -tags unit ./internal/service -run 'Test.*(Refusal|CyberPolicy|Continuation|InvalidEncryptedContent|PreviousResponseNotFound|CapacityShed|RequestScopedTransient|ServerOverloaded|AlphaSearch|SelectedWorkspace)|Test(UpdateService|NormalizeAccountTaxonomyName|AccountTaxonomyRules|AccountTestReasoningValidated|FilterConsoleAccounts|AccountConsolePlan|AccountFacetMatcher)' -count=1 + - name: Native gateway borrowing contracts + working-directory: backend + run: | + go test -p=1 -tags unit ./internal/service ./internal/repository ./internal/handler ./internal/handler/admin -run '^TestCodexGatewayBorrow' -count=1 + go test -p=1 -tags integration ./internal/repository -run '^TestCodexGatewayBorrowTestRepositoryIntegrationLifecycle$' -count=1 - name: Upstream automation classifier tests run: | python3 .github/scripts/test_upstream_risk.py @@ -135,6 +140,9 @@ jobs: src/components/payment/__tests__/StripePaymentInline.contrast.spec.ts \ src/views/user/__tests__/StripePaymentView.spec.ts \ src/views/admin/__tests__/ReasoningRecoveryView.spec.ts \ + src/api/admin/__tests__/codexGatewayBorrow.spec.ts \ + src/components/admin/codex/__tests__/BorrowPelicanPreview.spec.ts \ + src/views/admin/__tests__/CodexGatewayBorrowView.spec.ts \ src/components/account/__tests__/EditAccountModal.spec.ts \ src/utils/__tests__/openaiProtocolModeEdits.spec.ts \ src/components/account/__tests__/ModelWhitelistSelector.spec.ts \ diff --git a/backend/cmd/server/wire.go b/backend/cmd/server/wire.go index 5128eb110bf1..691e869b3761 100644 --- a/backend/cmd/server/wire.go +++ b/backend/cmd/server/wire.go @@ -130,6 +130,9 @@ func provideCleanup( antigravityOAuth *service.AntigravityOAuthService, grokOAuth *service.GrokOAuthService, openAIGateway *service.OpenAIGatewayService, + codexGatewayBorrow *service.CodexGatewayBorrowService, + codexGatewayBorrowTests *service.CodexGatewayBorrowTestRunner, + codexGatewayBorrowPreview *service.CodexGatewayBorrowPreviewService, scheduledTestRunner *service.ScheduledTestRunnerService, upstreamModelCatalogRefresh *service.UpstreamModelCatalogRefreshService, backupSvc *service.BackupService, @@ -157,6 +160,18 @@ func provideCleanup( // 应用层清理步骤可并行执行，基础设施资源（Redis/Ent）最后按顺序关闭。 parallelSteps := []cleanupStep{ + {"CodexGatewayBorrow", func() error { + if codexGatewayBorrow != nil { + codexGatewayBorrow.Stop() + } + if codexGatewayBorrowTests != nil { + codexGatewayBorrowTests.Stop() + } + if codexGatewayBorrowPreview != nil { + codexGatewayBorrowPreview.Stop() + } + return nil + }}, {"PluginManager", func() error { if pluginManager != nil { pluginManager.Stop() diff --git a/backend/cmd/server/wire_gen.go b/backend/cmd/server/wire_gen.go index e5182729a420..bfec3a57bc70 100644 --- a/backend/cmd/server/wire_gen.go +++ b/backend/cmd/server/wire_gen.go @@ -304,11 +304,20 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { auditLogHandler := admin.NewAuditLogHandler(auditLogService, totpService) systemPromptHandler := admin.NewSystemPromptHandler(systemPromptService) reasoningRecoveryHandler := admin.NewReasoningRecoveryHandler(reasoningRecoveryService) + codexGatewayBorrowProbeUpstream := repository.NewCodexGatewayBorrowProbeUpstream(configConfig) + codexGatewayBorrowService, err := service.ProvideCodexGatewayBorrowService(settingRepository, accountRepository, openAIGatewayService, codexGatewayBorrowProbeUpstream, tlsFingerprintProfileService) + if err != nil { + return nil, err + } + codexGatewayBorrowTestRepository := repository.NewCodexGatewayBorrowTestRepository(db) + codexGatewayBorrowTestRunner := service.NewCodexGatewayBorrowTestRunner(codexGatewayBorrowTestRepository, codexGatewayBorrowService, accountRepository) + codexGatewayBorrowPreviewService := service.NewCodexGatewayBorrowPreviewService(codexGatewayBorrowTestRepository) + codexGatewayBorrowHandler := admin.NewCodexGatewayBorrowHandler(codexGatewayBorrowService, codexGatewayBorrowTestRunner, codexGatewayBorrowPreviewService) upstreamBillingProbeService := service.ProvideUpstreamBillingProbeService(accountRepository, accountTestService, settingService, leaderLockCache, db) openCodeGoUsageService := service.ProvideOpenCodeGoUsageService(accountRepository, httpUpstream, settingService, leaderLockCache, db) idempotencyCoordinator := service.ProvideIdempotencyCoordinator(idempotencyRepository, configConfig) claudeResetCreditService := service.ProvideClaudeResetCreditService(accountRepository, claudeTokenProvider, proxyRepository, settingService, idempotencyCoordinator, leaderLockCache) - adminHandlers := handler.ProvideAdminHandlers(dashboardHandler, adminUserHandler, groupHandler, accountHandler, accountJobHandler, adminAnnouncementHandler, dataManagementHandler, backupHandler, oAuthHandler, openAIOAuthHandler, geminiOAuthHandler, antigravityOAuthHandler, grokOAuthHandler, cnProviderHandler, proxyHandler, adminRedeemHandler, promoHandler, settingHandler, opsHandler, systemHandler, adminSubscriptionHandler, adminUsageHandler, userAttributeHandler, errorPassthroughHandler, tlsFingerprintProfileHandler, pluginHandler, adminAPIKeyHandler, scheduledTestHandler, channelHandler, channelMonitorHandler, channelMonitorRequestTemplateHandler, contentModerationHandler, promptAdminHandler, paymentHandler, affiliateHandler, complianceHandler, auditLogHandler, systemPromptHandler, reasoningRecoveryHandler, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, claudeResetCreditService) + adminHandlers := handler.ProvideAdminHandlers(dashboardHandler, adminUserHandler, groupHandler, accountHandler, accountJobHandler, adminAnnouncementHandler, dataManagementHandler, backupHandler, oAuthHandler, openAIOAuthHandler, geminiOAuthHandler, antigravityOAuthHandler, grokOAuthHandler, cnProviderHandler, proxyHandler, adminRedeemHandler, promoHandler, settingHandler, opsHandler, systemHandler, adminSubscriptionHandler, adminUsageHandler, userAttributeHandler, errorPassthroughHandler, tlsFingerprintProfileHandler, pluginHandler, adminAPIKeyHandler, scheduledTestHandler, channelHandler, channelMonitorHandler, channelMonitorRequestTemplateHandler, contentModerationHandler, promptAdminHandler, paymentHandler, affiliateHandler, complianceHandler, auditLogHandler, systemPromptHandler, reasoningRecoveryHandler, codexGatewayBorrowHandler, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, claudeResetCreditService) usageRecordWorkerPool := service.NewUsageRecordWorkerPool(configConfig) userMsgQueueCache := repository.NewUserMsgQueueCache(redisClient) userMessageQueueService := service.ProvideUserMessageQueueService(userMsgQueueCache, rpmCache, configConfig) @@ -374,7 +383,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { channelMonitorV2Aggregator := service.ProvideChannelMonitorV2Aggregator(channelMonitorV2Repository, db, settingService) userPlatformQuotaUsageFlusher := service.ProvideUserPlatformQuotaUsageFlusher(configConfig, billingCache, serviceUserPlatformQuotaRepository, timingWheelService) openAIQuotaAutoResetService := service.ProvideOpenAIQuotaAutoResetService(accountRepository, openAIQuotaService, rateLimitService, idempotencyCoordinator, auditLogService, settingService, leaderLockCache) - v := provideCleanup(client, redisClient, gatewayCache, opsMetricsCollector, opsAggregationService, opsAlertEvaluatorService, opsCleanupService, opsScheduledReportService, opsSystemLogSink, opsService, opsIngressRejectAggregator, apiKeyService, authCacheInvalidationWorker, schedulerSnapshotService, tokenRefreshService, accountExpiryService, cnProviderBalanceCheckService, openAICodexVersionSyncService, codexClientIdentityBackfillService, claudeCodeVersionSyncService, proxyExpiryService, subscriptionExpiryService, usageCleanupService, idempotencyCleanupService, batchImageCleanupService, batchImageWorkerRuntime, pricingService, emailQueueService, billingCacheService, usageRecordWorkerPool, subscriptionService, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, grokOAuthService, openAIGatewayService, scheduledTestRunnerService, upstreamModelCatalogRefreshService, backupService, paymentOrderExpiryService, channelMonitorRunner, channelMonitorV2Aggregator, userPlatformQuotaUsageFlusher, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, auditLogService, openAIQuotaAutoResetService, promptService, accountJobRuntime, pluginManager) + v := provideCleanup(client, redisClient, gatewayCache, opsMetricsCollector, opsAggregationService, opsAlertEvaluatorService, opsCleanupService, opsScheduledReportService, opsSystemLogSink, opsService, opsIngressRejectAggregator, apiKeyService, authCacheInvalidationWorker, schedulerSnapshotService, tokenRefreshService, accountExpiryService, cnProviderBalanceCheckService, openAICodexVersionSyncService, codexClientIdentityBackfillService, claudeCodeVersionSyncService, proxyExpiryService, subscriptionExpiryService, usageCleanupService, idempotencyCleanupService, batchImageCleanupService, batchImageWorkerRuntime, pricingService, emailQueueService, billingCacheService, usageRecordWorkerPool, subscriptionService, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, grokOAuthService, openAIGatewayService, codexGatewayBorrowService, codexGatewayBorrowTestRunner, codexGatewayBorrowPreviewService, scheduledTestRunnerService, upstreamModelCatalogRefreshService, backupService, paymentOrderExpiryService, channelMonitorRunner, channelMonitorV2Aggregator, userPlatformQuotaUsageFlusher, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, auditLogService, openAIQuotaAutoResetService, promptService, accountJobRuntime, pluginManager) application := &Application{ Server: httpServer, PromptAudit: promptService, @@ -459,6 +468,9 @@ func provideCleanup( antigravityOAuth *service.AntigravityOAuthService, grokOAuth *service.GrokOAuthService, openAIGateway *service.OpenAIGatewayService, + codexGatewayBorrow *service.CodexGatewayBorrowService, + codexGatewayBorrowTests *service.CodexGatewayBorrowTestRunner, + codexGatewayBorrowPreview *service.CodexGatewayBorrowPreviewService, scheduledTestRunner *service.ScheduledTestRunnerService, upstreamModelCatalogRefresh *service.UpstreamModelCatalogRefreshService, backupSvc *service.BackupService, @@ -485,6 +497,18 @@ func provideCleanup( } parallelSteps := []cleanupStep{ + {"CodexGatewayBorrow", func() error { + if codexGatewayBorrow != nil { + codexGatewayBorrow.Stop() + } + if codexGatewayBorrowTests != nil { + codexGatewayBorrowTests.Stop() + } + if codexGatewayBorrowPreview != nil { + codexGatewayBorrowPreview.Stop() + } + return nil + }}, {"PluginManager", func() error { if pluginManager != nil { pluginManager.Stop() diff --git a/backend/cmd/server/wire_gen_test.go b/backend/cmd/server/wire_gen_test.go index 89253e8099c6..b7ccb7f5b1d7 100644 --- a/backend/cmd/server/wire_gen_test.go +++ b/backend/cmd/server/wire_gen_test.go @@ -97,6 +97,9 @@ func minimalDependencyCleanup(gatewayCache service.GatewayCache, autoReset *serv antigravityOAuthSvc, nil, // grokOAuth nil, // openAIGateway + nil, // codexGatewayBorrow + nil, // codexGatewayBorrowTests + nil, // codexGatewayBorrowPreview nil, // scheduledTestRunner nil, // upstreamModelCatalogRefresh nil, // backupSvc diff --git a/backend/internal/handler/admin/codex_gateway_borrow_handler.go b/backend/internal/handler/admin/codex_gateway_borrow_handler.go new file mode 100644 index 000000000000..5f5868fd1d26 --- /dev/null +++ b/backend/internal/handler/admin/codex_gateway_borrow_handler.go @@ -0,0 +1,270 @@ +package admin + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/response" + "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type CodexGatewayBorrowHandler struct { + core *service.CodexGatewayBorrowService + runner *service.CodexGatewayBorrowTestRunner + preview *service.CodexGatewayBorrowPreviewService +} + +func NewCodexGatewayBorrowHandler(core *service.CodexGatewayBorrowService, runner *service.CodexGatewayBorrowTestRunner, preview *service.CodexGatewayBorrowPreviewService) *CodexGatewayBorrowHandler { + return &CodexGatewayBorrowHandler{core: core, runner: runner, preview: preview} +} + +func codexGatewayBorrowPrivateHeaders(c *gin.Context) { + c.Header("Cache-Control", "private, no-store") + c.Header("Referrer-Policy", "no-referrer") + c.Header("X-Content-Type-Options", "nosniff") +} + +func (h *CodexGatewayBorrowHandler) GetConfig(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + config, err := h.core.GetConfig(c.Request.Context()) + if err != nil { + response.Error(c, http.StatusInternalServerError, err.Error()) + return + } + response.Success(c, config) +} + +func (h *CodexGatewayBorrowHandler) SaveConfig(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + var request service.CodexGatewayBorrowConfig + raw, err := c.GetRawData() + if err == nil { + request, err = service.DecodeCodexGatewayBorrowConfig(raw) + } + if err != nil { + response.BadRequest(c, err.Error()) + return + } + config, err := h.core.SaveConfig(c.Request.Context(), request) + if err != nil { + response.BadRequest(c, err.Error()) + return + } + middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_config_saved", "enabled": config.Enabled}) + response.Success(c, config) +} + +func (h *CodexGatewayBorrowHandler) Status(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + response.Success(c, h.core.Status()) +} + +func (h *CodexGatewayBorrowHandler) Prepare(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + status, err := h.core.Prepare(c.Request.Context()) + if err != nil { + response.BadRequest(c, err.Error()) + return + } + response.Success(c, status) +} + +func (h *CodexGatewayBorrowHandler) Verify(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + var request struct { + AccountID int64 `json:"account_id"` + Model string `json:"model"` + } + if err := c.ShouldBindJSON(&request); err != nil { + response.BadRequest(c, err.Error()) + return + } + if request.AccountID <= 0 || strings.TrimSpace(request.Model) == "" { + response.BadRequest(c, "account_id and model are required") + return + } + result, err := h.core.Verify(c.Request.Context(), request.AccountID, strings.TrimSpace(request.Model)) + if err != nil { + response.BadRequest(c, err.Error()) + return + } + response.Success(c, result) +} + +func codexGatewayBorrowTestError(c *gin.Context, err error) { + status := http.StatusInternalServerError + switch { + case errors.Is(err, service.ErrCodexGatewayBorrowTestReplayConflict): + status = http.StatusConflict + case errors.Is(err, service.ErrCodexGatewayBorrowTestInvalidRequest): + status = http.StatusBadRequest + case errors.Is(err, service.ErrCodexGatewayBorrowTestExpired): + status = http.StatusGone + case errors.Is(err, service.ErrCodexGatewayBorrowTestNotFound): + status = http.StatusNotFound + } + response.Error(c, status, err.Error()) +} + +func (h *CodexGatewayBorrowHandler) StartTests(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + var request service.CodexGatewayBorrowTestRequest + if err := c.ShouldBindJSON(&request); err != nil { + response.BadRequest(c, err.Error()) + return + } + if _, err := uuid.Parse(strings.TrimSpace(request.ClientTaskID)); err != nil { + response.BadRequest(c, "client_task_id must be a UUID") + return + } + if len(request.Targets) == 0 { + response.BadRequest(c, "targets is required") + return + } + var createdBy int64 + if subject, ok := middleware.GetAuthSubjectFromContext(c); ok { + createdBy = subject.UserID + } + task, replayed, err := h.runner.Create(c.Request.Context(), createdBy, request) + if err != nil { + codexGatewayBorrowTestError(c, err) + return + } + middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_manual_test", "task_id": task.ID, "replayed": replayed, "targets": task.Total}) + ctx, cancel := context.WithCancel(c.Request.Context()) + defer cancel() + c.Header("Content-Type", "text/event-stream") + c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") + c.Status(http.StatusOK) + var writeMu sync.Mutex + emit := func(event service.CodexGatewayBorrowTestEvent) { + if ctx.Err() != nil { + return + } + if event.Task != nil { + event.Task = h.preview.DecorateTask(event.Task) + } + if event.Result != nil { + event.Result = h.preview.DecorateResult(event.Result) + } + payload, marshalErr := json.Marshal(event) + if marshalErr != nil { + cancel() + return + } + writeMu.Lock() + defer writeMu.Unlock() + if ctx.Err() != nil { + return + } + if _, writeErr := fmt.Fprintf(c.Writer, "data: %s\n\n", payload); writeErr != nil { + cancel() + return + } + c.Writer.Flush() + } + emit(service.CodexGatewayBorrowTestEvent{Type: "task_start", Task: task}) + if replayed { + emit(service.CodexGatewayBorrowTestEvent{Type: "task_complete", Task: task}) + return + } + keepAliveDone, keepAliveStopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(keepAliveStopped) + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + select { + case <-keepAliveDone: + return + case <-ctx.Done(): + return + case <-ticker.C: + writeMu.Lock() + if _, writeErr := fmt.Fprint(c.Writer, ": keep-alive\n\n"); writeErr != nil { + cancel() + } else { + c.Writer.Flush() + } + writeMu.Unlock() + } + } + }() + runErr := h.runner.Run(ctx, task, emit) + close(keepAliveDone) + <-keepAliveStopped + if runErr != nil && ctx.Err() == nil { + // An unexpected persistence failure remains fully visible on the live + // stream even if the database cannot save that final failure observation. + task.Status, task.Error = "incomplete", runErr.Error() + emit(service.CodexGatewayBorrowTestEvent{Type: "task_complete", Task: task}) + } +} + +func (h *CodexGatewayBorrowHandler) ListTests(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + page, size := response.ParsePagination(c) + if rawSize := c.Query("size"); rawSize != "" { + if parsed, err := strconv.Atoi(rawSize); err == nil && parsed > 0 { + size = parsed + } + } + list, err := h.runner.List(c.Request.Context(), page, size) + if err != nil { + codexGatewayBorrowTestError(c, err) + return + } + response.Success(c, list) +} + +func (h *CodexGatewayBorrowHandler) GetTest(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + id := strings.TrimSpace(c.Param("id")) + if id == "last" { + list, err := h.runner.List(c.Request.Context(), 1, 1) + if err != nil { + codexGatewayBorrowTestError(c, err) + return + } + if len(list.Items) == 0 { + response.NotFound(c, "no unexpired pelican test") + return + } + id = list.Items[0].ID + } + parsed, err := uuid.Parse(id) + if err != nil { + response.BadRequest(c, "test ID must be a UUID or last") + return + } + task, err := h.runner.Get(c.Request.Context(), parsed.String()) + if err != nil { + codexGatewayBorrowTestError(c, err) + return + } + response.Success(c, h.preview.DecorateTask(task)) +} + +func (h *CodexGatewayBorrowHandler) ServePreview(c *gin.Context) { + codexGatewayBorrowPrivateHeaders(c) + c.Header("Content-Security-Policy", service.CodexGatewayBorrowPreviewCSP) + c.Header("X-Frame-Options", "SAMEORIGIN") + c.Header("Cross-Origin-Resource-Policy", "cross-origin") + html, err := h.preview.Resolve(c.Request.Context(), strings.TrimSpace(c.Param("cap"))) + if err != nil { + c.Status(http.StatusGone) + return + } + c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(html)) +} diff --git a/backend/internal/handler/admin/codex_gateway_borrow_handler_test.go b/backend/internal/handler/admin/codex_gateway_borrow_handler_test.go new file mode 100644 index 000000000000..7d1380a29527 --- /dev/null +++ b/backend/internal/handler/admin/codex_gateway_borrow_handler_test.go @@ -0,0 +1,141 @@ +//go:build unit + +package admin + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +type codexGatewayBorrowHandlerTestRepo struct { + task *service.CodexGatewayBorrowTestTask + result *service.CodexGatewayBorrowTestResult + creates int + writes int + reads int +} + +func (r *codexGatewayBorrowHandlerTestRepo) Create(context.Context, *service.CodexGatewayBorrowTestTask) (*service.CodexGatewayBorrowTestTask, bool, error) { + r.creates++ + return r.task, true, nil +} +func (r *codexGatewayBorrowHandlerTestRepo) Get(context.Context, string) (*service.CodexGatewayBorrowTestTask, error) { + r.reads++ + return r.task, nil +} +func (r *codexGatewayBorrowHandlerTestRepo) GetResult(context.Context, string) (*service.CodexGatewayBorrowTestResult, error) { + r.reads++ + return r.result, nil +} +func (r *codexGatewayBorrowHandlerTestRepo) List(context.Context, int, int) (*service.CodexGatewayBorrowTestList, error) { + return &service.CodexGatewayBorrowTestList{Items: []*service.CodexGatewayBorrowTestTask{r.task}}, nil +} +func (r *codexGatewayBorrowHandlerTestRepo) StartTask(context.Context, string, time.Time) error { + r.writes++ + return nil +} +func (r *codexGatewayBorrowHandlerTestRepo) SaveResult(context.Context, *service.CodexGatewayBorrowTestResult) error { + r.writes++ + return nil +} +func (r *codexGatewayBorrowHandlerTestRepo) FinishTask(context.Context, string, string, string, time.Time) error { + r.writes++ + return nil +} +func (r *codexGatewayBorrowHandlerTestRepo) MarkInterrupted(context.Context, time.Time) error { + return nil +} +func (r *codexGatewayBorrowHandlerTestRepo) CleanupExpired(context.Context, time.Time) error { + return nil +} + +func newCodexGatewayBorrowHandlerFixture(t *testing.T) (*CodexGatewayBorrowHandler, *codexGatewayBorrowHandlerTestRepo) { + t.Helper() + now := time.Now().UTC() + client, err := uuid.NewV7() + require.NoError(t, err) + result := &service.CodexGatewayBorrowTestResult{ID: uuid.NewString(), Status: "complete", AccountID: 42, ModelID: "gpt-6-astra", Effort: "high", RawAnswer: "", RawHTML: "", HTML: "", Error: "original upstream error token=admin-visible\x00tail", ExpiresAt: now.Add(time.Hour)} + task := &service.CodexGatewayBorrowTestTask{ID: uuid.NewString(), ClientTaskID: client.String(), Status: "complete", Total: 1, Completed: 1, CreatedAt: now, ExpiresAt: result.ExpiresAt, Results: []*service.CodexGatewayBorrowTestResult{result}} + result.TaskID = task.ID + repo := &codexGatewayBorrowHandlerTestRepo{task: task, result: result} + runner := service.NewCodexGatewayBorrowTestRunner(repo, nil, nil) + preview := service.NewCodexGatewayBorrowPreviewService(repo) + t.Cleanup(runner.Stop) + t.Cleanup(preview.Stop) + return NewCodexGatewayBorrowHandler(nil, runner, preview), repo +} + +func TestCodexGatewayBorrowHandlerPreviewUsesOnlyCapabilityAndOverridesCSP(t *testing.T) { + gin.SetMode(gin.TestMode) + h, repo := newCodexGatewayBorrowHandlerFixture(t) + decorated := h.preview.DecorateResult(repo.result) + r := gin.New() + r.Use(middleware.SecurityHeaders(config.CSPConfig{Enabled: true}, nil)) + r.GET("/api/v1/codex-gateway-borrow/preview/:cap/index.html", h.ServePreview) + r.GET("/", func(c *gin.Context) { c.String(http.StatusOK, "main app") }) + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, decorated.PreviewURL, nil) + request.Header.Set("Authorization", "Bearer SHOULD-NOT-APPEAR") + request.Header.Set("Cookie", "admin_jwt=SHOULD-NOT-APPEAR") + r.ServeHTTP(w, request) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, repo.result.HTML, w.Body.String()) + require.Equal(t, service.CodexGatewayBorrowPreviewCSP, w.Header().Get("Content-Security-Policy")) + require.Equal(t, "private, no-store", w.Header().Get("Cache-Control")) + require.Equal(t, "no-referrer", w.Header().Get("Referrer-Policy")) + require.Equal(t, "nosniff", w.Header().Get("X-Content-Type-Options")) + require.Equal(t, "SAMEORIGIN", w.Header().Get("X-Frame-Options")) + require.NotContains(t, w.Body.String(), "SHOULD-NOT-APPEAR") + require.NotContains(t, w.Body.String(), "bridge_token") + main := httptest.NewRecorder() + r.ServeHTTP(main, httptest.NewRequest(http.MethodGet, "/", nil)) + require.Contains(t, main.Header().Get("Content-Security-Policy"), "'nonce-") + require.NotEqual(t, service.CodexGatewayBorrowPreviewCSP, main.Header().Get("Content-Security-Policy")) + reads := repo.reads + unknown := httptest.NewRecorder() + r.ServeHTTP(unknown, httptest.NewRequest(http.MethodGet, "/api/v1/codex-gateway-borrow/preview/"+strings.Repeat("x", 43)+"/index.html", nil)) + require.Equal(t, http.StatusGone, unknown.Code) + require.Equal(t, reads, repo.reads) +} + +func TestCodexGatewayBorrowHandlerReadAndSSEReplayPreserveFullErrorsWithoutDispatch(t *testing.T) { + gin.SetMode(gin.TestMode) + h, repo := newCodexGatewayBorrowHandlerFixture(t) + r := gin.New() + r.GET("/tests/:id", h.GetTest) + r.POST("/tests", h.StartTests) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/tests/"+repo.task.ID, nil)) + require.Equal(t, http.StatusOK, w.Code) + var envelope struct { + Data service.CodexGatewayBorrowTestTask `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &envelope)) + require.Equal(t, repo.result.Error, envelope.Data.Results[0].Error) + require.NotEmpty(t, envelope.Data.Results[0].PreviewURL) + requestBody := `{"client_task_id":"` + repo.task.ClientTaskID + `","targets":[{"account_id":42,"model_id":"gpt-6-astra"}]}` + stream := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/tests", strings.NewReader(requestBody)) + request.Header.Set("Content-Type", "application/json") + r.ServeHTTP(stream, request) + require.Equal(t, http.StatusOK, stream.Code) + require.Equal(t, "text/event-stream", stream.Header().Get("Content-Type")) + require.Contains(t, stream.Body.String(), `"type":"task_start"`) + require.Contains(t, stream.Body.String(), `"type":"task_complete"`) + require.NotContains(t, stream.Body.String(), `"type":"result_started"`) + require.Contains(t, stream.Body.String(), "original upstream error token=admin-visible") + require.Equal(t, 1, repo.creates) + require.Zero(t, repo.writes, "SSE replay must never start or regenerate an existing task") +} diff --git a/backend/internal/handler/codex_gateway_borrow_failover_test.go b/backend/internal/handler/codex_gateway_borrow_failover_test.go new file mode 100644 index 000000000000..3138eb604d79 --- /dev/null +++ b/backend/internal/handler/codex_gateway_borrow_failover_test.go @@ -0,0 +1,27 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/stretchr/testify/require" +) + +func TestCodexGatewayBorrowFailoverSkipsSameAccountRetry(t *testing.T) { + account := &service.Account{ID: 2, Platform: service.PlatformOpenAI, Type: service.AccountTypeOAuth} + failure := service.NewCodexGatewayBorrowRequestFailure(errors.New("source proxy connection refused")) + require.Zero(t, openAISameAccountRetryLimit(account, failure, true)) + require.True(t, failure.ShouldRetryNextAccount()) + require.False(t, failure.ShouldReportAccountScheduleFailure()) + state := newOpenAIFailoverRetryState() + action := state.HandleHTTP(context.Background(), &service.OpenAIGatewayService{}, account, "gpt-6-astra", failure, true, 0, "borrow-test") + require.Equal(t, openAIFailoverRetrySwitchAccount, action) + require.Zero(t, state.sameAccountRetryCount[account.ID]) + // The new reason is narrow: ordinary transport and upstream failures retain + // their existing bounded retry on OAuth accounts. + require.Equal(t, 1, openAISameAccountRetryLimit(account, &service.UpstreamFailoverError{StatusCode: http.StatusBadGateway, Reason: service.OpenAITransientTransportFailureReason}, true)) + require.Equal(t, 1, openAISameAccountRetryLimit(account, &service.UpstreamFailoverError{StatusCode: http.StatusServiceUnavailable}, true)) +} diff --git a/backend/internal/handler/codex_gateway_borrow_ws_failure_test.go b/backend/internal/handler/codex_gateway_borrow_ws_failure_test.go new file mode 100644 index 000000000000..96d95fd9c631 --- /dev/null +++ b/backend/internal/handler/codex_gateway_borrow_ws_failure_test.go @@ -0,0 +1,23 @@ +//go:build unit + +package handler + +import ( + "errors" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/stretchr/testify/require" +) + +func TestCodexGatewayBorrowWSPreparationFailureDoesNotReportTargetHealth(t *testing.T) { + local := &service.CodexGatewayBorrowFailure{Cause: errors.New("dial tcp synthetic-source.invalid:443: connect: connection refused")} + require.False(t, shouldReportOpenAIWSProxyAccountFailure(local)) + classified := service.NewCodexGatewayBorrowRequestFailure(local) + require.True(t, classified.ShouldRetryNextAccount()) + require.False(t, shouldReportOpenAIWSProxyAccountFailure(classified)) + closed := service.NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, "local preparation unavailable", classified) + require.False(t, shouldReportOpenAIWSProxyAccountFailure(closed)) + require.True(t, shouldReportOpenAIWSProxyAccountFailure(errors.New("ordinary business websocket connection refused"))) +} diff --git a/backend/internal/handler/handler.go b/backend/internal/handler/handler.go index 947b96a5da6f..3b493f1c7450 100644 --- a/backend/internal/handler/handler.go +++ b/backend/internal/handler/handler.go @@ -46,6 +46,7 @@ type AdminHandlers struct { AuditLog *admin.AuditLogHandler SystemPrompt *admin.SystemPromptHandler ReasoningRecovery *admin.ReasoningRecoveryHandler + CodexGatewayBorrow *admin.CodexGatewayBorrowHandler } // Handlers contains all HTTP handlers diff --git a/backend/internal/handler/openai_failover_retry.go b/backend/internal/handler/openai_failover_retry.go index c444f8e43bdb..cdca81067a29 100644 --- a/backend/internal/handler/openai_failover_retry.go +++ b/backend/internal/handler/openai_failover_retry.go @@ -120,6 +120,11 @@ func openAISameAccountRetryLimit(account *service.Account, failoverErr *service. if account == nil || failoverErr == nil || !failoverErr.ShouldRetryNextAccount() { return 0 } + if failoverErr.Reason == service.CodexGatewayBorrowPreparationFailureReason { + // A local preparation failure uses normal switch-account handling, but + // replaying it on the same account is not a network/inference retry. + return 0 + } if account.Type == service.AccountTypeAPIKey { if !account.IsPoolMode() || failoverErr.StatusCode <= 0 || !account.IsPoolModeRetryableStatus(failoverErr.StatusCode) || diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 69eacbd2e153..6c15320dae1c 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -113,6 +113,9 @@ func openAIWSPostOutputCyberClose(err error) (*service.OpenAIWSClientCloseError, } func shouldReportOpenAIWSProxyAccountFailure(err error) bool { + if service.IsCodexGatewayBorrowRequestFailure(err) { + return false + } if _, postOutputCyber := openAIWSPostOutputCyberClose(err); postOutputCyber { return false } diff --git a/backend/internal/handler/wire.go b/backend/internal/handler/wire.go index 1a30dfb4f2a0..90a4e357ad52 100644 --- a/backend/internal/handler/wire.go +++ b/backend/internal/handler/wire.go @@ -50,6 +50,7 @@ func ProvideAdminHandlers( auditLogHandler *admin.AuditLogHandler, systemPromptHandler *admin.SystemPromptHandler, reasoningRecoveryHandler *admin.ReasoningRecoveryHandler, + codexGatewayBorrowHandler *admin.CodexGatewayBorrowHandler, upstreamBillingProbe *service.UpstreamBillingProbeService, ollamaCloudUsage *service.OllamaCloudUsageService, opencodeGoUsage *service.OpenCodeGoUsageService, @@ -99,6 +100,7 @@ func ProvideAdminHandlers( AuditLog: auditLogHandler, SystemPrompt: systemPromptHandler, ReasoningRecovery: reasoningRecoveryHandler, + CodexGatewayBorrow: codexGatewayBorrowHandler, } } @@ -299,6 +301,7 @@ var ProviderSet = wire.NewSet( admin.NewAuditLogHandler, admin.NewSystemPromptHandler, admin.NewReasoningRecoveryHandler, + admin.NewCodexGatewayBorrowHandler, ProvideAccountJobRuntime, // AdminHandlers and Handlers constructors diff --git a/backend/internal/repository/codex_gateway_borrow_test_repo.go b/backend/internal/repository/codex_gateway_borrow_test_repo.go new file mode 100644 index 000000000000..ddf8f78a6fe1 --- /dev/null +++ b/backend/internal/repository/codex_gateway_borrow_test_repo.go @@ -0,0 +1,272 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/Wei-Shaw/sub2api/internal/service" +) + +type codexGatewayBorrowTestRepository struct{ db *sql.DB } + +func NewCodexGatewayBorrowTestRepository(db *sql.DB) service.CodexGatewayBorrowTestRepository { + return &codexGatewayBorrowTestRepository{db: db} +} + +const codexGatewayBorrowTestTaskColumns = `id, client_task_id, created_by, request_hash, status, prompt, + created_at, expires_at, started_at, finished_at, total, completed, error` + +const codexGatewayBorrowTestResultColumns = `r.id, r.task_id, r.ordinal, r.account_id, r.account_name, r.model_id, + r.upstream_model, r.effort, r.status, r.raw_answer, r.raw_response, r.raw_html, r.html, r.error, + r.started_at, r.finished_at, r.duration_ms, t.expires_at` + +type codexGatewayBorrowTestScanner interface{ Scan(...any) error } + +func scanCodexGatewayBorrowTestTask(row codexGatewayBorrowTestScanner) (*service.CodexGatewayBorrowTestTask, error) { + task := &service.CodexGatewayBorrowTestTask{} + var started, finished sql.NullTime + var rawError []byte + if err := row.Scan(&task.ID, &task.ClientTaskID, &task.CreatedBy, &task.RequestHash, &task.Status, &task.Prompt, + &task.CreatedAt, &task.ExpiresAt, &started, &finished, &task.Total, &task.Completed, &rawError); err != nil { + return nil, err + } + if started.Valid { + task.StartedAt = &started.Time + } + if finished.Valid { + task.FinishedAt = &finished.Time + } + task.Error = string(rawError) + return task, nil +} + +func scanCodexGatewayBorrowTestResult(row codexGatewayBorrowTestScanner) (*service.CodexGatewayBorrowTestResult, error) { + result := &service.CodexGatewayBorrowTestResult{} + var answer, rawResponse, rawHTML, html, rawError []byte + var started, finished sql.NullTime + if err := row.Scan(&result.ID, &result.TaskID, &result.Ordinal, &result.AccountID, &result.AccountName, &result.ModelID, + &result.UpstreamModel, &result.Effort, &result.Status, &answer, &rawResponse, &rawHTML, &html, &rawError, + &started, &finished, &result.DurationMS, &result.ExpiresAt); err != nil { + return nil, err + } + result.RawAnswer, result.RawResponse, result.RawHTML, result.HTML, result.Error = string(answer), string(rawResponse), string(rawHTML), string(html), string(rawError) + if started.Valid { + result.StartedAt = &started.Time + } + if finished.Valid { + result.FinishedAt = &finished.Time + } + if result.Status != "complete" { + result.PreviewUnavailable = "test " + result.Status + } else if result.HTML == "" { + result.PreviewUnavailable = "answer contains no extractable HTML or SVG" + } + return result, nil +} + +// The conflict read intentionally includes expired UUIDs. Reposting a UUID can +// never re-run a task while its row exists, even before expired-row cleanup. +func (r *codexGatewayBorrowTestRepository) Create(ctx context.Context, task *service.CodexGatewayBorrowTestTask) (*service.CodexGatewayBorrowTestTask, bool, error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return nil, false, err + } + defer func() { _ = tx.Rollback() }() + stored, err := scanCodexGatewayBorrowTestTask(tx.QueryRowContext(ctx, `INSERT INTO codex_gateway_borrow_test_tasks + (id,client_task_id,created_by,request_hash,status,prompt,total,created_at,expires_at) + VALUES ($1,$2,$3,$4,'pending',$5,$6,$7,$8) + ON CONFLICT (client_task_id) DO NOTHING RETURNING `+codexGatewayBorrowTestTaskColumns, + task.ID, task.ClientTaskID, task.CreatedBy, task.RequestHash, task.Prompt, task.Total, task.CreatedAt, task.ExpiresAt)) + if errors.Is(err, sql.ErrNoRows) { + stored, err = scanCodexGatewayBorrowTestTask(tx.QueryRowContext(ctx, `SELECT `+codexGatewayBorrowTestTaskColumns+` + FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican' AND client_task_id=$1 FOR UPDATE`, task.ClientTaskID)) + if err != nil { + return nil, false, err + } + if !stored.ExpiresAt.After(time.Now().UTC()) { + return nil, true, service.ErrCodexGatewayBorrowTestExpired + } + if stored.RequestHash != task.RequestHash { + return nil, true, service.ErrCodexGatewayBorrowTestReplayConflict + } + if err = tx.Commit(); err != nil { + return nil, true, err + } + stored, err = r.Get(ctx, stored.ID) + return stored, true, err + } + if err != nil { + return nil, false, err + } + for _, result := range task.Results { + if _, err = tx.ExecContext(ctx, `INSERT INTO codex_gateway_borrow_test_results + (id,task_id,ordinal,account_id,account_name,model_id,effort,status) + VALUES ($1,$2,$3,$4,$5,$6,$7,'pending')`, result.ID, stored.ID, result.Ordinal, result.AccountID, result.AccountName, result.ModelID, result.Effort); err != nil { + return nil, false, err + } + } + if err = tx.Commit(); err != nil { + return nil, false, err + } + stored.Results = task.Results + return stored, false, nil +} + +func (r *codexGatewayBorrowTestRepository) Get(ctx context.Context, id string) (*service.CodexGatewayBorrowTestTask, error) { + task, err := scanCodexGatewayBorrowTestTask(r.db.QueryRowContext(ctx, `SELECT `+codexGatewayBorrowTestTaskColumns+` + FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican' AND (id=$1 OR client_task_id=$1) AND expires_at>CURRENT_TIMESTAMP`, id)) + if errors.Is(err, sql.ErrNoRows) { + return nil, service.ErrCodexGatewayBorrowTestNotFound + } + if err != nil { + return nil, err + } + rows, err := r.db.QueryContext(ctx, `SELECT `+codexGatewayBorrowTestResultColumns+` + FROM codex_gateway_borrow_test_results r JOIN codex_gateway_borrow_test_tasks t ON t.id=r.task_id + WHERE t.scope='manual_pelican' AND t.id=$1 AND t.expires_at>CURRENT_TIMESTAMP ORDER BY r.ordinal`, task.ID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + task.Results = make([]*service.CodexGatewayBorrowTestResult, 0, task.Total) + for rows.Next() { + result, scanErr := scanCodexGatewayBorrowTestResult(rows) + if scanErr != nil { + return nil, scanErr + } + task.Results = append(task.Results, result) + } + if err = rows.Err(); err != nil { + return nil, err + } + if !task.ExpiresAt.After(time.Now().UTC()) { + return nil, service.ErrCodexGatewayBorrowTestNotFound + } + return task, nil +} + +func (r *codexGatewayBorrowTestRepository) GetResult(ctx context.Context, id string) (*service.CodexGatewayBorrowTestResult, error) { + result, err := scanCodexGatewayBorrowTestResult(r.db.QueryRowContext(ctx, `SELECT `+codexGatewayBorrowTestResultColumns+` + FROM codex_gateway_borrow_test_results r JOIN codex_gateway_borrow_test_tasks t ON t.id=r.task_id + WHERE t.scope='manual_pelican' AND r.id=$1 AND t.expires_at>CURRENT_TIMESTAMP`, id)) + if errors.Is(err, sql.ErrNoRows) { + return nil, service.ErrCodexGatewayBorrowTestNotFound + } + if err == nil && !result.ExpiresAt.After(time.Now().UTC()) { + return nil, service.ErrCodexGatewayBorrowTestNotFound + } + return result, err +} + +func (r *codexGatewayBorrowTestRepository) List(ctx context.Context, page, size int) (*service.CodexGatewayBorrowTestList, error) { + if page < 1 { + page = 1 + } + if size < 1 { + size = 20 + } + if size > 100 { + size = 100 + } + list := &service.CodexGatewayBorrowTestList{Items: make([]*service.CodexGatewayBorrowTestTask, 0, size), Page: page, Size: size, PageSize: size} + if err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM codex_gateway_borrow_test_tasks + WHERE scope='manual_pelican' AND expires_at>CURRENT_TIMESTAMP`).Scan(&list.Total); err != nil { + return nil, err + } + rows, err := r.db.QueryContext(ctx, `SELECT `+codexGatewayBorrowTestTaskColumns+` FROM codex_gateway_borrow_test_tasks + WHERE scope='manual_pelican' AND expires_at>CURRENT_TIMESTAMP ORDER BY created_at DESC,id DESC LIMIT $1 OFFSET $2`, size, (page-1)*size) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + for rows.Next() { + task, scanErr := scanCodexGatewayBorrowTestTask(rows) + if scanErr != nil { + return nil, scanErr + } + if task.ExpiresAt.After(time.Now().UTC()) { + list.Items = append(list.Items, task) + } + } + return list, rows.Err() +} + +func codexGatewayBorrowRequireAffected(result sql.Result, err error) error { + if err != nil { + return err + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count == 0 { + return service.ErrCodexGatewayBorrowTestNotFound + } + return nil +} + +func (r *codexGatewayBorrowTestRepository) StartTask(ctx context.Context, id string, started time.Time) error { + return codexGatewayBorrowRequireAffected(r.db.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_tasks + SET status='running',started_at=$2,updated_at=CURRENT_TIMESTAMP + WHERE scope='manual_pelican' AND id=$1 AND status='pending' AND expires_at>CURRENT_TIMESTAMP`, id, started)) +} + +func (r *codexGatewayBorrowTestRepository) SaveResult(ctx context.Context, result *service.CodexGatewayBorrowTestResult) error { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if err = codexGatewayBorrowRequireAffected(tx.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_results r + SET account_name=$3,upstream_model=$4,effort=$5,status=$6,raw_answer=$7,raw_response=$8,raw_html=$9,html=$10,error=$11, + started_at=$12,finished_at=$13,duration_ms=$14,updated_at=CURRENT_TIMESTAMP + FROM codex_gateway_borrow_test_tasks t + WHERE r.id=$1 AND r.task_id=$2 AND t.id=r.task_id AND t.scope='manual_pelican' AND t.expires_at>CURRENT_TIMESTAMP`, + result.ID, result.TaskID, result.AccountName, result.UpstreamModel, result.Effort, result.Status, + []byte(result.RawAnswer), []byte(result.RawResponse), []byte(result.RawHTML), []byte(result.HTML), []byte(result.Error), + result.StartedAt, result.FinishedAt, result.DurationMS)); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_tasks t + SET completed=(SELECT COUNT(*) FROM codex_gateway_borrow_test_results r WHERE r.task_id=t.id + AND r.status NOT IN ('pending','running')),updated_at=CURRENT_TIMESTAMP + WHERE t.scope='manual_pelican' AND t.id=$1 AND t.expires_at>CURRENT_TIMESTAMP`, result.TaskID); err != nil { + return err + } + return tx.Commit() +} + +func (r *codexGatewayBorrowTestRepository) FinishTask(ctx context.Context, id, status, errorText string, finished time.Time) error { + return codexGatewayBorrowRequireAffected(r.db.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_tasks + SET status=$2,error=$3,finished_at=$4,updated_at=CURRENT_TIMESTAMP + WHERE scope='manual_pelican' AND id=$1 AND expires_at>CURRENT_TIMESTAMP`, id, status, []byte(errorText), finished)) +} + +func (r *codexGatewayBorrowTestRepository) MarkInterrupted(ctx context.Context, now time.Time) error { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + message := []byte("process stopped before the test finished; this task will not be replayed") + if _, err = tx.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_results r + SET status='incomplete',error=$2,finished_at=$1,updated_at=$1 + FROM codex_gateway_borrow_test_tasks t WHERE t.id=r.task_id AND t.scope='manual_pelican' + AND t.expires_at>$1 AND t.status IN ('pending','running') AND r.status IN ('pending','running')`, now, message); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_tasks t + SET status='incomplete',error=$2,finished_at=$1,completed=t.total,updated_at=$1 + WHERE t.scope='manual_pelican' AND t.expires_at>$1 AND t.status IN ('pending','running')`, now, message); err != nil { + return err + } + return tx.Commit() +} + +func (r *codexGatewayBorrowTestRepository) CleanupExpired(ctx context.Context, now time.Time) error { + _, err := r.db.ExecContext(ctx, `DELETE FROM codex_gateway_borrow_test_tasks + WHERE scope='manual_pelican' AND expires_at<=$1`, now) + return err +} diff --git a/backend/internal/repository/codex_gateway_borrow_test_repo_integration_test.go b/backend/internal/repository/codex_gateway_borrow_test_repo_integration_test.go new file mode 100644 index 000000000000..ff93f6f8fbbd --- /dev/null +++ b/backend/internal/repository/codex_gateway_borrow_test_repo_integration_test.go @@ -0,0 +1,214 @@ +//go:build integration + +package repository + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/Wei-Shaw/sub2api/migrations" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/stretchr/testify/require" +) + +// The real repository opens its own transactions, so a private schema keeps +// its expiry cleanup separate from every other integration fixture and ledger. +// Copy the harness connection configuration without displaying its credentials. +func codexGatewayBorrowPrivateIntegrationDB(t *testing.T, ctx context.Context) *sql.DB { + t.Helper() + conn, err := integrationDB.Conn(ctx) + require.NoError(t, err) + var connectionConfig *pgx.ConnConfig + err = conn.Raw(func(driverConnection any) error { + connection, ok := driverConnection.(*stdlib.Conn) + if !ok { + return errors.New("integration harness must use its pgx SQL driver") + } + connectionConfig = connection.Conn().Config().Copy() + return nil + }) + closeErr := conn.Close() + require.NoError(t, err) + require.NoError(t, closeErr) + + schema := "codex_borrow_it_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quotedSchema := pgx.Identifier{schema}.Sanitize() + _, err = integrationDB.ExecContext(ctx, "CREATE SCHEMA "+quotedSchema) + require.NoError(t, err) + var privateDB *sql.DB + t.Cleanup(func() { + if privateDB != nil { + require.NoError(t, privateDB.Close()) + } + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // This exact random schema was created above solely for this test. The + // shared public schema and its diagnostic rows are never deleted. + _, cleanupErr := integrationDB.ExecContext(cleanupCtx, "DROP SCHEMA "+quotedSchema+" CASCADE") + require.NoError(t, cleanupErr) + }) + if connectionConfig.RuntimeParams == nil { + connectionConfig.RuntimeParams = make(map[string]string) + } + connectionConfig.RuntimeParams["search_path"] = schema + privateDB = stdlib.OpenDB(*connectionConfig) + require.NoError(t, privateDB.PingContext(ctx)) + ddl, err := migrations.FS.ReadFile("275_codex_gateway_borrow_tests.sql") + require.NoError(t, err) + _, err = privateDB.ExecContext(ctx, string(ddl)) + require.NoError(t, err, "apply the actual new migration inside the private fixture") + return privateDB +} + +func codexGatewayBorrowIntegrationTask(t *testing.T, accountID int64) *service.CodexGatewayBorrowTestTask { + t.Helper() + clientID, err := uuid.NewV7() + require.NoError(t, err) + created := time.Now().UTC().Truncate(time.Microsecond) + task := &service.CodexGatewayBorrowTestTask{ + ID: uuid.NewString(), ClientTaskID: clientID.String(), CreatedBy: 7, + RequestHash: strings.Repeat("a", 64), Status: "pending", Prompt: service.CodexGatewayBorrowPelicanPrompt, + CreatedAt: created, ExpiresAt: created.Add(service.CodexGatewayBorrowTestTTL), Total: 1, + } + task.Results = []*service.CodexGatewayBorrowTestResult{{ + ID: uuid.NewString(), TaskID: task.ID, Ordinal: 1, AccountID: accountID, + AccountName: "private integration fixture", ModelID: "gpt-6-astra", Effort: "high", + Status: "pending", ExpiresAt: task.ExpiresAt, + }} + return task +} + +func TestCodexGatewayBorrowTestRepositoryIntegrationLifecycle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + db := codexGatewayBorrowPrivateIntegrationDB(t, ctx) + repo := NewCodexGatewayBorrowTestRepository(db) + task := codexGatewayBorrowIntegrationTask(t, 987654321) + + // No account fixture is needed: these are retained observations, not an + // account relationship or a model request. The genuine schema enforces this. + stored, replayed, err := repo.Create(ctx, task) + require.NoError(t, err) + require.False(t, replayed) + require.Equal(t, task.ID, stored.ID) + require.Equal(t, "pending", stored.Status) + require.Len(t, stored.Results, 1) + require.Equal(t, service.CodexGatewayBorrowTestTTL, stored.ExpiresAt.Sub(stored.CreatedAt)) + + started := time.Now().UTC().Truncate(time.Microsecond) + require.NoError(t, repo.StartTask(ctx, task.ID, started)) + result := task.Results[0] + finished := started.Add(250 * time.Millisecond) + result.Status, result.UpstreamModel = "complete", "gpt-6-astra-reported" + result.StartedAt, result.FinishedAt, result.DurationMS = &started, &finished, 250 + result.RawAnswer = "```svg\n鹈鹕\x00tail\n```" + result.RawResponse = "data: {\"original\":\"full stream\"}\n\n\x00end" + result.RawHTML = "鹈鹕\x00tail" + result.HTML = "" + result.RawHTML + "" + result.Error = "original upstream diagnostic token=admin-visible\x00untruncated tail" + require.NoError(t, repo.SaveResult(ctx, result)) + const taskError = "task observation\x00original diagnostic" + require.NoError(t, repo.FinishTask(ctx, task.ID, "complete", taskError, finished)) + + readResult, err := repo.GetResult(ctx, result.ID) + require.NoError(t, err) + require.Equal(t, result.RawAnswer, readResult.RawAnswer) + require.Equal(t, result.RawResponse, readResult.RawResponse) + require.Equal(t, result.RawHTML, readResult.RawHTML) + require.Equal(t, result.HTML, readResult.HTML) + require.Equal(t, result.Error, readResult.Error) + require.Equal(t, "gpt-6-astra", readResult.ModelID) + require.Equal(t, result.UpstreamModel, readResult.UpstreamModel) + require.Equal(t, result.DurationMS, readResult.DurationMS) + require.True(t, readResult.StartedAt.Equal(started)) + require.True(t, readResult.FinishedAt.Equal(finished)) + require.True(t, readResult.ExpiresAt.Equal(task.ExpiresAt)) + readTask, err := repo.Get(ctx, task.ClientTaskID) + require.NoError(t, err) + require.Equal(t, "complete", readTask.Status) + require.Equal(t, 1, readTask.Completed) + require.Equal(t, taskError, readTask.Error) + require.Len(t, readTask.Results, 1) + require.Equal(t, result.RawAnswer, readTask.Results[0].RawAnswer) + + // A different server-side ID cannot replace an existing client UUID. The + // replay returns the old complete task and its exact BYTEA-backed results. + replayInput := codexGatewayBorrowIntegrationTask(t, result.AccountID) + replayInput.ClientTaskID = task.ClientTaskID + old, replayed, err := repo.Create(ctx, replayInput) + require.NoError(t, err) + require.True(t, replayed) + require.Equal(t, task.ID, old.ID) + require.NotEqual(t, replayInput.ID, old.ID) + require.Equal(t, result.RawResponse, old.Results[0].RawResponse) + replayInput.RequestHash = strings.Repeat("b", 64) + _, replayed, err = repo.Create(ctx, replayInput) + require.True(t, replayed) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestReplayConflict) + var clientRows int + require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM codex_gateway_borrow_test_tasks WHERE client_task_id=$1`, task.ClientTaskID).Scan(&clientRows)) + require.Equal(t, 1, clientRows) + + keeper := codexGatewayBorrowIntegrationTask(t, 987654322) + _, replayed, err = repo.Create(ctx, keeper) + require.NoError(t, err) + require.False(t, replayed) + // Only a minimal, private canary is needed for the cleanup boundary. This + // table intentionally uses a diagnostic name and an old timestamp; the + // genuine shared diagnostic tables remain outside this pool's search path. + _, err = db.ExecContext(ctx, `CREATE TABLE ops_error_logs (id UUID PRIMARY KEY, created_at TIMESTAMPTZ NOT NULL, original_body BYTEA NOT NULL)`) + require.NoError(t, err) + canaryID := uuid.NewString() + canaryBody := []byte("retained diagnostic\x00original body") + _, err = db.ExecContext(ctx, `INSERT INTO ops_error_logs(id,created_at,original_body) VALUES ($1,$2,$3)`, canaryID, task.CreatedAt.Add(-72*time.Hour), canaryBody) + require.NoError(t, err) + + expiredCreated := time.Now().UTC().Add(-25 * time.Hour).Truncate(time.Microsecond) + _, err = db.ExecContext(ctx, `UPDATE codex_gateway_borrow_test_tasks SET created_at=$2,expires_at=$3 WHERE id=$1`, task.ID, expiredCreated, expiredCreated.Add(service.CodexGatewayBorrowTestTTL)) + require.NoError(t, err) + _, err = repo.Get(ctx, task.ID) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestNotFound) + _, err = repo.GetResult(ctx, result.ID) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestNotFound) + _, replayed, err = repo.Create(ctx, replayInput) + require.True(t, replayed) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestExpired) + list, err := repo.List(ctx, 1, 10) + require.NoError(t, err) + require.Equal(t, int64(1), list.Total) + require.Len(t, list.Items, 1) + require.Equal(t, keeper.ID, list.Items[0].ID) + + countTaskAndChildren := func(taskID string) (int, int) { + t.Helper() + var tasks, children int + require.NoError(t, db.QueryRowContext(ctx, `SELECT + (SELECT COUNT(*) FROM codex_gateway_borrow_test_tasks WHERE id=$1), + (SELECT COUNT(*) FROM codex_gateway_borrow_test_results WHERE task_id=$1)`, taskID).Scan(&tasks, &children)) + return tasks, children + } + tasks, children := countTaskAndChildren(task.ID) + require.Equal(t, 1, tasks, "expired rows stay physically present until cleanup") + require.Equal(t, 1, children) + // Invoke exactly the SQL method used by the runner's minute timer, without + // sleeping or starting any model/runner service. + require.NoError(t, repo.CleanupExpired(ctx, time.Now().UTC())) + tasks, children = countTaskAndChildren(task.ID) + require.Zero(t, tasks) + require.Zero(t, children, "the migration's real foreign key must cascade") + tasks, children = countTaskAndChildren(keeper.ID) + require.Equal(t, 1, tasks) + require.Equal(t, 1, children) + var retainedBody []byte + require.NoError(t, db.QueryRowContext(ctx, `SELECT original_body FROM ops_error_logs WHERE id=$1`, canaryID).Scan(&retainedBody)) + require.Equal(t, canaryBody, retainedBody) + _, err = repo.GetResult(ctx, keeper.Results[0].ID) + require.NoError(t, err, "cleanup must preserve the unexpired observation") +} diff --git a/backend/internal/repository/codex_gateway_borrow_test_repo_test.go b/backend/internal/repository/codex_gateway_borrow_test_repo_test.go new file mode 100644 index 000000000000..32d3a5b1fc9e --- /dev/null +++ b/backend/internal/repository/codex_gateway_borrow_test_repo_test.go @@ -0,0 +1,142 @@ +//go:build unit + +package repository + +import ( + "context" + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/Wei-Shaw/sub2api/migrations" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func newCodexGatewayBorrowRepoTest(t *testing.T) (*codexGatewayBorrowTestRepository, sqlmock.Sqlmock) { + t.Helper() + db, mock, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + return &codexGatewayBorrowTestRepository{db: db}, mock +} + +func codexGatewayBorrowTaskTestRows(task *service.CodexGatewayBorrowTestTask) *sqlmock.Rows { + return sqlmock.NewRows([]string{"id", "client_task_id", "created_by", "request_hash", "status", "prompt", "created_at", "expires_at", "started_at", "finished_at", "total", "completed", "error"}).AddRow( + task.ID, task.ClientTaskID, task.CreatedBy, task.RequestHash, task.Status, task.Prompt, task.CreatedAt, task.ExpiresAt, nil, nil, task.Total, task.Completed, []byte(task.Error)) +} + +func codexGatewayBorrowResultTestRows(result *service.CodexGatewayBorrowTestResult) *sqlmock.Rows { + return sqlmock.NewRows([]string{"id", "task_id", "ordinal", "account_id", "account_name", "model_id", "upstream_model", "effort", "status", "raw_answer", "raw_response", "raw_html", "html", "error", "started_at", "finished_at", "duration_ms", "expires_at"}).AddRow( + result.ID, result.TaskID, result.Ordinal, result.AccountID, result.AccountName, result.ModelID, result.UpstreamModel, result.Effort, result.Status, []byte(result.RawAnswer), []byte(result.RawResponse), []byte(result.RawHTML), []byte(result.HTML), []byte(result.Error), nil, nil, result.DurationMS, result.ExpiresAt) +} + +func codexGatewayBorrowRepoFixture(t *testing.T) *service.CodexGatewayBorrowTestTask { + t.Helper() + now := time.Now().UTC() + clientID, err := uuid.NewV7() + require.NoError(t, err) + task := &service.CodexGatewayBorrowTestTask{ID: uuid.NewString(), ClientTaskID: clientID.String(), CreatedBy: 7, RequestHash: "hash", Status: "pending", Prompt: service.CodexGatewayBorrowPelicanPrompt, CreatedAt: now, ExpiresAt: now.Add(service.CodexGatewayBorrowTestTTL), Total: 1} + task.Results = []*service.CodexGatewayBorrowTestResult{{ID: uuid.NewString(), TaskID: task.ID, Ordinal: 1, AccountID: 42, AccountName: "paused target", ModelID: "gpt-6-astra", Effort: "high", Status: "pending", ExpiresAt: task.ExpiresAt}} + return task +} + +func TestCodexGatewayBorrowTestRepositoryCreateAtomicAndReplayReadsWithoutWrites(t *testing.T) { + repo, mock := newCodexGatewayBorrowRepoTest(t) + task := codexGatewayBorrowRepoFixture(t) + mock.ExpectBegin() + mock.ExpectQuery("INSERT INTO codex_gateway_borrow_test_tasks").WithArgs(task.ID, task.ClientTaskID, task.CreatedBy, task.RequestHash, task.Prompt, task.Total, task.CreatedAt, task.ExpiresAt).WillReturnRows(codexGatewayBorrowTaskTestRows(task)) + result := task.Results[0] + mock.ExpectExec("INSERT INTO codex_gateway_borrow_test_results").WithArgs(result.ID, task.ID, 1, result.AccountID, result.AccountName, result.ModelID, result.Effort).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + stored, replayed, err := repo.Create(context.Background(), task) + require.NoError(t, err) + require.False(t, replayed) + require.Equal(t, task.ID, stored.ID) + require.Len(t, stored.Results, 1) + // A repeated client UUID gets the original full observation, not another child. + task.Status = "complete" + result.Status, result.RawAnswer, result.RawResponse, result.Error = "complete", "answer\x00tail", "full raw stream\x00end", "untruncated upstream diagnostic" + mock.ExpectBegin() + mock.ExpectQuery("INSERT INTO codex_gateway_borrow_test_tasks").WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectQuery("FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican' AND client_task_id=\\$1 FOR UPDATE").WithArgs(task.ClientTaskID).WillReturnRows(codexGatewayBorrowTaskTestRows(task)) + mock.ExpectCommit() + mock.ExpectQuery("FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican'.*expires_at>CURRENT_TIMESTAMP").WithArgs(task.ID).WillReturnRows(codexGatewayBorrowTaskTestRows(task)) + mock.ExpectQuery("FROM codex_gateway_borrow_test_results r JOIN codex_gateway_borrow_test_tasks t.*t.expires_at>CURRENT_TIMESTAMP").WithArgs(task.ID).WillReturnRows(codexGatewayBorrowResultTestRows(result)) + stored, replayed, err = repo.Create(context.Background(), task) + require.NoError(t, err) + require.True(t, replayed) + require.Equal(t, result.RawAnswer, stored.Results[0].RawAnswer) + require.Equal(t, result.RawResponse, stored.Results[0].RawResponse) + require.Equal(t, result.Error, stored.Results[0].Error) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCodexGatewayBorrowTestRepositoryRejectsConflictingAndExpiredReplay(t *testing.T) { + for _, expired := range []bool{false, true} { + t.Run(map[bool]string{false: "conflict", true: "expired"}[expired], func(t *testing.T) { + repo, mock := newCodexGatewayBorrowRepoTest(t) + task := codexGatewayBorrowRepoFixture(t) + original := *task + if expired { + original.CreatedAt = time.Now().Add(-25 * time.Hour) + original.ExpiresAt = original.CreatedAt.Add(service.CodexGatewayBorrowTestTTL) + } else { + original.RequestHash = "different" + } + mock.ExpectBegin() + mock.ExpectQuery("INSERT INTO codex_gateway_borrow_test_tasks").WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectQuery("client_task_id=\\$1 FOR UPDATE").WillReturnRows(codexGatewayBorrowTaskTestRows(&original)) + mock.ExpectRollback() + _, replayed, err := repo.Create(context.Background(), task) + require.True(t, replayed) + if expired { + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestExpired) + } else { + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestReplayConflict) + } + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + +func TestCodexGatewayBorrowTestRepositoryExpiredReadsAndScopedCleanup(t *testing.T) { + repo, mock := newCodexGatewayBorrowRepoTest(t) + id := uuid.NewString() + mock.ExpectQuery("FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican'.*expires_at>CURRENT_TIMESTAMP").WithArgs(id).WillReturnRows(sqlmock.NewRows([]string{"id"})) + _, err := repo.Get(context.Background(), id) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestNotFound) + mock.ExpectQuery("WHERE t.scope='manual_pelican' AND r.id=\\$1 AND t.expires_at>CURRENT_TIMESTAMP").WithArgs(id).WillReturnRows(sqlmock.NewRows([]string{"id"})) + _, err = repo.GetResult(context.Background(), id) + require.ErrorIs(t, err, service.ErrCodexGatewayBorrowTestNotFound) + now := time.Now().UTC() + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM codex_gateway_borrow_test_tasks WHERE scope='manual_pelican' AND expires_at<=$1")).WithArgs(now).WillReturnResult(sqlmock.NewResult(0, 2)) + require.NoError(t, repo.CleanupExpired(context.Background(), now)) + require.NoError(t, mock.ExpectationsWereMet()) + data, err := migrations.FS.ReadFile("275_codex_gateway_borrow_tests.sql") + require.NoError(t, err) + require.Contains(t, string(data), "task_id UUID NOT NULL REFERENCES codex_gateway_borrow_test_tasks(id) ON DELETE CASCADE") + require.Contains(t, string(data), "client_task_id UUID NOT NULL UNIQUE") + require.Contains(t, string(data), "expires_at - created_at = INTERVAL '24 hours'") + require.Contains(t, string(data), "raw_answer BYTEA NOT NULL") +} + +func TestCodexGatewayBorrowTestRepositorySavesFullRawValuesAndMarksRestartIncomplete(t *testing.T) { + repo, mock := newCodexGatewayBorrowRepoTest(t) + task := codexGatewayBorrowRepoFixture(t) + result := task.Results[0] + result.Status, result.UpstreamModel, result.RawAnswer, result.RawResponse, result.RawHTML, result.HTML, result.Error = "incomplete", "reported", "answer\x00tail", "SSE\x00raw", "source\x00raw", "html\x00raw", "API returned 403: entire original body\x00tail" + mock.ExpectBegin() + mock.ExpectExec("UPDATE codex_gateway_borrow_test_results r.*t.expires_at>CURRENT_TIMESTAMP").WithArgs(result.ID, result.TaskID, result.AccountName, result.UpstreamModel, result.Effort, result.Status, []byte(result.RawAnswer), []byte(result.RawResponse), []byte(result.RawHTML), []byte(result.HTML), []byte(result.Error), nil, nil, int64(0)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE codex_gateway_borrow_test_tasks t.*completed=.*status NOT IN").WithArgs(task.ID).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + require.NoError(t, repo.SaveResult(context.Background(), result)) + mock.ExpectBegin() + mock.ExpectExec("UPDATE codex_gateway_borrow_test_results r.*status='incomplete'.*t.scope='manual_pelican'.*r.status IN").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE codex_gateway_borrow_test_tasks t.*status='incomplete'.*t.scope='manual_pelican'").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + require.NoError(t, repo.MarkInterrupted(context.Background(), time.Now().UTC())) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/internal/repository/codex_gateway_borrow_upstream.go b/backend/internal/repository/codex_gateway_borrow_upstream.go new file mode 100644 index 000000000000..ae469db94087 --- /dev/null +++ b/backend/internal/repository/codex_gateway_borrow_upstream.go @@ -0,0 +1,171 @@ +package repository + +import ( + "crypto/sha256" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "slices" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/Wei-Shaw/sub2api/internal/service" +) + +const ( + upstreamProtocolModeCodexBorrowTargetH1 = "codex_borrow_target_h1_no_reuse" + codexBorrowProbeHeaderTimeout = 45 * time.Second +) + +// codexGatewayBorrowProbeUpstream owns two independent client managers. In +// particular, probes cannot consume the business client's eviction budget or +// change its HTTP/2 proxy fallback state, even for the same account and proxy. +type codexGatewayBorrowProbeUpstream struct { + source *httpUpstreamService + target *httpUpstreamService +} + +func NewCodexGatewayBorrowProbeUpstream(cfg *config.Config) service.CodexGatewayBorrowProbeUpstream { + newManager := func(purpose service.HTTPUpstreamProfile) *httpUpstreamService { + return &httpUpstreamService{ + cfg: cloneCodexBorrowProbeHTTPConfig(cfg), + clients: make(map[string]*upstreamClientEntry), + codexBorrowPurpose: purpose, + } + } + return &codexGatewayBorrowProbeUpstream{ + source: newManager(service.HTTPUpstreamProfileCodexBorrowSource), + target: newManager(service.HTTPUpstreamProfileCodexBorrowTarget), + } +} + +func (s *codexGatewayBorrowProbeUpstream) Do(req *http.Request, proxyURL string, accountID int64, _ int) (*http.Response, error) { + manager, req, err := s.requestManager(req) + if err != nil { + return nil, err + } + // Probe capacity follows the copied global transport settings, independently + // of the account's gateway concurrency slots and business pool dimensions. + return manager.Do(req, proxyURL, accountID, 0) +} + +func (s *codexGatewayBorrowProbeUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, _ int, profile *tlsfingerprint.Profile) (*http.Response, error) { + manager, req, err := s.requestManager(req) + if err != nil { + return nil, err + } + return manager.DoWithTLS(req, proxyURL, accountID, 0, profile) +} + +func (s *codexGatewayBorrowProbeUpstream) requestManager(req *http.Request) (*httpUpstreamService, *http.Request, error) { + if req == nil || req.URL == nil { + return nil, nil, errors.New("borrow probe request URL is nil") + } + switch service.HTTPUpstreamProfileFromContext(req.Context()) { + case service.HTTPUpstreamProfileCodexBorrowSource: + // Source sends the normal OpenAI transport policy through its own pool. + ctx := service.WithHTTPUpstreamProfile(req.Context(), service.HTTPUpstreamProfileOpenAI) + return s.source, req.Clone(ctx), nil + case service.HTTPUpstreamProfileCodexBorrowTarget: + clone := req.Clone(req.Context()) + clone.Close = true + return s.target, clone, nil + default: + return nil, nil, errors.New("borrow probe request purpose is missing") + } +} + +// Copy only the configuration read by the upstream transport. The managers +// neither mutate nor retain slices belonging to the business configuration. +func cloneCodexBorrowProbeHTTPConfig(cfg *config.Config) *config.Config { + if cfg == nil { + return nil + } + allowlist := cfg.Security.URLAllowlist + allowlist.UpstreamHosts = slices.Clone(allowlist.UpstreamHosts) + allowlist.PricingHosts = slices.Clone(allowlist.PricingHosts) + allowlist.CRSHosts = slices.Clone(allowlist.CRSHosts) + return &config.Config{ + Gateway: config.GatewayConfig{ + ConnectionPoolIsolation: cfg.Gateway.ConnectionPoolIsolation, + MaxIdleConns: cfg.Gateway.MaxIdleConns, + MaxIdleConnsPerHost: cfg.Gateway.MaxIdleConnsPerHost, + MaxConnsPerHost: cfg.Gateway.MaxConnsPerHost, + IdleConnTimeoutSeconds: cfg.Gateway.IdleConnTimeoutSeconds, + ResponseHeaderTimeout: cfg.Gateway.ResponseHeaderTimeout, + OpenAIResponseHeaderTimeout: cfg.Gateway.OpenAIResponseHeaderTimeout, + OpenAIHTTP2: cfg.Gateway.OpenAIHTTP2, + MaxUpstreamClients: cfg.Gateway.MaxUpstreamClients, + ClientIdleTTLSeconds: cfg.Gateway.ClientIdleTTLSeconds, + }, + Security: config.SecurityConfig{URLAllowlist: allowlist}, + } +} + +func (s *httpUpstreamService) codexBorrowClientCacheKey(key string) string { + if s.codexBorrowPurpose == service.HTTPUpstreamProfileDefault { + return key + } + return string(s.codexBorrowPurpose) + "|" + key +} + +// Fingerprinted probe clients must be keyed by the full resolved profile, not +// merely its display name. A bound profile can change without changing its name. +func codexBorrowTLSProfileKey(profile *tlsfingerprint.Profile) string { + encoded, _ := json.Marshal(profile) // Profile contains only JSON-safe values. + return fmt.Sprintf("%x", sha256.Sum256(encoded)) +} + +func cloneCodexBorrowTLSProfile(profile *tlsfingerprint.Profile) *tlsfingerprint.Profile { + if profile == nil { + return nil + } + clone := *profile + clone.CipherSuites = slices.Clone(profile.CipherSuites) + clone.Curves = slices.Clone(profile.Curves) + clone.PointFormats = slices.Clone(profile.PointFormats) + clone.SignatureAlgorithms = slices.Clone(profile.SignatureAlgorithms) + clone.ALPNProtocols = slices.Clone(profile.ALPNProtocols) + clone.SupportedVersions = slices.Clone(profile.SupportedVersions) + clone.KeyShareGroups = slices.Clone(profile.KeyShareGroups) + clone.PSKModes = slices.Clone(profile.PSKModes) + clone.Extensions = slices.Clone(profile.Extensions) + return &clone +} + +func (s *httpUpstreamService) buildCodexBorrowTLSFingerprintTransport(settings poolSettings, proxyURL *url.URL, profile *tlsfingerprint.Profile) (*http.Transport, error) { + if s.codexBorrowPurpose != service.HTTPUpstreamProfileCodexBorrowTarget { + return buildUpstreamTransportWithTLSFingerprint(settings, proxyURL, profile) + } + // Keep the selected cipher suites, extensions and other fingerprint fields, + // while excluding H2 from the actual ClientHello used by every target dialer. + profile = cloneCodexBorrowTLSProfile(profile) + if profile != nil { + profile.ALPNProtocols = []string{"http/1.1"} + } + transport, err := buildUpstreamTransportWithTLSFingerprint(settings, proxyURL, profile) + if err != nil { + return nil, err + } + configureCodexBorrowTargetTransport(transport) + return transport, nil +} + +func configureCodexBorrowTargetTransport(transport *http.Transport) { + transport.ForceAttemptHTTP2 = false + transport.DisableKeepAlives = true + transport.DisableCompression = true + transport.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper) + transport.Protocols = &http.Protocols{} + transport.Protocols.SetHTTP1(true) + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } else { + transport.TLSClientConfig = transport.TLSClientConfig.Clone() + } + transport.TLSClientConfig.NextProtos = []string{"http/1.1"} +} diff --git a/backend/internal/repository/codex_gateway_borrow_upstream_test.go b/backend/internal/repository/codex_gateway_borrow_upstream_test.go new file mode 100644 index 000000000000..f8aae92cd36c --- /dev/null +++ b/backend/internal/repository/codex_gateway_borrow_upstream_test.go @@ -0,0 +1,380 @@ +package repository + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/stretchr/testify/require" +) + +func codexBorrowProbeTestConfig(maxClients int) *config.Config { + return &config.Config{Gateway: config.GatewayConfig{ + ConnectionPoolIsolation: config.ConnectionPoolIsolationAccountProxy, + MaxIdleConns: 4, + MaxIdleConnsPerHost: 2, + MaxConnsPerHost: 2, + MaxUpstreamClients: maxClients, + ClientIdleTTLSeconds: 900, + OpenAIHTTP2: config.GatewayOpenAIHTTP2Config{ + Enabled: true, + AllowProxyFallbackToHTTP1: true, + FallbackErrorThreshold: 1, + }, + }} +} + +func codexBorrowProbeTestRequest(t *testing.T, purpose service.HTTPUpstreamProfile, rawURL string) *http.Request { + t.Helper() + ctx := service.WithHTTPUpstreamProfile(t.Context(), purpose) + ctx = service.WithHTTPUpstreamRedirectsDisabled(ctx) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + require.NoError(t, err) + return req +} + +func closeCodexBorrowProbeTestClients(t *testing.T, managers ...*httpUpstreamService) { + t.Helper() + t.Cleanup(func() { + for _, manager := range managers { + manager.mu.RLock() + clients := make([]*http.Client, 0, len(manager.clients)) + for _, entry := range manager.clients { + clients = append(clients, entry.client) + } + manager.mu.RUnlock() + for _, client := range clients { + client.CloseIdleConnections() + } + } + }) +} + +func TestCodexGatewayBorrowProbeUpstream_IsolatesClientBudgetsAndFallbacks(t *testing.T) { + cfg := codexBorrowProbeTestConfig(1) + business, ok := NewHTTPUpstream(cfg).(*httpUpstreamService) + require.True(t, ok) + probe, ok := NewCodexGatewayBorrowProbeUpstream(cfg).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + closeCodexBorrowProbeTestClients(t, business, probe.source, probe.target) + require.NotSame(t, cfg, probe.source.cfg) + require.NotSame(t, cfg, probe.target.cfg) + require.NotSame(t, probe.source.cfg, probe.target.cfg) + + businessEntry, err := business.acquireClientWithProfile("", 7, 1, service.HTTPUpstreamProfileOpenAI) + require.NoError(t, err) + sourceEntry, err := probe.source.acquireClientWithProfile("", 7, 0, service.HTTPUpstreamProfileOpenAI) + require.NoError(t, err) + targetEntry, err := probe.target.acquireClientWithProfile("", 7, 0, service.HTTPUpstreamProfileCodexBorrowTarget) + require.NoError(t, err) + require.NotSame(t, businessEntry.client, sourceEntry.client) + require.NotSame(t, businessEntry.client, targetEntry.client) + require.NotSame(t, sourceEntry.client, targetEntry.client) + + _, err = probe.source.acquireClientWithProfile("", 8, 0, service.HTTPUpstreamProfileOpenAI) + require.ErrorIs(t, err, errUpstreamClientLimitReached) + // Releasing one source client permits only that manager's eviction. + atomic.AddInt64(&sourceEntry.inFlight, -1) + replacement, err := probe.source.acquireClientWithProfile("", 8, 0, service.HTTPUpstreamProfileOpenAI) + require.NoError(t, err) + require.NotSame(t, sourceEntry, replacement) + require.Len(t, business.clients, 1) + require.Len(t, probe.source.clients, 1) + require.Len(t, probe.target.clients, 1) + for _, entry := range business.clients { + require.Same(t, businessEntry, entry) + } + for _, entry := range probe.target.clients { + require.Same(t, targetEntry, entry) + } + for key := range probe.source.clients { + require.True(t, strings.HasPrefix(key, string(service.HTTPUpstreamProfileCodexBorrowSource)+"|")) + } + for key := range probe.target.clients { + require.True(t, strings.HasPrefix(key, string(service.HTTPUpstreamProfileCodexBorrowTarget)+"|")) + } + + proxyKey := "http://probe-proxy.example:8080" + parsedProxy, err := url.Parse(proxyKey) + require.NoError(t, err) + probe.source.recordOpenAIHTTP2Failure(service.HTTPUpstreamProfileOpenAI, upstreamProtocolModeOpenAIH2, proxyKey, errors.New("GOAWAY")) + require.Equal(t, upstreamProtocolModeOpenAIH1Fallback, probe.source.resolveProtocolMode(service.HTTPUpstreamProfileOpenAI, proxyKey, parsedProxy)) + require.Equal(t, upstreamProtocolModeOpenAIH2, business.resolveProtocolMode(service.HTTPUpstreamProfileOpenAI, proxyKey, parsedProxy)) + require.False(t, probe.target.isOpenAIHTTP2FallbackActive(proxyKey)) + require.Equal(t, upstreamProtocolModeCodexBorrowTargetH1, probe.target.resolveProtocolMode(service.HTTPUpstreamProfileCodexBorrowTarget, proxyKey, parsedProxy)) + + // TLS clients consume the same bounded cache within their own purpose only. + _, err = probe.source.acquireClientWithTLS("", 9, 0, &tlsfingerprint.Profile{Name: "test"}, service.HTTPUpstreamProfileOpenAI) + require.ErrorIs(t, err, errUpstreamClientLimitReached) + require.Same(t, businessEntry, business.clients[buildCacheKey(config.ConnectionPoolIsolationAccountProxy, directProxyKey, 7, upstreamProtocolModeOpenAIH2)]) +} + +func TestCodexGatewayBorrowProbeUpstream_UsesGlobalProbeCapacity(t *testing.T) { + for _, purpose := range []service.HTTPUpstreamProfile{ + service.HTTPUpstreamProfileCodexBorrowSource, + service.HTTPUpstreamProfileCodexBorrowTarget, + } { + t.Run(string(purpose), func(t *testing.T) { + cfg := codexBorrowProbeTestConfig(2) + cfg.Security.URLAllowlist.UpstreamHosts = []string{"original.example"} + probe, ok := NewCodexGatewayBorrowProbeUpstream(cfg).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + closeCodexBorrowProbeTestClients(t, probe.source, probe.target) + cfg.Gateway.MaxConnsPerHost = 1 + cfg.Security.URLAllowlist.UpstreamHosts[0] = "changed.example" + require.Equal(t, 2, probe.source.cfg.Gateway.MaxConnsPerHost) + require.Equal(t, "original.example", probe.target.cfg.Security.URLAllowlist.UpstreamHosts[0]) + + arrived := make(chan struct{}, 2) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + arrived <- struct{}{} + <-release + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + t.Cleanup(unblock) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + results := make(chan error, 2) + for range 2 { + req := codexBorrowProbeTestRequest(t, purpose, srv.URL) + req = req.WithContext(service.WithHTTPUpstreamProfile(ctx, purpose)) + go func() { + resp, err := probe.Do(req, "", 7, 1) + if err == nil { + err = resp.Body.Close() + } + results <- err + }() + } + for range 2 { + select { + case <-arrived: + case <-ctx.Done(): + unblock() + t.Fatal("probe requests inherited the account concurrency of one") + } + } + unblock() + for range 2 { + require.NoError(t, <-results) + } + }) + } +} + +func TestCodexGatewayBorrowProbeUpstream_TargetUsesFreshHTTP1Connections(t *testing.T) { + type observedRequest struct { + protocol int + remote string + close bool + cookie string + identity string + } + observed := make(chan observedRequest, 3) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + observed <- observedRequest{r.ProtoMajor, r.RemoteAddr, r.Close, r.Header.Get("Cookie"), r.Header.Get("User-Agent")} + http.SetCookie(w, &http.Cookie{Name: "__cflb", Value: "response-cookie"}) + _, _ = io.WriteString(w, "probe") + })) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + probe, ok := NewCodexGatewayBorrowProbeUpstream(codexBorrowProbeTestConfig(4)).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + closeCodexBorrowProbeTestClients(t, probe.source, probe.target) + roots := x509.NewCertPool() + roots.AddCert(srv.Certificate()) + for _, pair := range []struct { + manager *httpUpstreamService + profile service.HTTPUpstreamProfile + }{ + {probe.source, service.HTTPUpstreamProfileOpenAI}, + {probe.target, service.HTTPUpstreamProfileCodexBorrowTarget}, + } { + entry, err := pair.manager.getClientEntry("", 7, 0, pair.profile, false, true) + require.NoError(t, err) + require.Nil(t, entry.client.Jar) + transport, ok := entry.client.Transport.(*http.Transport) + require.True(t, ok) + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + transport.TLSClientConfig.RootCAs = roots + } + + for _, purpose := range []service.HTTPUpstreamProfile{ + service.HTTPUpstreamProfileCodexBorrowSource, + service.HTTPUpstreamProfileCodexBorrowTarget, + service.HTTPUpstreamProfileCodexBorrowTarget, + } { + req := codexBorrowProbeTestRequest(t, purpose, srv.URL) + req.Header.Set("Cookie", "__cflb=explicit-target-cookie") + req.Header.Set("User-Agent", "codex-probe-identity") + resp, err := probe.Do(req, "", 7, 1) + require.NoError(t, err) + _, err = io.Copy(io.Discard, resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.False(t, req.Close, "wrapper must not mutate the original request") + } + source, firstTarget, secondTarget := <-observed, <-observed, <-observed + require.Equal(t, 2, source.protocol, "source retains configured OpenAI protocol policy") + for _, target := range []observedRequest{firstTarget, secondTarget} { + require.Equal(t, 1, target.protocol) + require.True(t, target.close) + require.Equal(t, "__cflb=explicit-target-cookie", target.cookie) + require.Equal(t, "codex-probe-identity", target.identity) + } + require.NotEqual(t, firstTarget.remote, secondTarget.remote, "target attempts require new TCP connections") +} + +func TestCodexGatewayBorrowProbeUpstream_TLSProfileCacheAndTargetALPN(t *testing.T) { + probe, ok := NewCodexGatewayBorrowProbeUpstream(codexBorrowProbeTestConfig(8)).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + closeCodexBorrowProbeTestClients(t, probe.source, probe.target) + profile := &tlsfingerprint.Profile{ + Name: "bound-profile", + CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, + ALPNProtocols: []string{"h2", "http/1.1"}, + } + first, err := probe.source.getClientEntryWithTLS("", 7, 0, profile, service.HTTPUpstreamProfileOpenAI, false, true) + require.NoError(t, err) + second, err := probe.source.getClientEntryWithTLS("", 7, 0, cloneCodexBorrowTLSProfile(profile), service.HTTPUpstreamProfileOpenAI, false, true) + require.NoError(t, err) + require.Same(t, first, second) + changed := cloneCodexBorrowTLSProfile(profile) + changed.CipherSuites[0] = tls.TLS_AES_256_GCM_SHA384 + third, err := probe.source.getClientEntryWithTLS("", 7, 0, changed, service.HTTPUpstreamProfileOpenAI, false, true) + require.NoError(t, err) + require.NotSame(t, first, third, "same-name profiles with different TLS values require separate clients") + + targetEntry, err := probe.target.getClientEntryWithTLS("", 7, 0, profile, service.HTTPUpstreamProfileCodexBorrowTarget, false, true) + require.NoError(t, err) + require.NotSame(t, first.client, targetEntry.client) + targetTransport, ok := targetEntry.client.Transport.(*http.Transport) + require.True(t, ok) + require.True(t, targetTransport.DisableKeepAlives) + require.False(t, targetTransport.ForceAttemptHTTP2) + require.False(t, targetTransport.Protocols.HTTP2()) + require.True(t, targetTransport.Protocols.HTTP1()) + require.NotNil(t, targetTransport.DialTLSContext) + require.Equal(t, codexBorrowProbeHeaderTimeout, targetTransport.ResponseHeaderTimeout) + require.Contains(t, targetEntry.poolKey, upstreamProtocolModeCodexBorrowTargetH1) + require.Nil(t, targetEntry.client.Jar) + for key := range probe.target.clients { + require.Contains(t, key, "|profile:"+codexBorrowTLSProfileKey(profile)) + require.Contains(t, key, upstreamProtocolModeCodexBorrowTargetH1) + } + + // Read the actual ClientHello, then reject it before certificate verification. + // This checks the configured dialers without an external TLS/model request. + type hello struct { + protocols []string + ciphers []uint16 + } + hellos := make(chan hello, 2) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + tlsConn := tls.Server(conn, &tls.Config{GetConfigForClient: func(info *tls.ClientHelloInfo) (*tls.Config, error) { + hellos <- hello{slices.Clone(info.SupportedProtos), slices.Clone(info.CipherSuites)} + return nil, errors.New("local ClientHello capture complete") + }}) + _ = tlsConn.Handshake() + _ = conn.Close() + } + }() + for _, purpose := range []service.HTTPUpstreamProfile{service.HTTPUpstreamProfileCodexBorrowSource, service.HTTPUpstreamProfileCodexBorrowTarget} { + req := codexBorrowProbeTestRequest(t, purpose, "https://"+listener.Addr().String()) + resp, err := probe.DoWithTLS(req, "", 7, 1, profile) + require.Error(t, err, "local capture rejects the handshake after observing ClientHello") + require.Nil(t, resp) + } + readHello := func() hello { + select { + case captured := <-hellos: + return captured + case <-time.After(5 * time.Second): + t.Fatal("fingerprint request did not send a ClientHello") + return hello{} + } + } + sourceHello, targetHello := readHello(), readHello() + require.Equal(t, profile.ALPNProtocols, sourceHello.protocols) + require.Equal(t, []string{"http/1.1"}, targetHello.protocols) + require.Equal(t, profile.CipherSuites, sourceHello.ciphers) + require.Equal(t, profile.CipherSuites, targetHello.ciphers) + require.Equal(t, []string{"h2", "http/1.1"}, profile.ALPNProtocols, "target must not change the caller's profile") +} + +func TestCodexGatewayBorrowProbeUpstream_PreservesProxyAndURLPolicy(t *testing.T) { + for _, purpose := range []service.HTTPUpstreamProfile{service.HTTPUpstreamProfileCodexBorrowSource, service.HTTPUpstreamProfileCodexBorrowTarget} { + t.Run(string(purpose), func(t *testing.T) { + var calls atomic.Int64 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(proxy.Close) + probe, ok := NewCodexGatewayBorrowProbeUpstream(nil).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + closeCodexBorrowProbeTestClients(t, probe.source, probe.target) + req := codexBorrowProbeTestRequest(t, purpose, "http://upstream.invalid/borrow") + resp, err := probe.DoWithTLS(req, proxy.URL, 7, 1, &tlsfingerprint.Profile{Name: "unused-over-http"}) + require.NoError(t, err) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + require.Equal(t, int64(1), calls.Load()) + + cfg := codexBorrowProbeTestConfig(2) + cfg.Security.URLAllowlist.Enabled = true + protected := NewCodexGatewayBorrowProbeUpstream(cfg) + req = codexBorrowProbeTestRequest(t, purpose, "https://127.0.0.1/borrow") + _, err = protected.Do(req, proxy.URL, 7, 1) + require.Error(t, err) + require.Equal(t, int64(1), calls.Load(), "blocked private URL must not reach the proxy") + }) + } + + probe, ok := NewCodexGatewayBorrowProbeUpstream(nil).(*codexGatewayBorrowProbeUpstream) + require.True(t, ok) + proxyURL, err := url.Parse("https://proxy.example:8443") + require.NoError(t, err) + transport, err := probe.target.buildCodexBorrowTLSFingerprintTransport(poolSettings{}, proxyURL, &tlsfingerprint.Profile{Name: "bound"}) + require.NoError(t, err) + // Preserve the existing HTTPS-proxy fingerprint fallback, still with H1 and + // no connection reuse for the target, rather than bypassing the proxy. + require.NotNil(t, transport.Proxy) + require.Nil(t, transport.DialTLSContext) + require.True(t, transport.DisableKeepAlives) + require.False(t, transport.Protocols.HTTP2()) + resolved, err := transport.Proxy(&http.Request{URL: &url.URL{Scheme: "https", Host: "upstream.example"}}) + require.NoError(t, err) + require.Equal(t, proxyURL.String(), resolved.String()) +} diff --git a/backend/internal/repository/http_upstream.go b/backend/internal/repository/http_upstream.go index 8a55f03fa230..757607d97fc5 100644 --- a/backend/internal/repository/http_upstream.go +++ b/backend/internal/repository/http_upstream.go @@ -167,6 +167,9 @@ type httpUpstreamService struct { clients map[string]*upstreamClientEntry // 客户端缓存池，key 由隔离策略决定 // OpenAI 走 HTTP/HTTPS 代理时的 H2->H1 回退状态（key=标准化 proxyKey） openAIHTTP2Fallbacks sync.Map + // Set only on the dedicated borrow probe managers; ordinary gateway pools + // keep their existing keys, TLS policy and eviction budget. + codexBorrowPurpose service.HTTPUpstreamProfile } // NewHTTPUpstream 创建通用 HTTP 上游服务 @@ -562,9 +565,19 @@ func (s *httpUpstreamService) getClientEntryWithTLS(proxyURL string, accountID i } settings := s.resolvePoolSettings(isolation, accountConcurrency) settings = s.applyProfilePoolSettings(settings, upstreamProfile) + protocolMode := upstreamProtocolModeDefault + if s.codexBorrowPurpose == service.HTTPUpstreamProfileCodexBorrowTarget { + protocolMode = upstreamProtocolModeCodexBorrowTargetH1 + } // TLS 指纹客户端使用独立的缓存键，加 "tls:" 前缀 - cacheKey := "tls:" + buildCacheKey(isolation, proxyKey, accountID, upstreamProtocolModeDefault) - poolKey := buildPoolKey(settings, upstreamProtocolModeDefault) + ":tls" + cacheKey := "tls:" + buildCacheKey(isolation, proxyKey, accountID, protocolMode) + poolKey := buildPoolKey(settings, protocolMode) + ":tls" + if s.codexBorrowPurpose != service.HTTPUpstreamProfileDefault { + profile = cloneCodexBorrowTLSProfile(profile) + profileKey := "|profile:" + codexBorrowTLSProfileKey(profile) + cacheKey = s.codexBorrowClientCacheKey(cacheKey + profileKey) + poolKey += profileKey + } now := time.Now() nowUnix := now.UnixNano() @@ -615,7 +628,7 @@ func (s *httpUpstreamService) getClientEntryWithTLS(proxyURL string, accountID i // 创建带 TLS 指纹的 Transport slog.Debug("tls_fingerprint_creating_new_client", "account_id", accountID, "cache_key", cacheKey, "proxy", proxyKey) - transport, err := buildUpstreamTransportWithTLSFingerprint(settings, parsedProxy, profile) + transport, err := s.buildCodexBorrowTLSFingerprintTransport(settings, parsedProxy, profile) if err != nil { s.mu.Unlock() return nil, fmt.Errorf("build TLS fingerprint transport: %w", err) @@ -631,6 +644,9 @@ func (s *httpUpstreamService) getClientEntryWithTLS(proxyURL string, accountID i proxyKey: proxyKey, poolKey: poolKey, } + if s.codexBorrowPurpose != service.HTTPUpstreamProfileDefault { + entry.protocolMode = protocolMode + } atomic.StoreInt64(&entry.lastUsed, nowUnix) if markInFlight { atomic.StoreInt64(&entry.inFlight, 1) @@ -726,7 +742,7 @@ func (s *httpUpstreamService) getClientEntry(proxyURL string, accountID int64, a settings := s.resolvePoolSettings(isolation, accountConcurrency) settings = s.applyProfilePoolSettings(settings, profile) // 构建缓存键（根据隔离策略不同） - cacheKey := buildCacheKey(isolation, proxyKey, accountID, protocolMode) + cacheKey := s.codexBorrowClientCacheKey(buildCacheKey(isolation, proxyKey, accountID, protocolMode)) // 构建连接池配置键（用于检测配置变更） poolKey := buildPoolKey(settings, protocolMode) @@ -970,6 +986,8 @@ func (s *httpUpstreamService) resolvePoolSettings(isolation string, accountConcu func (s *httpUpstreamService) applyProfilePoolSettings(settings poolSettings, profile service.HTTPUpstreamProfile) poolSettings { switch profile { + case service.HTTPUpstreamProfileCodexBorrowTarget: + settings.responseHeaderTimeout = codexBorrowProbeHeaderTimeout case service.HTTPUpstreamProfileOpenAI: settings.responseHeaderTimeout = 0 if s != nil && s.cfg != nil && s.cfg.Gateway.OpenAIResponseHeaderTimeout > 0 { @@ -1061,6 +1079,9 @@ func (s *httpUpstreamService) resolveOpenAIHTTP2Settings() openAIHTTP2Settings { } func (s *httpUpstreamService) resolveProtocolMode(profile service.HTTPUpstreamProfile, proxyKey string, parsedProxy *url.URL) string { + if profile == service.HTTPUpstreamProfileCodexBorrowTarget { + return upstreamProtocolModeCodexBorrowTargetH1 + } if profile == service.HTTPUpstreamProfileLongStream { return upstreamProtocolModeLongStreamH2 } @@ -1395,6 +1416,8 @@ func buildUpstreamTransport(settings poolSettings, proxyURL *url.URL, protocolMo transport.DisableCompression = true } switch protocolMode { + case upstreamProtocolModeCodexBorrowTargetH1: + configureCodexBorrowTargetTransport(transport) case upstreamProtocolModeLongStreamH2, upstreamProtocolModeOpenAIH2: transport.ForceAttemptHTTP2 = true // 显式配置 http2 并启用 PING 健康探测，剔除代理/NAT 静默掐断的死连接， diff --git a/backend/internal/repository/wire.go b/backend/internal/repository/wire.go index 0065c61f3c50..6e3aee88e638 100644 --- a/backend/internal/repository/wire.go +++ b/backend/internal/repository/wire.go @@ -75,6 +75,7 @@ var ProviderSet = wire.NewSet( NewAdminAccountRepository, NewScheduledTestPlanRepository, // 定时测试计划仓储 NewScheduledTestResultRepository, // 定时测试结果仓储 + NewCodexGatewayBorrowTestRepository, NewProxyRepository, NewRedeemCodeRepository, NewPromoCodeRepository, @@ -164,6 +165,7 @@ var ProviderSet = wire.NewSet( NewClaudeUsageFetcher, NewClaudeOAuthClient, NewHTTPUpstream, + NewCodexGatewayBorrowProbeUpstream, NewOpenAIOAuthClient, NewOpenAIReferralClient, NewGrokOAuthClient, diff --git a/backend/internal/server/routes/admin.go b/backend/internal/server/routes/admin.go index 8c1e0688458a..1dd14476a020 100644 --- a/backend/internal/server/routes/admin.go +++ b/backend/internal/server/routes/admin.go @@ -22,6 +22,7 @@ func RegisterAdminRoutes( ) { // 插件 UI 使用短时能力 URL，仅提供经过安装校验的静态资源。 v1.GET("/plugin-ui/:token/*path", h.Admin.Plugin.ServeUIAsset) + v1.GET("/codex-gateway-borrow/preview/:cap/index.html", h.Admin.CodexGatewayBorrow.ServePreview) admin := v1.Group("/admin") admin.Use(gin.HandlerFunc(adminAuth)) @@ -131,6 +132,7 @@ func RegisterAdminRoutes( // 推理恢复全局开关 registerReasoningRecoveryRoutes(admin, h) + registerCodexGatewayBorrowRoutes(admin, h) // 邀请返利（专属用户管理） registerAffiliateRoutes(admin, h) @@ -140,6 +142,18 @@ func RegisterAdminRoutes( } } +func registerCodexGatewayBorrowRoutes(admin *gin.RouterGroup, h *handler.Handlers) { + borrow := admin.Group("/codex-gateway-borrow") + borrow.GET("/config", h.Admin.CodexGatewayBorrow.GetConfig) + borrow.PUT("/config", h.Admin.CodexGatewayBorrow.SaveConfig) + borrow.GET("/status", h.Admin.CodexGatewayBorrow.Status) + borrow.POST("/prepare", h.Admin.CodexGatewayBorrow.Prepare) + borrow.POST("/verify", h.Admin.CodexGatewayBorrow.Verify) + borrow.POST("/tests", h.Admin.CodexGatewayBorrow.StartTests) + borrow.GET("/tests", h.Admin.CodexGatewayBorrow.ListTests) + borrow.GET("/tests/:id", h.Admin.CodexGatewayBorrow.GetTest) +} + func registerAccountJobRoutes(admin *gin.RouterGroup, h *handler.Handlers) { jobs := admin.Group("/account-jobs") { diff --git a/backend/internal/service/codex_gateway_borrow.go b/backend/internal/service/codex_gateway_borrow.go new file mode 100644 index 000000000000..7ac331caadb0 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow.go @@ -0,0 +1,815 @@ +package service + +// Cookie scope, the 230-second lease and two-shot STATE comparison are adapted +// from ranxi2001/sub2api at 5ca3cca21eeaf4ca8a694a7f2f8f0ecd9575c549: +// backend/internal/repository/{codex_gateway_pin,astra_target_validation}.go +// and backend/internal/service/openai_codex_state_probe.go (LGPL-3.0). +// This native implementation owns no exit routing, account health or quota. + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "slices" + "strings" + "sync" + "time" + + infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "golang.org/x/sync/singleflight" +) + +const SettingKeyCodexGatewayBorrowConfig = "codex_gateway_borrow_config" + +const ( + codexGatewayBorrowTTL = 230 * time.Second + codexGatewayBorrowTimeout = 90 * time.Second + codexGatewayBorrowShotTimeout = 45 * time.Second + codexGatewayBorrowFailureWait = 15 * time.Second + codexGatewayBorrowStoreTimeout = 5 * time.Second + codexGatewayBorrowSourceModel = "gpt-6-astra" +) + +var ( + ErrCodexGatewayBorrowUnavailable = infraerrors.ServiceUnavailable("CODEX_GATEWAY_BORROW_UNAVAILABLE", "no qualified Codex gateway borrow route is available") + ErrCodexGatewayBorrowBusy = infraerrors.Conflict("CODEX_GATEWAY_BORROW_BUSY", "target validation is already in progress") + ErrCodexGatewayBorrowChanged = infraerrors.Conflict("CODEX_GATEWAY_BORROW_CHANGED", "gateway borrow configuration or route changed") +) + +// A local route qualification failure is distinct from an actual business +// request failing upstream. Callers must surface it before ordinary account +// failover, stream-timeout or health hooks. Unwrap preserves the specific cause. +type CodexGatewayBorrowFailure struct{ Cause error } + +func (e *CodexGatewayBorrowFailure) Error() string { + if e == nil || e.Cause == nil { + return "Codex gateway borrow qualification failed" + } + return "Codex gateway borrow qualification failed: " + e.Cause.Error() +} +func (e *CodexGatewayBorrowFailure) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} +func IsCodexGatewayBorrowFailure(err error) bool { + var failure *CodexGatewayBorrowFailure + return errors.As(err, &failure) +} + +// CodexGatewayBorrowProbeUpstream is deliberately a distinct injection type. +// Its implementation must own separate clients, pools and TLS caches, including +// separation between source acquisition and target observation. +type CodexGatewayBorrowProbeUpstream interface{ HTTPUpstream } + +type CodexGatewayBorrowConfig struct { + Enabled bool `json:"enabled"` + SourceAccountIDs []int64 `json:"source_account_ids"` + TargetAccountIDs []int64 `json:"target_account_ids"` + Models []string `json:"models"` +} + +func DefaultCodexGatewayBorrowConfig() CodexGatewayBorrowConfig { + return CodexGatewayBorrowConfig{SourceAccountIDs: []int64{}, TargetAccountIDs: []int64{}, Models: []string{"gpt-6-astra", "gpt-6.1-sol"}} +} + +func DecodeCodexGatewayBorrowConfig(raw []byte) (CodexGatewayBorrowConfig, error) { + cfg := DefaultCodexGatewayBorrowConfig() + if trimmed := bytes.TrimSpace(raw); len(trimmed) == 0 || trimmed[0] != '{' { + return cfg, errors.New("configuration must be a JSON object") + } + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(&cfg); err != nil { + return cfg, err + } + var extra any + if err := d.Decode(&extra); err != io.EOF { + return cfg, errors.New("configuration must contain one JSON object") + } + return normalizeCodexGatewayBorrowConfig(cfg) +} + +func normalizeCodexGatewayBorrowConfig(cfg CodexGatewayBorrowConfig) (CodexGatewayBorrowConfig, error) { + if cfg.SourceAccountIDs == nil { + cfg.SourceAccountIDs = []int64{} + } + if cfg.TargetAccountIDs == nil { + cfg.TargetAccountIDs = []int64{} + } + if cfg.Models == nil { + cfg.Models = DefaultCodexGatewayBorrowConfig().Models + } + seen := map[int64]bool{} + for _, ids := range [][]int64{cfg.SourceAccountIDs, cfg.TargetAccountIDs} { + for _, id := range ids { + if id <= 0 || seen[id] { + return cfg, errors.New("source and target account IDs must be positive, unique and disjoint") + } + seen[id] = true + } + } + models := map[string]bool{} + for _, model := range cfg.Models { + if (model != "gpt-6-astra" && model != "gpt-6.1-sol") || models[model] { + return cfg, errors.New("models must be a unique selection of gpt-6-astra and gpt-6.1-sol") + } + models[model] = true + } + if cfg.Enabled && (len(cfg.SourceAccountIDs) == 0 || len(cfg.TargetAccountIDs) == 0 || len(cfg.Models) == 0) { + return cfg, errors.New("enabled gateway borrowing requires sources, targets and models") + } + return cloneCodexGatewayBorrowConfig(cfg), nil +} + +func cloneCodexGatewayBorrowConfig(cfg CodexGatewayBorrowConfig) CodexGatewayBorrowConfig { + cfg.SourceAccountIDs = append([]int64{}, cfg.SourceAccountIDs...) + cfg.TargetAccountIDs = append([]int64{}, cfg.TargetAccountIDs...) + cfg.Models = append([]string{}, cfg.Models...) + return cfg +} + +type CodexGatewayBorrowCandidateStatus struct { + SourceAccountID int64 `json:"source_account_id"` + ExpiresAt time.Time `json:"expires_at"` + RemainingSeconds int64 `json:"remaining_seconds"` + CookieFingerprint string `json:"cookie_fingerprint"` +} + +type CodexGatewayBorrowSourceStatus struct { + AccountID int64 `json:"account_id"` + State string `json:"state"` + Reason string `json:"reason"` + CheckedAt *time.Time `json:"checked_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + RemainingSeconds int64 `json:"remaining_seconds"` + Error string `json:"error,omitempty"` +} + +type CodexGatewayBorrowVerification struct { + AccountID int64 `json:"account_id"` + Model string `json:"model"` + Success bool `json:"success"` + Reason string `json:"reason"` + Error string `json:"error,omitempty"` + MintStatus int `json:"mint_status"` + ContinueStatus int `json:"continue_status"` + Minted bool `json:"minted"` + NewTicket bool `json:"new_ticket"` + ReportedModel string `json:"reported_model,omitempty"` + CheckedAt time.Time `json:"checked_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +type CodexGatewayBorrowTargetStatus struct { + CodexGatewayBorrowVerification + State string `json:"state"` + CacheValid bool `json:"cache_valid"` + RemainingSeconds int64 `json:"remaining_seconds"` + RetryAfter *time.Time `json:"retry_after,omitempty"` +} + +type CodexGatewayBorrowStatus struct { + Enabled bool `json:"enabled"` + Revision uint64 `json:"revision"` + GeneratedAt time.Time `json:"generated_at"` + Preparing bool `json:"preparing"` + Config CodexGatewayBorrowConfig `json:"config"` + Candidate *CodexGatewayBorrowCandidateStatus `json:"candidate,omitempty"` + Sources []CodexGatewayBorrowSourceStatus `json:"sources"` + Targets []CodexGatewayBorrowTargetStatus `json:"targets"` + ModelEfforts map[string][]string `json:"model_efforts"` +} + +// Application contains only opaque identities. The selected cookie itself is +// available solely in the cloned outbound request's Cookie header. +type CodexGatewayBorrowApplication struct { + Applied bool + SourceAccountID int64 + ExpiresAt time.Time + Fingerprint string + CookieFingerprint string +} + +type codexGatewayBorrowCandidate struct { + cookie http.Cookie + expires time.Time + sourceID int64 +} + +type codexGatewayBorrowTargetKey struct { + accountID int64 + model string +} +type codexGatewayBorrowTargetCheck struct { + key string + cookieKey string + expires time.Time + retryAfter time.Time + validating bool + result CodexGatewayBorrowVerification +} + +type CodexGatewayBorrowService struct { + settings SettingRepository + accounts AccountRepository + gateway *OpenAIGatewayService + upstream CodexGatewayBorrowProbeUpstream + // A profile supplied by a business caller is reused verbatim. Current Codex + // HTTP and WS business paths use standard TLS, so manual preparation uses nil. + tlsProfiles *TLSFingerprintProfileService + storeMu sync.Mutex + mu sync.Mutex + config CodexGatewayBorrowConfig + loaded bool + revision uint64 + revisionCtx context.Context + cancel context.CancelFunc + candidate *codexGatewayBorrowCandidate + sources map[int64]CodexGatewayBorrowSourceStatus + targets map[codexGatewayBorrowTargetKey]codexGatewayBorrowTargetCheck + preparing bool + prepareRuns int + prepareFailedUntil time.Time + prepareFailure string + sourceFlight singleflight.Group + targetProbeMu sync.Mutex + wsAnchors codexGatewayBorrowWSAnchorStore +} + +func NewCodexGatewayBorrowService(settings SettingRepository, accounts AccountRepository, gateway *OpenAIGatewayService, upstream CodexGatewayBorrowProbeUpstream, tlsProfiles *TLSFingerprintProfileService) *CodexGatewayBorrowService { + ctx, cancel := context.WithCancel(context.Background()) + s := &CodexGatewayBorrowService{settings: settings, accounts: accounts, gateway: gateway, upstream: upstream, tlsProfiles: tlsProfiles, + config: DefaultCodexGatewayBorrowConfig(), revisionCtx: ctx, cancel: cancel, + sources: map[int64]CodexGatewayBorrowSourceStatus{}, targets: map[codexGatewayBorrowTargetKey]codexGatewayBorrowTargetCheck{}} + if gateway != nil { + gateway.SetCodexGatewayBorrowService(s) + } + return s +} + +func ProvideCodexGatewayBorrowService(settings SettingRepository, accounts AccountRepository, gateway *OpenAIGatewayService, upstream CodexGatewayBorrowProbeUpstream, tlsProfiles *TLSFingerprintProfileService) (*CodexGatewayBorrowService, error) { + s := NewCodexGatewayBorrowService(settings, accounts, gateway, upstream, tlsProfiles) + ctx, cancel := context.WithTimeout(context.Background(), codexGatewayBorrowStoreTimeout) + defer cancel() + if _, err := s.GetConfig(ctx); err != nil { + return nil, err + } + return s, nil +} + +func (s *OpenAIGatewayService) SetCodexGatewayBorrowService(borrow *CodexGatewayBorrowService) { + s.gatewayBorrow = borrow +} + +func (s *CodexGatewayBorrowService) ConfigSnapshot() CodexGatewayBorrowConfig { + if s == nil { + return DefaultCodexGatewayBorrowConfig() + } + s.mu.Lock() + defer s.mu.Unlock() + return cloneCodexGatewayBorrowConfig(s.config) +} + +func (s *CodexGatewayBorrowService) GetConfig(ctx context.Context) (CodexGatewayBorrowConfig, error) { + if s == nil || s.settings == nil { + return DefaultCodexGatewayBorrowConfig(), ErrCodexGatewayBorrowUnavailable + } + s.storeMu.Lock() + defer s.storeMu.Unlock() + dbCtx, cancel := context.WithTimeout(ctx, codexGatewayBorrowStoreTimeout) + defer cancel() + raw, err := s.settings.GetValue(dbCtx, SettingKeyCodexGatewayBorrowConfig) + cfg := DefaultCodexGatewayBorrowConfig() + if !errors.Is(err, ErrSettingNotFound) { + if err != nil { + return cfg, err + } + cfg, err = DecodeCodexGatewayBorrowConfig([]byte(raw)) + if err != nil { + return cfg, fmt.Errorf("stored gateway borrow configuration: %w", err) + } + } + s.publishConfig(cfg, false) + return cfg, nil +} + +func (s *CodexGatewayBorrowService) SaveConfig(ctx context.Context, cfg CodexGatewayBorrowConfig) (CodexGatewayBorrowConfig, error) { + if s == nil || s.settings == nil || s.accounts == nil { + return cfg, ErrCodexGatewayBorrowUnavailable + } + cfg, err := normalizeCodexGatewayBorrowConfig(cfg) + if err != nil { + return cfg, err + } + // Validation only loads local rows and resolves existing parent credentials. + // It never discovers models, touches quota or changes any account fields. + for _, id := range append(append([]int64{}, cfg.SourceAccountIDs...), cfg.TargetAccountIDs...) { + a, err := s.accounts.GetByID(ctx, id) + if err != nil || !codexGatewayBorrowAccountSupported(a) { + return cfg, fmt.Errorf("account %d must be an existing OpenAI OAuth-like account", id) + } + if _, err := resolveCredentialAccount(ctx, s.accounts, a); err != nil { + return cfg, fmt.Errorf("account %d has no valid credential parent", id) + } + } + raw, err := json.Marshal(cfg) + if err != nil { + return cfg, err + } + s.storeMu.Lock() + defer s.storeMu.Unlock() + dbCtx, cancel := context.WithTimeout(ctx, codexGatewayBorrowStoreTimeout) + defer cancel() + if err := s.settings.Set(dbCtx, SettingKeyCodexGatewayBorrowConfig, string(raw)); err != nil { + return cfg, err + } + s.publishConfig(cfg, true) + return cfg, nil +} + +func (s *CodexGatewayBorrowService) publishConfig(cfg CodexGatewayBorrowConfig, saved bool) { + s.mu.Lock() + changed := !s.loaded || !slices.Equal(cfg.SourceAccountIDs, s.config.SourceAccountIDs) || !slices.Equal(cfg.TargetAccountIDs, s.config.TargetAccountIDs) || !slices.Equal(cfg.Models, s.config.Models) || cfg.Enabled != s.config.Enabled + if !changed && !saved { + s.mu.Unlock() + return + } + s.cancel() + s.revisionCtx, s.cancel = context.WithCancel(context.Background()) + s.revision++ + s.config, s.loaded = cloneCodexGatewayBorrowConfig(cfg), true + s.candidate = nil + s.sources = map[int64]CodexGatewayBorrowSourceStatus{} + s.targets = map[codexGatewayBorrowTargetKey]codexGatewayBorrowTargetCheck{} + s.prepareFailedUntil, s.prepareFailure, s.preparing, s.prepareRuns = time.Time{}, "", false, 0 + ctx := s.revisionCtx + s.mu.Unlock() + // Exactly one finite preparation per save. Loading settings or restarting + // never calls a model. Individual source/target deadlines remain 90 seconds. + if saved && cfg.Enabled { + go func() { + prepareCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + _, _ = s.Prepare(prepareCtx) + }() + } +} + +func (s *CodexGatewayBorrowService) Stop() { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.cancel() +} + +func codexGatewayBorrowAccountSupported(a *Account) bool { + return a != nil && a.IsOpenAIOAuthLike() && !a.IsOpenAIAgentIdentity() && !a.IsSyntheticUITest() +} + +func CodexGatewayBorrowRequestEligible(req *http.Request) bool { + return req != nil && req.URL != nil && req.Method == http.MethodPost && req.URL.Scheme == "https" && req.URL.Host == "chatgpt.com" && req.URL.User == nil && + (req.Host == "" || req.Host == "chatgpt.com") && (req.URL.Path == "/backend-api/codex/responses" || req.URL.Path == "/backend-api/codex/responses/lite") +} + +func (s *CodexGatewayBorrowService) RequestEligible(req *http.Request) bool { + return CodexGatewayBorrowRequestEligible(req) +} +func (s *CodexGatewayBorrowService) ModelEligible(model string) bool { + cfg := s.ConfigSnapshot() + return cfg.Enabled && slices.Contains(cfg.Models, model) +} + +type codexGatewayBorrowModelContextKey struct{} +type codexGatewayBorrowObservationContextKey struct{} + +// The token provider may refresh credentials normally, but must not attach an +// observation failure to account health, scheduling or quota. +func WithCodexGatewayBorrowObservation(ctx context.Context) context.Context { + return context.WithValue(ctx, codexGatewayBorrowObservationContextKey{}, true) +} +func IsCodexGatewayBorrowObservation(ctx context.Context) bool { + return ctx != nil && ctx.Value(codexGatewayBorrowObservationContextKey{}) == true +} + +// Builders stage the model they actually resolved. This gate does not inspect, +// consume, copy or limit the business body (which can be the full 100 MiB). +func WithCodexGatewayBorrowModel(ctx context.Context, model string) context.Context { + return context.WithValue(ctx, codexGatewayBorrowModelContextKey{}, model) +} +func CodexGatewayBorrowModelFromContext(ctx context.Context) string { + if ctx == nil { + return "" + } + model, _ := ctx.Value(codexGatewayBorrowModelContextKey{}).(string) + return model +} + +func (s *CodexGatewayBorrowService) Status() CodexGatewayBorrowStatus { + if s == nil { + return CodexGatewayBorrowStatus{Config: DefaultCodexGatewayBorrowConfig(), Sources: []CodexGatewayBorrowSourceStatus{}, Targets: []CodexGatewayBorrowTargetStatus{}, ModelEfforts: codexGatewayBorrowModelEfforts()} + } + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now() + out := CodexGatewayBorrowStatus{Enabled: s.config.Enabled, Revision: s.revision, GeneratedAt: now, Preparing: s.preparing || s.prepareRuns > 0, + Config: cloneCodexGatewayBorrowConfig(s.config), Sources: []CodexGatewayBorrowSourceStatus{}, Targets: []CodexGatewayBorrowTargetStatus{}, ModelEfforts: codexGatewayBorrowModelEfforts()} + if c := s.candidate; c != nil { + out.Candidate = &CodexGatewayBorrowCandidateStatus{SourceAccountID: c.sourceID, ExpiresAt: c.expires, RemainingSeconds: borrowRemaining(c.expires, now), CookieFingerprint: borrowHash(c.cookie.Value)} + } + for _, id := range s.config.SourceAccountIDs { + row, ok := s.sources[id] + if !ok { + row = CodexGatewayBorrowSourceStatus{AccountID: id, State: "waiting", Reason: "not_prepared"} + } + if !s.config.Enabled { + row.State, row.Reason = "disabled", "disabled" + } + if row.ExpiresAt != nil { + row.RemainingSeconds = borrowRemaining(*row.ExpiresAt, now) + if row.State == "ready" && row.RemainingSeconds == 0 { + row.State, row.Reason = "expired", "route_expired" + } + } + out.Sources = append(out.Sources, row) + } + for _, id := range s.config.TargetAccountIDs { + for _, model := range s.config.Models { + row := CodexGatewayBorrowTargetStatus{CodexGatewayBorrowVerification: CodexGatewayBorrowVerification{AccountID: id, Model: model, Reason: "not_verified"}, State: "waiting"} + if c, ok := s.targets[codexGatewayBorrowTargetKey{id, model}]; ok { + row.CodexGatewayBorrowVerification = c.result + row.CacheValid = c.result.Success && s.candidate != nil && c.cookieKey == borrowHash(s.candidate.cookie.Value) && now.Before(c.expires) && now.Before(s.candidate.expires) + expires := c.expires + if s.candidate != nil && c.cookieKey == borrowHash(s.candidate.cookie.Value) && s.candidate.expires.Before(expires) { + expires = s.candidate.expires + } + if !expires.IsZero() { + row.ExpiresAt = &expires + } + row.RemainingSeconds = borrowRemaining(expires, now) + row.State = "rejected" + if row.CacheValid { + row.State = "ready" + } else if c.validating { + row.State = "validating" + } else if c.result.Success { + row.State, row.Reason = "expired", "route_expired" + } + if now.Before(c.retryAfter) && !c.result.Success { + retry := c.retryAfter + row.RetryAfter = &retry + } + } + if !s.config.Enabled { + row.State, row.Reason, row.CacheValid = "disabled", "disabled", false + } + out.Targets = append(out.Targets, row) + } + } + return out +} + +func borrowRemaining(expires, now time.Time) int64 { + remaining := int64(expires.Sub(now).Seconds()) + if remaining < 0 { + return 0 + } + return remaining +} +func borrowHash(value string) string { + h := sha256.Sum256([]byte(value)) + return hex.EncodeToString(h[:]) +} + +func borrowRevisionContext(parent, revision context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(parent, timeout) + stop := context.AfterFunc(revision, cancel) + return ctx, func() { stop(); cancel() } +} + +func (s *CodexGatewayBorrowService) Prepare(ctx context.Context) (status CodexGatewayBorrowStatus, err error) { + if s == nil { + return s.Status(), ErrCodexGatewayBorrowUnavailable + } + s.mu.Lock() + cfg, rev, revisionCtx := cloneCodexGatewayBorrowConfig(s.config), s.revision, s.revisionCtx + s.mu.Unlock() + if !cfg.Enabled { + return s.Status(), ErrCodexGatewayBorrowUnavailable + } + s.mu.Lock() + if s.revision != rev { + s.mu.Unlock() + return s.Status(), ErrCodexGatewayBorrowChanged + } + s.prepareRuns++ + s.mu.Unlock() + defer func() { + s.mu.Lock() + if s.revision == rev { + s.prepareRuns-- + } + s.mu.Unlock() + status = s.Status() + }() + if err := s.prepareSource(ctx, rev, revisionCtx); err != nil { + return s.Status(), err + } + for _, id := range cfg.TargetAccountIDs { + for _, model := range cfg.Models { + if revisionCtx.Err() != nil { + return s.Status(), ErrCodexGatewayBorrowChanged + } + a, req, proxy, err := s.accountTemplate(ctx, id, model) + if err != nil { + return s.Status(), err + } + if _, _, err := s.Apply(req, a, model, proxy, nil, false); err != nil { + return s.Status(), err + } + } + } + return s.Status(), nil +} + +func (s *CodexGatewayBorrowService) prepareSource(ctx context.Context, rev uint64, revisionCtx context.Context) error { + if s == nil { + return ErrCodexGatewayBorrowUnavailable + } + result := s.sourceFlight.DoChan(fmt.Sprint(rev), func() (any, error) { + s.mu.Lock() + if s.revision != rev || !s.config.Enabled { + s.mu.Unlock() + return nil, ErrCodexGatewayBorrowChanged + } + if s.candidate != nil && time.Until(s.candidate.expires) > codexGatewayBorrowTimeout { + s.mu.Unlock() + return nil, nil + } + if time.Now().Before(s.prepareFailedUntil) { + detail := s.prepareFailure + s.mu.Unlock() + if detail != "" { + return nil, errors.New(detail) + } + return nil, ErrCodexGatewayBorrowUnavailable + } + ids := append([]int64{}, s.config.SourceAccountIDs...) + s.preparing = true + s.mu.Unlock() + probeCtx, cancel := borrowRevisionContext(ctx, revisionCtx, codexGatewayBorrowTimeout) + defer cancel() + var lastErr error = ErrCodexGatewayBorrowUnavailable + for _, id := range ids { + if probeCtx.Err() != nil { + lastErr = probeCtx.Err() + break + } + candidate, detail, err := s.acquireSource(probeCtx, id) + now := time.Now() + s.mu.Lock() + if s.revision != rev || revisionCtx.Err() != nil { + s.mu.Unlock() + return nil, ErrCodexGatewayBorrowChanged + } + row := CodexGatewayBorrowSourceStatus{AccountID: id, State: "rejected", Reason: "source_probe_failed", CheckedAt: &now, Error: detail} + if err == nil && candidate != nil && now.Before(candidate.expires) { + // Seeing the same value while its original lease is live never + // renews it, even when a different source returns that value. + if old := s.candidate; old != nil && now.Before(old.expires) && old.cookie.Value == candidate.cookie.Value && old.expires.Before(candidate.expires) { + candidate.expires = old.expires + } + if s.candidate == nil || s.candidate.cookie.Value != candidate.cookie.Value || s.candidate.cookie.Path != candidate.cookie.Path { + s.targets = map[codexGatewayBorrowTargetKey]codexGatewayBorrowTargetCheck{} + } + s.candidate = candidate + row.State, row.Reason, row.Error, row.ExpiresAt = "ready", "source_qualified", "", &candidate.expires + s.sources[id] = row + s.preparing = false + s.prepareFailedUntil, s.prepareFailure = time.Time{}, "" + s.mu.Unlock() + return nil, nil + } + if err != nil { + lastErr = err + if detail != "" { + lastErr = errors.New(detail) + } + } + s.sources[id] = row + s.mu.Unlock() + } + s.mu.Lock() + if s.revision == rev { + s.preparing = false + s.prepareFailedUntil = time.Now().Add(codexGatewayBorrowFailureWait) + s.prepareFailure = lastErr.Error() + } + s.mu.Unlock() + return nil, lastErr + }) + select { + case <-ctx.Done(): + return ctx.Err() + case done := <-result: + return done.Err + } +} + +func borrowRequestFingerprint(req *http.Request, model, proxy string, profile *tlsfingerprint.Profile, cookie string) string { + tlsJSON, _ := json.Marshal(profile) + identity, _ := json.Marshal([]string{cookie, model, proxy, req.Header.Get("Authorization"), req.Header.Get("ChatGPT-Account-ID"), req.Header.Get("User-Agent"), req.Header.Get("Originator"), req.Header.Get("Version"), req.Header.Get("X-Codex-Turn-State"), string(tlsJSON)}) + return borrowHash(string(identity)) +} + +// Apply changes only __oailb on a request clone. The target's auth, STATE, +// __cflb, other cookies, body and proxy stay exactly as supplied by its caller. +// cachedOnly is used by manual generation: it never acquires or validates. +func (s *CodexGatewayBorrowService) Apply(req *http.Request, account *Account, model, proxy string, profile *tlsfingerprint.Profile, cachedOnly bool) (*http.Request, *CodexGatewayBorrowApplication, error) { + wire, application, err := s.apply(req, account, model, proxy, profile, cachedOnly, false) + if err != nil && !IsCodexGatewayBorrowFailure(err) { + err = &CodexGatewayBorrowFailure{Cause: err} + } + return wire, application, err +} + +func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, model, proxy string, profile *tlsfingerprint.Profile, cachedOnly, force bool) (*http.Request, *CodexGatewayBorrowApplication, error) { + if s == nil || !CodexGatewayBorrowRequestEligible(req) || account == nil { + return req, nil, nil + } + s.mu.Lock() + if !s.config.Enabled || !slices.Contains(s.config.Models, model) || !slices.Contains(s.config.TargetAccountIDs, account.ID) { + s.mu.Unlock() + return req, nil, nil + } + rev, revisionCtx := s.revision, s.revisionCtx + candidate := s.candidate + s.mu.Unlock() + if !codexGatewayBorrowAccountSupported(account) { + return req, nil, ErrCodexGatewayBorrowUnavailable + } + if candidate == nil || !time.Now().Before(candidate.expires) { + if cachedOnly { + return req, nil, ErrCodexGatewayBorrowUnavailable + } + if err := s.prepareSource(req.Context(), rev, revisionCtx); err != nil { + return req, nil, err + } + s.mu.Lock() + candidate = s.candidate + s.mu.Unlock() + } + if candidate == nil || !time.Now().Before(candidate.expires) || !borrowCookiePathMatches(req.URL.Path, candidate.cookie.Path) { + return req, nil, ErrCodexGatewayBorrowUnavailable + } + key := borrowRequestFingerprint(req, model, proxy, profile, candidate.cookie.Value) + targetKey := codexGatewayBorrowTargetKey{account.ID, model} + // Cache hits must precede the global validation lock. One cold target must + // not make another already-qualified target fail with "busy". + if !force { + if check, ok, err := s.cachedTarget(rev, targetKey, key, candidate); ok || err != nil { + if err != nil { + return req, nil, err + } + return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires), nil + } + } + if cachedOnly { + return req, nil, ErrCodexGatewayBorrowUnavailable + } + if !s.targetProbeMu.TryLock() { + return req, nil, ErrCodexGatewayBorrowBusy + } + defer s.targetProbeMu.Unlock() + if !force { + if check, ok, err := s.cachedTarget(rev, targetKey, key, candidate); ok || err != nil { + if err != nil { + return req, nil, err + } + return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires), nil + } + } + s.mu.Lock() + if !s.candidateCurrentLocked(rev, candidate) { + if s.revision == rev { + delete(s.targets, targetKey) + } + s.mu.Unlock() + return req, nil, ErrCodexGatewayBorrowChanged + } + s.targets[targetKey] = codexGatewayBorrowTargetCheck{key: key, cookieKey: borrowHash(candidate.cookie.Value), validating: true, + result: CodexGatewayBorrowVerification{AccountID: account.ID, Model: model, Reason: "validating", CheckedAt: time.Now()}} + s.mu.Unlock() + probeCtx, cancel := borrowRevisionContext(req.Context(), revisionCtx, codexGatewayBorrowTimeout) + defer cancel() + result := s.probeTarget(probeCtx, req, account, model, proxy, profile, candidate) + s.mu.Lock() + if !s.candidateCurrentLocked(rev, candidate) { + if s.revision == rev { + delete(s.targets, targetKey) + } + s.mu.Unlock() + return req, nil, ErrCodexGatewayBorrowChanged + } + check := codexGatewayBorrowTargetCheck{key: key, cookieKey: borrowHash(candidate.cookie.Value), expires: candidate.expires, result: result} + if !result.Success { + check.retryAfter = time.Now().Add(codexGatewayBorrowFailureWait) + } + s.targets[targetKey] = check + s.mu.Unlock() + if !result.Success { + return req, nil, ErrCodexGatewayBorrowUnavailable + } + return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires), nil +} + +func (s *CodexGatewayBorrowService) candidateCurrentLocked(rev uint64, candidate *codexGatewayBorrowCandidate) bool { + return s.revision == rev && s.config.Enabled && s.revisionCtx.Err() == nil && s.candidate != nil && s.candidate.sourceID == candidate.sourceID && s.candidate.cookie.Value == candidate.cookie.Value && s.candidate.cookie.Path == candidate.cookie.Path && s.candidate.expires.Equal(candidate.expires) && time.Now().Before(s.candidate.expires) && time.Now().Before(candidate.expires) +} + +func (s *CodexGatewayBorrowService) cachedTarget(rev uint64, key codexGatewayBorrowTargetKey, fingerprint string, candidate *codexGatewayBorrowCandidate) (codexGatewayBorrowTargetCheck, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.candidateCurrentLocked(rev, candidate) { + return codexGatewayBorrowTargetCheck{}, false, ErrCodexGatewayBorrowChanged + } + old, ok := s.targets[key] + if !ok || old.key != fingerprint || old.validating { + return old, false, nil + } + now := time.Now() + if old.result.Success && now.Before(old.expires) { + if candidate.expires.Before(old.expires) { + old.expires = candidate.expires + } + return old, true, nil + } + if !old.result.Success && now.Before(old.retryAfter) { + return old, false, ErrCodexGatewayBorrowUnavailable + } + return old, false, nil +} + +func borrowApplication(candidate *codexGatewayBorrowCandidate, fingerprint string, expires time.Time) *CodexGatewayBorrowApplication { + return &CodexGatewayBorrowApplication{Applied: true, SourceAccountID: candidate.sourceID, ExpiresAt: expires, Fingerprint: fingerprint, CookieFingerprint: borrowHash(candidate.cookie.Value)} +} + +func borrowApplyCookie(req *http.Request, candidate *codexGatewayBorrowCandidate, check codexGatewayBorrowTargetCheck) *http.Request { + clone := req.Clone(req.Context()) + replaceCodexGatewayBorrowCookie(clone.Header, "__oailb", candidate.cookie.Value) + return clone +} + +func replaceCodexGatewayBorrowCookie(headers http.Header, name, value string) { + var kept []string + for _, header := range headers.Values("Cookie") { + for _, part := range strings.Split(header, ";") { + part = strings.TrimSpace(part) + cookieName, _, ok := strings.Cut(part, "=") + if part != "" && (!ok || strings.TrimSpace(cookieName) != name) { + kept = append(kept, part) + } + } + } + if value != "" { + kept = append(kept, (&http.Cookie{Name: name, Value: value}).String()) + } + if len(kept) == 0 { + headers.Del("Cookie") + } else { + headers.Set("Cookie", strings.Join(kept, "; ")) + } +} + +func borrowCookiePathMatches(path, scope string) bool { + return path == scope || (strings.HasPrefix(path, scope) && (strings.HasSuffix(scope, "/") || strings.HasPrefix(strings.TrimPrefix(path, scope), "/"))) +} + +func (s *CodexGatewayBorrowService) Verify(ctx context.Context, accountID int64, model string) (CodexGatewayBorrowVerification, error) { + result := CodexGatewayBorrowVerification{AccountID: accountID, Model: model, CheckedAt: time.Now(), Reason: "not_configured"} + cfg := s.ConfigSnapshot() + if !cfg.Enabled || !slices.Contains(cfg.TargetAccountIDs, accountID) || !slices.Contains(cfg.Models, model) { + return result, ErrCodexGatewayBorrowUnavailable + } + a, req, proxy, err := s.accountTemplate(ctx, accountID, model) + if err != nil { + return result, err + } + _, _, err = s.apply(req, a, model, proxy, nil, false, true) + s.mu.Lock() + if check, ok := s.targets[codexGatewayBorrowTargetKey{accountID, model}]; ok { + result = check.result + } + s.mu.Unlock() + return result, err +} diff --git a/backend/internal/service/codex_gateway_borrow_core_test.go b/backend/internal/service/codex_gateway_borrow_core_test.go new file mode 100644 index 000000000000..8bb9e4825bd3 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_core_test.go @@ -0,0 +1,476 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// Embedding nil ports makes any health, quota, recovery or account write panic. +// Every request in these tests terminates at the synthetic HTTPUpstream below. +type borrowCoreAccounts struct { + AccountRepository + rows map[int64]*Account +} + +func (r *borrowCoreAccounts) GetByID(_ context.Context, id int64) (*Account, error) { + return r.rows[id], nil +} + +type borrowCoreSettings struct { + SettingRepository + raw string + writes int +} + +func (s *borrowCoreSettings) GetValue(context.Context, string) (string, error) { + if s.raw == "" { + return "", ErrSettingNotFound + } + return s.raw, nil +} +func (s *borrowCoreSettings) Set(_ context.Context, _, value string) error { + s.raw, s.writes = value, s.writes+1 + return nil +} + +type borrowCoreProbe func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) + +func (f borrowCoreProbe) Do(r *http.Request, proxy string, id int64, concurrency int) (*http.Response, error) { + return f(r, proxy, id, concurrency, nil) +} +func (f borrowCoreProbe) DoWithTLS(r *http.Request, proxy string, id int64, concurrency int, p *tlsfingerprint.Profile) (*http.Response, error) { + return f(r, proxy, id, concurrency, p) +} + +func borrowCoreAccount(id int64) *Account { + return &Account{ID: id, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Concurrency: 4, + Credentials: map[string]any{"access_token": "synthetic-token", "chatgpt_account_id": "synthetic-chat-id"}, Extra: map[string]any{}} +} + +func newBorrowCoreTest(t *testing.T, upstream borrowCoreProbe, rows ...*Account) *CodexGatewayBorrowService { + t.Helper() + repo := &borrowCoreAccounts{rows: map[int64]*Account{}} + for _, row := range rows { + repo.rows[row.ID] = row + } + gateway := &OpenAIGatewayService{accountRepo: repo} + s := NewCodexGatewayBorrowService(nil, repo, gateway, upstream, nil) + s.publishConfig(CodexGatewayBorrowConfig{Enabled: true, SourceAccountIDs: []int64{1}, TargetAccountIDs: []int64{2, 3}, Models: []string{"gpt-6-astra", "gpt-6.1-sol"}}, false) + t.Cleanup(s.Stop) + return s +} + +func borrowCoreCandidate(s *CodexGatewayBorrowService, expires time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + s.candidate = &codexGatewayBorrowCandidate{cookie: http.Cookie{Name: "__oailb", Value: "synthetic-borrowed-cookie", Secure: true, Path: "/"}, expires: expires, sourceID: 1} +} + +func borrowCoreSSE(model, text string) string { + output := []map[string]any{} + if text != "" { + output = append(output, map[string]any{"type": "message", "content": []map[string]any{{"type": "output_text", "text": text}}}) + } + raw, _ := json.Marshal(map[string]any{"type": "response.completed", "response": map[string]any{"status": "completed", "model": model, "output": output}}) + return "data: " + string(raw) + "\n\n" +} + +func borrowCoreResponse(model, text, state string, cookies ...string) *http.Response { + h := make(http.Header) + if state != "" { + h.Set("X-Codex-Turn-State", state) + } + for _, cookie := range cookies { + h.Add("Set-Cookie", cookie) + } + return &http.Response{StatusCode: http.StatusOK, Header: h, Body: io.NopCloser(strings.NewReader(borrowCoreSSE(model, text)))} +} + +func borrowCoreBody(t *testing.T, req *http.Request) []byte { + t.Helper() + data, err := io.ReadAll(req.Body) + require.NoError(t, err) + if req.Header.Get("Content-Encoding") == "zstd" { + decoder, err := zstd.NewReader(nil) + require.NoError(t, err) + defer decoder.Close() + data, err = decoder.DecodeAll(data, nil) + require.NoError(t, err) + } + return data +} + +func TestCodexGatewayBorrowConfigDefaultsAndLoadNeverProbe(t *testing.T) { + var calls atomic.Int32 + probe := borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + calls.Add(1) + return nil, errors.New("must not call upstream") + }) + store := &borrowCoreSettings{} + s, err := ProvideCodexGatewayBorrowService(store, nil, nil, probe, nil) + require.NoError(t, err) + t.Cleanup(s.Stop) + require.Equal(t, DefaultCodexGatewayBorrowConfig(), s.ConfigSnapshot()) + require.Zero(t, store.writes) + require.Zero(t, calls.Load()) + store.raw = `{"enabled":true,"source_account_ids":[1],"target_account_ids":[2],"models":["gpt-6.1-sol"]}` + cfg, err := s.GetConfig(context.Background()) + require.NoError(t, err) + require.True(t, cfg.Enabled) + require.Zero(t, calls.Load()) + for _, raw := range []string{`null`, `{"enabled":true}`, `{"source_account_ids":[1],"target_account_ids":[1]}`, `{"models":["gpt-6-luna"]}`, `{"models":["gpt-6-astra","gpt-6-astra"]}`, `{"unknown":true}`, `{} {}`} { + _, err := DecodeCodexGatewayBorrowConfig([]byte(raw)) + require.Error(t, err, raw) + } + _, err = DecodeCodexGatewayBorrowConfig([]byte(`{"enabled":false}`)) + require.NoError(t, err) +} + +func TestCodexGatewayBorrowCookieScopeAndExpiry(t *testing.T) { + now := time.Now() + valid := http.Cookie{Name: "__oailb", Value: "route", Secure: true, Path: "/backend-api/codex"} + for _, tc := range []struct { + name string + change func(*http.Cookie) + ttl time.Duration + valid bool + }{ + {"session", func(*http.Cookie) {}, 230 * time.Second, true}, + {"actual max age", func(c *http.Cookie) { c.MaxAge = 30 }, 30 * time.Second, true}, + {"local cap", func(c *http.Cookie) { c.MaxAge = 1000000 }, 230 * time.Second, true}, + {"actual expires", func(c *http.Cookie) { c.Expires = now.Add(20 * time.Second) }, 20 * time.Second, true}, + {"default path", func(c *http.Cookie) { c.Path = "" }, 230 * time.Second, true}, + {"foreign domain", func(c *http.Cookie) { c.Domain = "example.invalid" }, 0, false}, + {"wrong path", func(c *http.Cookie) { c.Path = "/backend-api/codex/responses-other" }, 0, false}, + {"not secure", func(c *http.Cookie) { c.Secure = false }, 0, false}, + {"deleted", func(c *http.Cookie) { c.MaxAge = -1 }, 0, false}, + {"expired", func(c *http.Cookie) { c.Expires = now.Add(-time.Second) }, 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + cookie := valid + tc.change(&cookie) + candidate := borrowCandidateFromCookies([]*http.Cookie{&cookie}, "/backend-api/codex/responses", now) + if !tc.valid { + require.Nil(t, candidate) + return + } + require.NotNil(t, candidate) + require.Equal(t, now.Add(tc.ttl), candidate.expires) + }) + } + deleted := valid + deleted.MaxAge = -1 + require.Nil(t, borrowCandidateFromCookies([]*http.Cookie{&valid, &deleted}, "/backend-api/codex/responses", now)) +} + +func TestCodexGatewayBorrowSourceUsesOwnExitAndNeverExtendsSameCookie(t *testing.T) { + proxyID := int64(11) + source := borrowCoreAccount(1) + source.ProxyID, source.Proxy = &proxyID, &Proxy{Protocol: "http", Host: "source.invalid", Port: 8080} + var calls int + s := newBorrowCoreTest(t, func(req *http.Request, proxy string, id int64, _ int, profile *tlsfingerprint.Profile) (*http.Response, error) { + calls++ + require.Equal(t, int64(1), id) + require.Equal(t, source.Proxy.URL(), proxy) + require.Nil(t, profile) + require.Equal(t, HTTPUpstreamProfileCodexBorrowSource, HTTPUpstreamProfileFromContext(req.Context())) + require.True(t, IsCodexGatewayBorrowObservation(req.Context())) + require.True(t, HTTPUpstreamRedirectsDisabled(req.Context())) + deadline, ok := req.Context().Deadline() + require.True(t, ok) + require.InDelta(t, 90, time.Until(deadline).Seconds(), 2) + body := borrowCoreBody(t, req) + require.Equal(t, "gpt-6-astra", gjson.GetBytes(body, "model").String()) + require.Equal(t, "medium", gjson.GetBytes(body, "reasoning.effort").String()) + require.Equal(t, "Reply with OK only.", gjson.GetBytes(body, "input.0.content.0.text").String()) + require.Empty(t, req.Header.Get("Cookie")) + require.Empty(t, req.Header.Get("X-Codex-Turn-State")) + return borrowCoreResponse("gpt-6-astra", "OK", "source-state", "__oailb=synthetic-borrowed-cookie; Secure; Path=/; Max-Age=230", "__cflb=source-cookie; Secure; Path=/"), nil + }, source) + expires := time.Now().Add(50 * time.Second) + borrowCoreCandidate(s, expires) + require.NoError(t, s.prepareSource(context.Background(), s.revision, s.revisionCtx)) + require.Equal(t, 1, calls) + require.Equal(t, expires, s.candidate.expires) + borrowCoreCandidate(s, time.Now().Add(150*time.Second)) + require.NoError(t, s.prepareSource(context.Background(), s.revision, s.revisionCtx)) + require.Equal(t, 1, calls, "fresh candidate skips a new source invocation") +} + +func TestCodexGatewayBorrowTargetCacheModelAndFingerprintIsolation(t *testing.T) { + var calls int + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(req *http.Request, proxy string, id int64, _ int, profile *tlsfingerprint.Profile) (*http.Response, error) { + calls++ + require.Equal(t, int64(2), id) + require.Equal(t, "target-exit", proxy) + require.Nil(t, profile) + require.Equal(t, HTTPUpstreamProfileCodexBorrowTarget, HTTPUpstreamProfileFromContext(req.Context())) + require.True(t, IsCodexGatewayBorrowObservation(req.Context())) + require.True(t, req.Close) + deadline, ok := req.Context().Deadline() + require.True(t, ok) + require.InDelta(t, 45, time.Until(deadline).Seconds(), 2) + body := borrowCoreBody(t, req) + model := gjson.GetBytes(body, "model").String() + require.Equal(t, "Reply with OK.", gjson.GetBytes(body, "input.0.content.0.text").String()) + require.False(t, gjson.GetBytes(body, "reasoning.effort").Exists()) + borrowed, err := req.Cookie("__oailb") + require.NoError(t, err) + require.Equal(t, "synthetic-borrowed-cookie", borrowed.Value) + if req.Header.Get("X-Codex-Turn-State") == "" { + require.NotContains(t, req.Header.Get("Cookie"), "source-cookie") + return borrowCoreResponse(model, "", "minted-state", "__cflb=target-cookie; Secure; Path=/"), nil + } + require.Equal(t, "minted-state", req.Header.Get("X-Codex-Turn-State")) + require.Contains(t, req.Header.Get("Cookie"), "__cflb=target-cookie") + // Completed with no visible text is valid for the STATE criterion. + return borrowCoreResponse(model, "", ""), nil + }, a) + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + _, template, _, err := s.accountTemplate(context.Background(), 2, "gpt-6-astra") + require.NoError(t, err) + template.Header.Set("Cookie", "__oailb=old; __cflb=business-cookie; other=keep") + template.Header.Set("X-Codex-Turn-State", "business-state") + for _, model := range []string{"gpt-6-astra", "gpt-6.1-sol"} { + wire, app, err := s.Apply(template, a, model, "target-exit", nil, false) + require.NoError(t, err) + require.True(t, app.Applied) + require.Equal(t, "business-state", wire.Header.Get("X-Codex-Turn-State")) + require.Contains(t, wire.Header.Get("Cookie"), "__cflb=business-cookie") + require.Contains(t, wire.Header.Get("Cookie"), "other=keep") + require.NotContains(t, wire.Header.Get("Cookie"), "minted-state") + require.Contains(t, template.Header.Get("Cookie"), "__oailb=old", "input request is not modified") + } + require.Equal(t, 4, calls) + s.targetProbeMu.Lock() + _, app, err := s.Apply(template, a, "gpt-6.1-sol", "target-exit", nil, false) + require.NoError(t, err) + require.True(t, app.Applied, "hot cache is read before global target lock") + s.targetProbeMu.Unlock() + for _, header := range []string{"Authorization", "ChatGPT-Account-ID", "User-Agent", "Originator", "Version", "X-Codex-Turn-State"} { + req := template.Clone(template.Context()) + req.Header.Set(header, req.Header.Get(header)+"-changed") + _, _, err := s.Apply(req, a, "gpt-6.1-sol", "target-exit", nil, true) + require.Error(t, err, header) + } + _, _, err = s.Apply(template, a, "gpt-6.1-sol", "other-exit", nil, true) + require.Error(t, err) + _, _, err = s.Apply(template, a, "gpt-6.1-sol", "target-exit", &tlsfingerprint.Profile{Name: "changed"}, true) + require.Error(t, err) + require.Equal(t, 4, calls, "cached-only mismatches do not validate") +} + +func TestCodexGatewayBorrowTargetChangedStateRouteAndFailureCooldown(t *testing.T) { + for _, mode := range []string{"state", "cookie", "missing", "incomplete"} { + t.Run(mode, func(t *testing.T) { + var calls int + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + calls++ + borrowCoreBody(t, req) + if mode == "missing" { + return borrowCoreResponse("gpt-6.1-sol", "", ""), nil + } + if mode == "cookie" { + return borrowCoreResponse("gpt-6.1-sol", "", "first", "__oailb=changed; Secure; Path=/"), nil + } + if mode == "incomplete" { + return &http.Response{StatusCode: 200, Header: http.Header{"X-Codex-Turn-State": []string{"first"}}, Body: io.NopCloser(strings.NewReader("data: [DONE]\n\n"))}, nil + } + state := "first" + if calls == 2 { + state = "changed" + } + return borrowCoreResponse("gpt-6.1-sol", "", state), nil + }, a) + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + _, req, _, err := s.accountTemplate(context.Background(), 2, "gpt-6.1-sol") + require.NoError(t, err) + _, _, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.Error(t, err) + initial := calls + _, _, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.Error(t, err) + require.Equal(t, initial, calls, "failure stays in 15-second cooldown") + status := s.Status() + require.False(t, status.Targets[1].CacheValid) + require.NotNil(t, status.Targets[1].RetryAfter) + }) + } +} + +func TestCodexGatewayBorrowSourceSingleFlightCooldownAndRevisionCancellation(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + s := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + if calls.Add(1) == 1 { + close(started) + } + <-release + return &http.Response{StatusCode: http.StatusForbidden, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"full synthetic source failure"}}`))}, nil + }, borrowCoreAccount(1)) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { defer wg.Done(); _ = s.prepareSource(context.Background(), s.revision, s.revisionCtx) }() + } + <-started + close(release) + wg.Wait() + require.Equal(t, int32(1), calls.Load()) + err := s.prepareSource(context.Background(), s.revision, s.revisionCtx) + require.ErrorContains(t, err, "full synthetic source failure") + require.Equal(t, int32(1), calls.Load()) + started, release = make(chan struct{}), make(chan struct{}) + calls.Store(0) + s2 := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + close(started) + <-release + return borrowCoreResponse("gpt-6-astra", "OK", "", "__oailb=late; Secure; Path=/"), nil + }, borrowCoreAccount(1)) + done := make(chan error, 1) + rev, revCtx := s2.revision, s2.revisionCtx + go func() { done <- s2.prepareSource(context.Background(), rev, revCtx) }() + <-started + s2.publishConfig(DefaultCodexGatewayBorrowConfig(), false) + close(release) + require.Error(t, <-done) + require.Nil(t, s2.Status().Candidate) + require.Empty(t, s2.Status().Sources) +} + +type borrowCoreUntouchedBody struct{ reads int } + +func (b *borrowCoreUntouchedBody) Read([]byte) (int, error) { + b.reads++ + return 0, errors.New("business body must not be inspected") +} +func (*borrowCoreUntouchedBody) Close() error { return nil } + +func TestCodexGatewayBorrowGatePreserves100MiBBodyAndOtherRoutes(t *testing.T) { + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + t.Fatal("must not probe") + return nil, nil + }, a) + body := &borrowCoreUntouchedBody{} + req, err := http.NewRequest(http.MethodPost, chatgptCodexURL, body) + require.NoError(t, err) + req.ContentLength = 100 << 20 + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + key := borrowRequestFingerprint(req, "gpt-6.1-sol", "", nil, s.candidate.cookie.Value) + s.targets[codexGatewayBorrowTargetKey{2, "gpt-6.1-sol"}] = codexGatewayBorrowTargetCheck{key: key, cookieKey: borrowHash(s.candidate.cookie.Value), expires: s.candidate.expires, result: CodexGatewayBorrowVerification{Success: true}} + wire, app, err := s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.NoError(t, err) + require.True(t, app.Applied) + require.Same(t, body, wire.Body) + require.Equal(t, int64(100<<20), wire.ContentLength) + require.Zero(t, body.reads) + for _, url := range []string{"https://chatgpt.com/backend-api/codex/responses/compact", "https://chatgpt.com/backend-api/codex/models", "https://api.openai.com/v1/responses", "https://chatgpt.com:443/backend-api/codex/responses", "https://other.invalid/backend-api/codex/responses"} { + other, _ := http.NewRequest(http.MethodPost, url, body) + result, app, err := s.Apply(other, a, "gpt-6.1-sol", "", nil, false) + require.NoError(t, err) + require.Same(t, other, result) + require.Nil(t, app) + } + result, app, err := s.Apply(req, a, "gpt-6-luna", "", nil, false) + require.NoError(t, err) + require.Same(t, req, result) + require.Nil(t, app) + require.Zero(t, body.reads) +} + +func TestCodexGatewayBorrowPelicanCachedOnlyPausedTargetAndFullOutput(t *testing.T) { + var calls int + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + calls++ + body := borrowCoreBody(t, req) + if gjson.GetBytes(body, "input.0.content.0.text").String() == CodexGatewayBorrowPelicanPrompt { + require.Equal(t, "high", gjson.GetBytes(body, "reasoning.effort").String()) + return borrowCoreResponse("gpt-6.1-sol", "鹈鹕骑自行车", ""), nil + } + state := "first" + if calls == 2 { + state = "" + } + return borrowCoreResponse("gpt-6.1-sol", "", state), nil + }, a) + result, err := s.GeneratePelican(context.Background(), 2, "gpt-6.1-sol", "high") + require.NoError(t, err) + require.Equal(t, "skipped", result.Status) + require.Zero(t, calls) + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + _, req, _, err := s.accountTemplate(context.Background(), 2, "gpt-6.1-sol") + require.NoError(t, err) + _, _, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.NoError(t, err) + require.Equal(t, 2, calls) + a.Status, a.Schedulable = "paused", false + result, err = s.GeneratePelican(context.Background(), 2, "gpt-6.1-sol", "") + require.NoError(t, err) + require.Equal(t, "complete", result.Status) + require.Equal(t, "鹈鹕骑自行车", result.RawAnswer) + require.Contains(t, result.RawResponse, "response.completed") + require.Equal(t, 3, calls) + require.Equal(t, "paused", a.Status) + require.False(t, a.Schedulable) + result, err = s.GeneratePelican(context.Background(), 2, "gpt-6-astra", "high") + require.NoError(t, err) + require.Equal(t, "skipped", result.Status) + require.Equal(t, 3, calls) +} + +func TestCodexGatewayBorrowSSERequiresTerminalAndPreservesErrors(t *testing.T) { + for _, stream := range []string{"data: [DONE]\n\n", `data: {"type":"response.output_text.delta","delta":"partial"}` + "\n\n", `data: {"type":"response.failed","response":{"error":{"code":"synthetic_error","message":"full upstream message"}}}` + "\n\n", "data: {bad json}\n\n"} { + got := parseCodexGatewayBorrowSSE([]byte(stream), "gpt-6-astra", codexGatewayBorrowProbeMaxBody) + require.False(t, got.complete) + if strings.Contains(stream, "full upstream message") { + require.Contains(t, got.errorText, "full upstream message") + } + } + createdOnly := `data: {"type":"response.created","response":{"model":"gpt-6-astra"}}` + "\n\n" + `data: {"type":"response.completed","response":{"status":"completed","output":[]}}` + "\n\n" + got := parseCodexGatewayBorrowSSE([]byte(createdOnly), "gpt-6-astra", codexGatewayBorrowProbeMaxBody) + require.True(t, got.complete) + require.Empty(t, got.model, "source requires a final model, not earlier advertised identity") + got = parseCodexGatewayBorrowSSE([]byte(borrowCoreSSE("gpt-6.1-sol", "answer")), "gpt-6-astra", codexGatewayBorrowProbeMaxBody) + require.True(t, got.complete) + require.True(t, got.modelMismatch) + require.Equal(t, "answer", got.answer) +} + +func TestCodexGatewayBorrowQualificationFailuresRemainTyped(t *testing.T) { + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + return nil, errors.New("synthetic source connection refused") + }, borrowCoreAccount(1), a) + _, req, _, err := s.accountTemplate(context.Background(), 2, "gpt-6-astra") + require.NoError(t, err) + _, _, err = s.Apply(req, a, "gpt-6-astra", "", nil, false) + require.True(t, IsCodexGatewayBorrowFailure(err)) + require.ErrorContains(t, err, "synthetic source connection refused") + _, _, err = s.Apply(req, a, "gpt-6-astra", "", nil, true) + require.True(t, IsCodexGatewayBorrowFailure(err)) + require.ErrorIs(t, err, ErrCodexGatewayBorrowUnavailable) + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + s.targetProbeMu.Lock() + _, _, err = s.Apply(req, a, "gpt-6-astra", "", nil, false) + s.targetProbeMu.Unlock() + require.True(t, IsCodexGatewayBorrowFailure(err)) + require.ErrorIs(t, err, ErrCodexGatewayBorrowBusy) + require.False(t, IsCodexGatewayBorrowFailure(errors.New("ordinary business failure"))) +} diff --git a/backend/internal/service/codex_gateway_borrow_http.go b/backend/internal/service/codex_gateway_borrow_http.go new file mode 100644 index 000000000000..6cd2d2accac8 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_http.go @@ -0,0 +1,189 @@ +package service + +import ( + "context" + "errors" + "net/http" + "slices" + "time" + + "github.com/gin-gonic/gin" +) + +const CodexGatewayBorrowPreparationFailureReason GatewayFailureReason = "codex_gateway_borrow_preparation_failed" + +// The existing failover owner may select another account. A local preparation +// failure is not an inference attempt or evidence against the selected account. +func NewCodexGatewayBorrowRequestFailure(_ error) *UpstreamFailoverError { + return &UpstreamFailoverError{ + StatusCode: http.StatusServiceUnavailable, + ResponseBody: []byte(`{"error":{"type":"upstream_error","code":"CODEX_GATEWAY_BORROW_UNAVAILABLE","message":"no qualified Codex gateway borrow route is available"}}`), + Scope: GatewayFailureScopeRequest, + Reason: CodexGatewayBorrowPreparationFailureReason, + NextAccountAction: NextAccountRetry, + RequestScopedTransient: true, + SuppressAccountHealthPenalty: true, + SafeToFailoverAfterWrite: true, + ClientStatusCode: http.StatusServiceUnavailable, + ClientErrorCode: "CODEX_GATEWAY_BORROW_UNAVAILABLE", + ClientErrorType: "upstream_error", + ClientMessage: "no qualified Codex gateway borrow route is available", + } +} + +func IsCodexGatewayBorrowRequestFailure(err error) bool { + if IsCodexGatewayBorrowFailure(err) { + return true + } + var failure *UpstreamFailoverError + return errors.As(err, &failure) && failure.Reason == CodexGatewayBorrowPreparationFailureReason +} + +// Record before converting to failover metadata: the administrator keeps the +// complete local/source cause, while exhausted client responses use their own +// existing error contract. No account or scheduler state is touched here. +func RecordCodexGatewayBorrowPreparationFailure(c *gin.Context, account *Account, err error) { + if err == nil { + return + } + detail := err.Error() + setOpsUpstreamError(c, http.StatusServiceUnavailable, detail, detail) + event := OpsUpstreamErrorEvent{ + UpstreamStatusCode: http.StatusServiceUnavailable, + Kind: string(CodexGatewayBorrowPreparationFailureReason), + Scope: string(GatewayFailureScopeRequest), + Reason: string(CodexGatewayBorrowPreparationFailureReason), + Message: detail, + Detail: detail, + } + if account != nil { + event.ProxyID, event.ProxyName = opsUpstreamProxyID(account), opsUpstreamProxyName(account) + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +type codexGatewayBorrowHTTPPreparationContextKey struct{} +type codexGatewayBorrowHTTPFirstOutputContextKey struct{} + +type codexGatewayBorrowHTTPPreparation struct { + service *CodexGatewayBorrowService + revision uint64 + accountID int64 + model string + proxy string + application *CodexGatewayBorrowApplication +} + +func (s *OpenAIGatewayService) codexGatewayBorrowHTTPConfigured(account *Account, model string) bool { + if s == nil || s.gatewayBorrow == nil || !codexGatewayBorrowAccountSupported(account) { + return false + } + borrow := s.gatewayBorrow + borrow.mu.Lock() + defer borrow.mu.Unlock() + return borrow.config.Enabled && slices.Contains(borrow.config.TargetAccountIDs, account.ID) && slices.Contains(borrow.config.Models, model) +} + +// Qualify the final request using the original client's cancellation/deadline, +// before the business first-output guard. WithContext shares Body/GetBody: this +// phase neither reads nor copies the business body, including a 100 MiB body. +func (s *OpenAIGatewayService) prepareCodexGatewayBorrowHTTP(ctx context.Context, req *http.Request, account *Account, model, proxy string) (*http.Request, time.Duration, error) { + started := time.Now() + prepared := codexGatewayBorrowHTTPPreparation{service: s.gatewayBorrow, accountID: account.ID, model: model, proxy: proxy} + if prepared.service != nil { + prepared.service.mu.Lock() + prepared.revision = prepared.service.revision + prepared.service.mu.Unlock() + borrowed, application, err := prepared.service.Apply(req.WithContext(ctx), account, model, proxy, nil, false) + if err != nil { + return nil, 0, err + } + req, prepared.application = borrowed.WithContext(req.Context()), application + } + req = req.WithContext(context.WithValue(req.Context(), codexGatewayBorrowHTTPPreparationContextKey{}, prepared)) + if prepared.application != nil && prepared.application.Applied { + return req, time.Since(started), nil + } + return req, 0, nil +} + +// Forward already performed its one preparation before starting the business +// clock. At dispatch only matching cached evidence may be used. Expiry, a save +// or an identity change must not start another probe underneath that clock. +func (s *OpenAIGatewayService) applyCodexGatewayBorrowHTTP(req *http.Request, account *Account, model, proxy string) (*http.Request, error) { + prepared, ok := req.Context().Value(codexGatewayBorrowHTTPPreparationContextKey{}).(codexGatewayBorrowHTTPPreparation) + if !ok { + if s.gatewayBorrow == nil { + return req, nil + } + borrowed, _, err := s.gatewayBorrow.Apply(req, account, model, proxy, nil, false) + return borrowed, err + } + changed := func() (*http.Request, error) { + return nil, &CodexGatewayBorrowFailure{Cause: ErrCodexGatewayBorrowChanged} + } + if prepared.application == nil || !prepared.application.Applied { + if s.codexGatewayBorrowHTTPConfigured(account, model) && CodexGatewayBorrowRequestEligible(req) { + return changed() + } + return req, nil + } + if err := req.Context().Err(); err != nil { + return nil, err + } + if s.gatewayBorrow != prepared.service || account.ID != prepared.accountID || model != prepared.model || proxy != prepared.proxy { + return changed() + } + prepared.service.mu.Lock() + current := prepared.service.revision == prepared.revision && prepared.service.revisionCtx.Err() == nil + prepared.service.mu.Unlock() + if !current || !time.Now().Before(prepared.application.ExpiresAt) { + return changed() + } + borrowed, application, err := prepared.service.Apply(req, account, model, proxy, nil, true) + if err != nil { + return nil, err + } + if application == nil || !application.Applied || application.Fingerprint != prepared.application.Fingerprint || + application.CookieFingerprint != prepared.application.CookieFingerprint || application.SourceAccountID != prepared.application.SourceAccountID || + !application.ExpiresAt.Equal(prepared.application.ExpiresAt) { + return changed() + } + return borrowed, nil +} + +type codexGatewayBorrowRecoveryContext struct { + context.Context + values context.Context +} + +func (c codexGatewayBorrowRecoveryContext) Value(key any) any { return c.values.Value(key) } + +// A prepared recovery request already carries the caller's cancellation and +// deadline. Preserve that exact context unless an old header guard was replaced. +// In that race retain request values, but bind lifecycle to the original caller +// without adding cancellation callbacks or letting the old guard cancel it. +func codexGatewayBorrowHTTPPreparedContext(caller, prepared context.Context, replacedGuard, recovery bool) context.Context { + if !replacedGuard { + return prepared + } + values := context.WithoutCancel(prepared) + if recovery { + return codexGatewayBorrowRecoveryContext{Context: caller, values: values} + } + return values +} + +func withCodexGatewayBorrowHTTPFirstOutputStart(ctx context.Context, start time.Time) context.Context { + return context.WithValue(ctx, codexGatewayBorrowHTTPFirstOutputContextKey{}, start) +} + +func codexGatewayBorrowHTTPFirstOutputStart(ctx context.Context, fallback time.Time) time.Time { + if ctx != nil { + if start, ok := ctx.Value(codexGatewayBorrowHTTPFirstOutputContextKey{}).(time.Time); ok { + return start + } + } + return fallback +} diff --git a/backend/internal/service/codex_gateway_borrow_http_test.go b/backend/internal/service/codex_gateway_borrow_http_test.go new file mode 100644 index 000000000000..f94b84b29a4a --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_http_test.go @@ -0,0 +1,333 @@ +package service + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type borrowHTTPMutationRepo struct { + *borrowCoreAccounts + writes atomic.Int32 +} + +func (r *borrowHTTPMutationRepo) SetError(context.Context, int64, string) error { + r.writes.Add(1) + return nil +} +func (r *borrowHTTPMutationRepo) SetRateLimited(context.Context, int64, time.Time) error { + r.writes.Add(1) + return nil +} +func (r *borrowHTTPMutationRepo) SetTempUnschedulable(context.Context, int64, time.Time, string) error { + r.writes.Add(1) + return nil +} +func (r *borrowHTTPMutationRepo) UpdateExtra(context.Context, int64, map[string]any) error { + r.writes.Add(1) + return nil +} + +func TestCodexGatewayBorrowHTTPPreparationExcludedFromFirstOutputBudgets(t *testing.T) { + gin.SetMode(gin.TestMode) + var probeCalls, businessCalls atomic.Int32 + probe := borrowCoreProbe(func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + if probeCalls.Add(1) == 1 { + // Longer than the entire business first-output budget. This local + // observation is bound by the original client, not that business guard. + timer := time.NewTimer(1200 * time.Millisecond) + defer timer.Stop() + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-timer.C: + } + return borrowCoreResponse("gpt-6-astra", "", "synthetic-state"), nil + } + return borrowCoreResponse("gpt-6-astra", "", ""), nil + }) + account := borrowCoreAccount(2) + account.Schedulable = true + borrow := newBorrowCoreTest(t, probe, account) + borrowCoreCandidate(borrow, time.Now().Add(time.Minute)) + repo := &borrowHTTPMutationRepo{borrowCoreAccounts: &borrowCoreAccounts{rows: map[int64]*Account{2: account}}} + borrow.accounts, borrow.gateway.accountRepo = repo, repo + svc := borrow.gateway + svc.cfg = &config.Config{Gateway: config.GatewayConfig{OpenAIFirstOutputTimeoutSeconds: 1, MaxLineSize: defaultMaxLineSize}} + svc.rateLimitService = &RateLimitService{accountRepo: repo} + svc.httpUpstream = borrowCoreProbe(func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + businessCalls.Add(1) + // Exercise both response-header and semantic-output waiting, after + // preparation already exceeded the original one-second deadline. + timer := time.NewTimer(100 * time.Millisecond) + defer timer.Stop() + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-timer.C: + } + reader, writer := io.Pipe() + go func() { + defer func() { _ = writer.Close() }() + select { + case <-req.Context().Done(): + return + case <-time.After(100 * time.Millisecond): + } + _, _ = io.WriteString(writer, "data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\n"+borrowCoreSSE("gpt-6-astra", "OK")) + }() + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: reader}, nil + }) + body := []byte(`{"model":"gpt-6-astra","stream":true,"reasoning":{"effort":"low"},"input":"hello"}`) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + clientCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)).WithContext(clientCtx) + started := time.Now() + result, err := svc.Forward(clientCtx, c, account, body) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, int32(2), probeCalls.Load()) + require.Equal(t, int32(1), businessCalls.Load()) + require.Contains(t, recorder.Body.String(), "OK") + require.GreaterOrEqual(t, result.Duration, 1200*time.Millisecond) + require.NotNil(t, result.FirstTokenMs) + require.GreaterOrEqual(t, *result.FirstTokenMs, 1200, "TTFT must retain preparation wall time") + require.GreaterOrEqual(t, time.Since(started), 1200*time.Millisecond) + require.Zero(t, repo.writes.Load()) + require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) + require.False(t, svc.isOpenAIAccountModelRuntimeBlocked(account, "gpt-6-astra")) +} + +func TestCodexGatewayBorrowHTTPRecoveryPreservesCallerContext(t *testing.T) { + for _, lifecycle := range []string{"cancel", "deadline"} { + for _, replacedGuard := range []bool{false, true} { + name := lifecycle + "/prepared_context" + if replacedGuard { + name = lifecycle + "/replaced_guard" + } + t.Run(name, func(t *testing.T) { + type callerValueKey struct{} + values := context.WithValue(context.Background(), callerValueKey{}, "retained") + caller, cancel := context.WithCancel(values) + if lifecycle == "deadline" { + cancel() + caller, cancel = context.WithTimeout(values, 250*time.Millisecond) + } + defer cancel() + transport, _ := detachUpstreamContext(caller) + transport = WithHTTPUpstreamProfile(transport, HTTPUpstreamProfileOpenAI) + var oldGuard *openAIFirstOutputHeaderGuard + if replacedGuard { + transport, oldGuard = newOpenAIFirstOutputHeaderGuard(transport, func() {}, time.Now().Add(time.Minute)) + defer oldGuard.close() + } + account := borrowCoreAccount(2) + var probes, businessCalls atomic.Int32 + borrow := newBorrowCoreTest(t, borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + if probes.Add(1) == 1 { + time.Sleep(10 * time.Millisecond) + return borrowCoreResponse("gpt-6-astra", "", "synthetic-state"), nil + } + return borrowCoreResponse("gpt-6-astra", "", ""), nil + }), account) + borrowCoreCandidate(borrow, time.Now().Add(time.Minute)) + body := []byte(`{"model":"gpt-6-astra","input":"synthetic recovery"}`) + req, err := http.NewRequestWithContext(transport, http.MethodPost, chatgptCodexURL, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer synthetic-target") + recovery := &openAIReasoningRecoveryState{ctx: caller, account: account, token: "synthetic-target", enabled: true, retryUsed: true, retryBody: body} + recovery.identity = openAIReasoningRequestIdentity(account, req, "", recovery.token) + defer recovery.Close() + req, _, err = recovery.PrepareRequest(req, body, "") + require.NoError(t, err) + prepared, elapsed, err := borrow.gateway.prepareCodexGatewayBorrowHTTP(caller, req, account, "gpt-6-astra", "") + require.NoError(t, err) + require.Equal(t, int32(2), probes.Load(), "fixture must qualify the recovery request") + require.Positive(t, elapsed) + if oldGuard != nil { + oldGuard.close() + } + adjusted := codexGatewayBorrowHTTPPreparedContext(caller, prepared.Context(), replacedGuard, recovery.RecoveryAttempt()) + if !replacedGuard { + require.Same(t, prepared.Context(), adjusted) + } + callerDeadline, callerHasDeadline := caller.Deadline() + deadline, hasDeadline := adjusted.Deadline() + require.Equal(t, callerHasDeadline, hasDeadline) + require.Equal(t, callerDeadline, deadline) + require.Equal(t, "retained", adjusted.Value(callerValueKey{})) + require.Equal(t, HTTPUpstreamProfileOpenAI, HTTPUpstreamProfileFromContext(adjusted)) + require.True(t, HTTPUpstreamRedirectsDisabled(adjusted)) + require.NotNil(t, adjusted.Value(codexGatewayBorrowHTTPPreparationContextKey{})) + require.NoError(t, adjusted.Err()) + require.NoError(t, context.Cause(adjusted), "replaced header guard must not supply the cancellation cause") + prepared = prepared.WithContext(adjusted) + if lifecycle == "cancel" { + cancel() + } + select { + case <-adjusted.Done(): + case <-time.After(time.Second): + t.Fatal("qualified recovery lost the caller's lifecycle") + } + borrow.gateway.httpUpstream = borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + businessCalls.Add(1) + return nil, errors.New("expired recovery must not dispatch") + }) + _, err = borrow.gateway.doOpenAICodexUpstream(prepared, account, "", "gpt-6-astra") + require.ErrorIs(t, err, adjusted.Err()) + require.Equal(t, int32(2), probes.Load()) + require.Zero(t, businessCalls.Load()) + }) + } + } +} + +func TestCodexGatewayBorrowHTTPFailureUsesAccountNeutralFailover(t *testing.T) { + for _, cause := range []error{errors.New("source proxy connection refused: complete diagnostic"), ErrCodexGatewayBorrowBusy, ErrCodexGatewayBorrowUnavailable, ErrCodexGatewayBorrowChanged} { + t.Run(cause.Error(), func(t *testing.T) { + account := borrowCoreAccount(2) + repo := &borrowHTTPMutationRepo{borrowCoreAccounts: &borrowCoreAccounts{}} + svc := &OpenAIGatewayService{accountRepo: repo, rateLimitService: &RateLimitService{accountRepo: repo}} + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + err := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account, &CodexGatewayBorrowFailure{Cause: cause}, false) + var failure *UpstreamFailoverError + require.ErrorAs(t, err, &failure) + require.Equal(t, GatewayFailureScopeRequest, failure.Scope) + require.Equal(t, CodexGatewayBorrowPreparationFailureReason, failure.Reason) + require.True(t, failure.SuppressAccountHealthPenalty) + require.True(t, failure.ShouldRetryNextAccount()) + require.False(t, failure.ShouldReportAccountScheduleFailure()) + // This is the same exhaustion owner used by the handler after its + // narrow no-same-account-retry policy selects another account. + svc.CooldownOpenAIRetryExhausted(context.Background(), account, "gpt-6-astra", failure) + require.Zero(t, repo.writes.Load()) + require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) + require.False(t, svc.isOpenAIAccountModelRuntimeBlocked(account, "gpt-6-astra")) + value, ok := c.Get(OpsUpstreamErrorsKey) + require.True(t, ok) + events, ok := value.([]*OpsUpstreamErrorEvent) + require.True(t, ok) + require.Len(t, events, 1) + require.Contains(t, events[0].Detail, cause.Error()) + }) + } + ordinary := (&OpenAIGatewayService{}).handleOpenAIUpstreamTransportError(context.Background(), nil, borrowCoreAccount(2), errors.New("connection refused"), false) + var failure *UpstreamFailoverError + require.ErrorAs(t, ordinary, &failure) + require.Equal(t, OpenAIPersistentTransportFailureReason, failure.Reason) + require.False(t, failure.SuppressAccountHealthPenalty) +} + +func TestCodexGatewayBorrowHTTPPreparedDispatchNeverReprepares(t *testing.T) { + for _, change := range []string{"expired", "configuration", "identity"} { + t.Run(change, func(t *testing.T) { + var probeCalls, businessCalls atomic.Int32 + probe := borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + if probeCalls.Add(1) == 1 { + return borrowCoreResponse("gpt-6-astra", "", "synthetic-state"), nil + } + return borrowCoreResponse("gpt-6-astra", "", ""), nil + }) + account := borrowCoreAccount(2) + borrow := newBorrowCoreTest(t, probe, account) + borrowCoreCandidate(borrow, time.Now().Add(time.Minute)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, chatgptCodexURL, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer synthetic-target") + prepared, _, err := borrow.gateway.prepareCodexGatewayBorrowHTTP(context.Background(), req, account, "gpt-6-astra", "") + require.NoError(t, err) + switch change { + case "expired": + borrow.mu.Lock() + borrow.candidate.expires = time.Now().Add(-time.Second) + borrow.mu.Unlock() + case "configuration": + cfg := borrow.ConfigSnapshot() + cfg.Enabled = false + borrow.publishConfig(cfg, false) + case "identity": + prepared.Header.Set("Authorization", "Bearer changed-target") + } + borrow.gateway.httpUpstream = borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + businessCalls.Add(1) + return nil, errors.New("must not dispatch on changed evidence") + }) + _, err = borrow.gateway.doOpenAICodexUpstream(prepared, account, "", "gpt-6-astra") + require.True(t, IsCodexGatewayBorrowFailure(err), "changed evidence must fail locally") + require.Equal(t, int32(2), probeCalls.Load(), "the guarded dispatch may only check cached evidence") + require.Zero(t, businessCalls.Load()) + }) + } +} + +type borrowHTTPUnreadBody struct{ reads atomic.Int32 } + +func (r *borrowHTTPUnreadBody) Read([]byte) (int, error) { + r.reads.Add(1) + return 0, errors.New("business body must not be inspected by the borrow gate") +} +func (*borrowHTTPUnreadBody) Close() error { return nil } + +func TestCodexGatewayBorrowHTTPOrdinaryGatingDoesNotInspectBody(t *testing.T) { + for _, ordinary := range []string{"other_model", "other_host", "other_method", "other_path"} { + t.Run(ordinary, func(t *testing.T) { + var probes, business atomic.Int32 + probe := borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + probes.Add(1) + return nil, errors.New("ordinary request must not probe") + }) + account := borrowCoreAccount(2) + borrow := newBorrowCoreTest(t, probe, account) + model := "gpt-6-astra" + req, err := http.NewRequest(http.MethodPost, chatgptCodexURL, nil) + require.NoError(t, err) + switch ordinary { + case "other_model": + model = "gpt-5.5" + case "other_host": + req.URL.Host = "api.openai.com" + case "other_method": + req.Method = http.MethodGet + case "other_path": + req.URL.Path += "/compact" + } + body := &borrowHTTPUnreadBody{} + req.Body, req.ContentLength = body, 100<<20 + req.GetBody = func() (io.ReadCloser, error) { t.Fatal("borrow gate must not reread GetBody"); return nil, nil } + prepared, elapsed, err := borrow.gateway.prepareCodexGatewayBorrowHTTP(context.Background(), req, account, model, "") + require.NoError(t, err) + require.Zero(t, elapsed) + require.Same(t, body, prepared.Body) + borrow.gateway.httpUpstream = borrowCoreProbe(func(got *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + business.Add(1) + require.Same(t, body, got.Body) + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("ordinary"))}, nil + }) + oldCompression := codexRequestZstd.Load() + SetCodexRequestZstdEnabled(false) + defer SetCodexRequestZstdEnabled(oldCompression) + resp, err := borrow.gateway.doOpenAICodexUpstream(prepared, account, "", model) + require.NoError(t, err) + _ = resp.Body.Close() + require.Zero(t, probes.Load()) + require.Zero(t, body.reads.Load()) + require.Equal(t, int32(1), business.Load()) + }) + } +} diff --git a/backend/internal/service/codex_gateway_borrow_pelican.go b/backend/internal/service/codex_gateway_borrow_pelican.go new file mode 100644 index 000000000000..ad198d270aa2 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_pelican.go @@ -0,0 +1,123 @@ +package service + +import ( + "context" + "errors" + "net/http" + "slices" + "strings" + "time" +) + +type CodexGatewayBorrowPelicanResult struct { + RawAnswer string `json:"raw_answer"` + RawResponse string `json:"raw_response"` + ModelID string `json:"model_id"` + Effort string `json:"effort"` + Status string `json:"status"` + Error string `json:"error"` +} + +func codexGatewayBorrowModelEfforts() map[string][]string { + result := map[string][]string{} + for _, model := range DefaultCodexGatewayBorrowConfig().Models { + levels := configuredCodexGPTReasoningLevels(model) + efforts := make([]string, 0, len(levels)) + for _, level := range levels { + // The local manifest's Ultra entry describes Codex orchestration. + // The inference observation sends only native API effort values. + if level.Effort != "ultra" { + efforts = append(efforts, level.Effort) + } + } + result[model] = efforts + } + return result +} + +// GeneratePelican observes exactly one request on an already-qualified route. +// Missing, expired or mismatched evidence produces "skipped" without acquiring +// a source cookie or validating a target. The runner owns its bounded parallel +// work, persistence and preview; this method changes no route or account state. +func (s *CodexGatewayBorrowService) GeneratePelican(ctx context.Context, accountID int64, model, effort string) (*CodexGatewayBorrowPelicanResult, error) { + result := &CodexGatewayBorrowPelicanResult{ModelID: model, Effort: effort, Status: "skipped"} + if effort == "" { + effort, result.Effort = "high", "high" + } + if !slices.Contains(codexGatewayBorrowModelEfforts()[model], effort) { + result.Error = "reasoning effort is unsupported for this model" + return result, errors.New(result.Error) + } + cfg := s.ConfigSnapshot() + if !cfg.Enabled || !slices.Contains(cfg.TargetAccountIDs, accountID) || !slices.Contains(cfg.Models, model) { + result.Error = "target model is not configured for gateway borrowing" + return result, nil + } + s.mu.Lock() + rev, revisionCtx := s.revision, s.revisionCtx + s.mu.Unlock() + s.mu.Lock() + cached := s.candidate != nil && time.Now().Before(s.candidate.expires) + s.mu.Unlock() + if !cached { + result.Error = "qualified route cache is missing or expired" + return result, nil + } + requestCtx, cancel := borrowRevisionContext(ctx, revisionCtx, codexGatewayBorrowTimeout) + defer cancel() + a, template, proxy, err := s.accountTemplate(requestCtx, accountID, model, true) + if err != nil { + result.Error = err.Error() + return result, err + } + if supported, _ := AccountTestReasoningOptions(a, model); !slices.Contains(supported, effort) { + result.Error = "selected reasoning effort is not supported by the account's local model metadata" + return result, errors.New(result.Error) + } + // Use the same finalized target identity as manual verification. Effort + // and prompt do not enter the route key; model/auth/exit/business STATE do. + req, err := borrowObservationRequest(requestCtx, template.Header, model, CodexGatewayBorrowPelicanPrompt, effort, "", nil, HTTPUpstreamProfileCodexBorrowTarget) + if err != nil { + result.Error = err.Error() + return result, err + } + borrowed, applied, err := s.Apply(req, a, model, proxy, nil, true) + if err != nil || applied == nil || !applied.Applied { + result.Error = "matching qualified route cache is missing, expired or changed" + return result, nil + } + s.mu.Lock() + valid := s.revision == rev && revisionCtx.Err() == nil && s.candidate != nil && borrowHash(s.candidate.cookie.Value) == applied.CookieFingerprint && s.candidate.expires.After(time.Now()) + s.mu.Unlock() + if !valid { + result.Error = "gateway borrow configuration or route changed before dispatch" + return result, nil + } + // The final cached check happened immediately before this single dispatch. + // It does not write qualification results, even when the response fails. + shot, sendErr := s.sendObservation(borrowed, a, proxy, nil, model, codexGatewayBorrowPelicanMaxBody) + result.RawAnswer, result.RawResponse = shot.answer, shot.raw + if shot.model != "" { + result.ModelID = shot.model + } + if sendErr != nil { + result.Status, result.Error = "failed", borrowObservationError(shot, sendErr) + if shot.incomplete { + result.Status = "incomplete" + } + return result, sendErr + } + if shot.status != http.StatusOK || shot.errorText != "" { + result.Status, result.Error = "failed", borrowObservationError(shot, nil) + if shot.incomplete { + result.Status = "incomplete" + } + return result, nil + } + if !shot.complete || strings.TrimSpace(shot.answer) == "" { + result.Status, result.Error = "incomplete", borrowObservationError(shot, nil) + return result, nil + } + result.Status = "complete" + return result, nil +} diff --git a/backend/internal/service/codex_gateway_borrow_preview.go b/backend/internal/service/codex_gateway_borrow_preview.go new file mode 100644 index 000000000000..a9a5ea17b93c --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_preview.go @@ -0,0 +1,239 @@ +package service + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "regexp" + "strings" + "sync" + "time" +) + +const ( + CodexGatewayBorrowPreviewTTL = 30 * time.Minute + codexGatewayBorrowPreviewMaxCapabilities = 4096 + // The HTTP header applies before any model-authored markup is parsed. The + // iframe has the same sandbox without allow-same-origin, so its origin is opaque. + CodexGatewayBorrowPreviewCSP = "default-src 'none'; sandbox allow-scripts; script-src 'unsafe-inline' http: https: data: blob:; style-src 'unsafe-inline' http: https: data: blob:; img-src http: https: data: blob:; font-src http: https: data: blob:; media-src data: blob:; connect-src 'none'; frame-src 'none'; child-src 'none'; worker-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'self'; navigate-to 'none'" +) + +var ( + codexGatewayBorrowHTMLMarker = regexp.MustCompile(`(?i)<(?:!doctype\s+html|html|svg)[\s>]`) + codexGatewayBorrowHTMLTag = regexp.MustCompile(`(?i)]`) + codexGatewayBorrowSVGTag = regexp.MustCompile(`(?i)]`) + ErrCodexGatewayBorrowPreviewUnavailable = errors.New("preview capability unavailable or expired") +) + +// Extraction follows ranxi2001/sub2api@5ca3cca's fenced/bare HTML/SVG +// selection and minimum SVG wrapper. Selecting the last valid fence follows +// iwyxdxl/gpttesticu@1b3e237's extractHtml. Their sanitizing/CSP rewrites are +// deliberately not used: raw source and JavaScript remain intact, with the +// boundary enforced by the response header and opaque sandbox instead. +// rawHTML is the selected source before any SVG wrapper is added. +func ExtractCodexGatewayBorrowHTML(answer string) (rawHTML, html string) { + var selected string + var outside strings.Builder + for cursor := 0; cursor < len(answer); { + opening := strings.Index(answer[cursor:], "```") + if opening < 0 { + _, _ = outside.WriteString(answer[cursor:]) + break + } + opening += cursor + _, _ = outside.WriteString(answer[cursor:opening]) + bodyStart := opening + 3 + closing := strings.Index(answer[bodyStart:], "```") + bodyEnd := len(answer) + if closing >= 0 { + bodyEnd = bodyStart + closing + } + body, allowed := codexGatewayBorrowHTMLFenceContent(answer[bodyStart:bodyEnd]) + if allowed && codexGatewayBorrowHTMLMarker.MatchString(body) { + selected = strings.TrimSpace(body) + } + if closing < 0 { + break + } + cursor = bodyEnd + 3 + } + if selected == "" { + selected = strings.TrimSpace(outside.String()) + } + start := codexGatewayBorrowHTMLMarker.FindStringIndex(selected) + if start == nil { + return "", "" + } + selected = strings.TrimSpace(selected[start[0]:]) + lower := strings.ToLower(selected) + if end := strings.LastIndex(lower, ""); end >= 0 { + selected = selected[:end+len("")] + } + rawHTML = selected + if !codexGatewayBorrowHTMLTag.MatchString(selected) && codexGatewayBorrowSVGTag.MatchString(selected) { + return rawHTML, `Pelican test` + selected + `` + } + return rawHTML, selected +} + +func codexGatewayBorrowHTMLFenceContent(body string) (string, bool) { + if newline := strings.IndexByte(body, '\n'); newline >= 0 { + language := strings.ToLower(strings.TrimSpace(body[:newline])) + if language == "" || language == "html" || language == "htm" || language == "xml" || language == "svg" { + return body[newline+1:], true + } + } + // Inline fenced markup has no standalone language line. + trimmed := strings.TrimSpace(body) + for _, language := range []string{"html", "htm", "xml", "svg"} { + if len(trimmed) > len(language) && strings.EqualFold(trimmed[:len(language)], language) && (trimmed[len(language)] == ' ' || trimmed[len(language)] == '\t') { + return strings.TrimSpace(trimmed[len(language):]), true + } + } + return trimmed, strings.HasPrefix(trimmed, "<") +} + +type codexGatewayBorrowPreviewCapability struct { + ResultID string + ExpiresAt time.Time +} + +type CodexGatewayBorrowPreviewService struct { + repo CodexGatewayBorrowTestRepository + mu sync.Mutex + capabilities map[string]codexGatewayBorrowPreviewCapability + stopCh chan struct{} + doneCh chan struct{} + stopOnce sync.Once + now func() time.Time + randomToken func() (string, error) +} + +func NewCodexGatewayBorrowPreviewService(repo CodexGatewayBorrowTestRepository) *CodexGatewayBorrowPreviewService { + p := &CodexGatewayBorrowPreviewService{repo: repo, capabilities: make(map[string]codexGatewayBorrowPreviewCapability), + stopCh: make(chan struct{}), doneCh: make(chan struct{}), now: time.Now, randomToken: randomCodexGatewayBorrowPreviewToken} + go p.cleanupLoop() + return p +} + +func randomCodexGatewayBorrowPreviewToken() (string, error) { + var bytes [32]byte + if _, err := rand.Read(bytes[:]); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(bytes[:]), nil +} + +func (p *CodexGatewayBorrowPreviewService) Stop() { + p.stopOnce.Do(func() { close(p.stopCh); <-p.doneCh; p.mu.Lock(); clear(p.capabilities); p.mu.Unlock() }) +} + +func (p *CodexGatewayBorrowPreviewService) cleanupLoop() { + defer close(p.doneCh) + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-p.stopCh: + return + case <-ticker.C: + p.mu.Lock() + p.pruneLocked(p.now().UTC()) + p.mu.Unlock() + } + } +} + +func (p *CodexGatewayBorrowPreviewService) pruneLocked(now time.Time) { + for token, capability := range p.capabilities { + if !capability.ExpiresAt.After(now) { + delete(p.capabilities, token) + } + } +} + +// DecorateResult only adds an independently minted read capability. It makes no +// upstream request and never turns incomplete HTML into a successful test. +func (p *CodexGatewayBorrowPreviewService) DecorateResult(result *CodexGatewayBorrowTestResult) *CodexGatewayBorrowTestResult { + decorated := *result + decorated.PreviewURL, decorated.PreviewExpiresAt = "", nil + if decorated.Status != "complete" { + decorated.PreviewUnavailable = "test " + decorated.Status + return &decorated + } + if decorated.HTML == "" { + decorated.PreviewUnavailable = "answer contains no extractable HTML or SVG" + return &decorated + } + now := p.now().UTC() + if !decorated.ExpiresAt.After(now) { + decorated.PreviewUnavailable = "test expired" + return &decorated + } + expires := now.Add(CodexGatewayBorrowPreviewTTL) + if decorated.ExpiresAt.Before(expires) { + expires = decorated.ExpiresAt + } + token, err := p.randomToken() + if err != nil { + decorated.PreviewUnavailable = "preview capability creation failed: " + err.Error() + return &decorated + } + p.mu.Lock() + p.pruneLocked(now) + // Bound the in-memory capability store even if many administrators keep + // reloading results. The oldest capability expires first and is evicted. + if len(p.capabilities) >= codexGatewayBorrowPreviewMaxCapabilities { + var oldestToken string + var oldest time.Time + for candidate, capability := range p.capabilities { + if oldestToken == "" || capability.ExpiresAt.Before(oldest) { + oldestToken, oldest = candidate, capability.ExpiresAt + } + } + delete(p.capabilities, oldestToken) + } + p.capabilities[token] = codexGatewayBorrowPreviewCapability{ResultID: decorated.ID, ExpiresAt: expires} + p.mu.Unlock() + decorated.PreviewURL = "/api/v1/codex-gateway-borrow/preview/" + token + "/index.html" + decorated.PreviewExpiresAt, decorated.PreviewUnavailable = &expires, "" + return &decorated +} + +func (p *CodexGatewayBorrowPreviewService) DecorateTask(task *CodexGatewayBorrowTestTask) *CodexGatewayBorrowTestTask { + decorated := *task + if task.Results != nil { + decorated.Results = make([]*CodexGatewayBorrowTestResult, 0, len(task.Results)) + for _, result := range task.Results { + decorated.Results = append(decorated.Results, p.DecorateResult(result)) + } + } + return &decorated +} + +// Resolve checks the capability and re-reads exactly its bound result. Expired +// task data is inaccessible even when the periodic SQL cleanup has not run. +func (p *CodexGatewayBorrowPreviewService) Resolve(ctx context.Context, token string) (string, error) { + if len(token) != 43 { + return "", ErrCodexGatewayBorrowPreviewUnavailable + } + p.mu.Lock() + now := p.now().UTC() + capability, found := p.capabilities[token] + if !found || !capability.ExpiresAt.After(now) { + delete(p.capabilities, token) + p.mu.Unlock() + return "", ErrCodexGatewayBorrowPreviewUnavailable + } + p.mu.Unlock() + result, err := p.repo.GetResult(ctx, capability.ResultID) + finishedRead := p.now().UTC() + if err != nil || result == nil || result.ID != capability.ResultID || result.Status != "complete" || result.HTML == "" || !result.ExpiresAt.After(finishedRead) || !capability.ExpiresAt.After(finishedRead) { + p.mu.Lock() + delete(p.capabilities, token) + p.mu.Unlock() + return "", ErrCodexGatewayBorrowPreviewUnavailable + } + return result.HTML, nil +} diff --git a/backend/internal/service/codex_gateway_borrow_preview_test.go b/backend/internal/service/codex_gateway_borrow_preview_test.go new file mode 100644 index 000000000000..71ba1ce8d109 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_preview_test.go @@ -0,0 +1,105 @@ +//go:build unit + +package service + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestCodexGatewayBorrowHTMLExtractionPreservesSelectedSourceAndScripts(t *testing.T) { + const final = `` + raw, html := ExtractCodexGatewayBorrowHTML("reasoning\n```json\n{}\n```\n```html\nold\n```\n```html\n" + final + "\n```\nfinished") + require.Equal(t, final, raw) + require.Equal(t, final, html) + require.NotContains(t, html, "Content-Security-Policy") + const svg = `` + raw, html = ExtractCodexGatewayBorrowHTML("```svg\n" + svg + "\n```") + require.Equal(t, svg, raw) + require.Contains(t, html, svg) + require.Contains(t, html, ``) + require.NotContains(t, html, "DOMPurify") + raw, html = ExtractCodexGatewayBorrowHTML("Explanation\npelican\nmore prose") + require.Equal(t, `pelican`, raw) + require.Equal(t, raw, html) + raw, html = ExtractCodexGatewayBorrowHTML("There is a pelican riding a bicycle.") + require.Empty(t, raw) + require.Empty(t, html) + raw, html = ExtractCodexGatewayBorrowHTML("```json\n{\"example\":\"quoted markup\"}\n```") + require.Empty(t, raw, "markup quoted in a non-HTML code fence is not a preview") + require.Empty(t, html) +} + +func TestCodexGatewayBorrowHTMLExtractionAcceptsUnclosedValidFenceWithoutRepair(t *testing.T) { + const partial = `
+