diff --git a/.downstream/native-domains.md b/.downstream/native-domains.md index 3ce5db23942f..c4b296081f18 100644 --- a/.downstream/native-domains.md +++ b/.downstream/native-domains.md @@ -82,8 +82,10 @@ request usage and upstream completion. Pelican stays at `/admin/pelican-tests`; its legacy comparison URL redirects there. Page reads and navigation only read local data. Unsaved configuration stays in browser-session storage. Saving an enabled configuration starts exactly one finite preparation, shown as queued -before the worker starts. Status polls every five seconds only while work is -active; countdowns expire routes without renewing them. +before the worker starts. While enabled and visible, status polls every five +seconds, including idle periods so later business failures become visible. Hidden pages stop polling and +refresh on return. Failed or stale reads cannot display current readiness; +countdowns expire routes without renewing them. `GET/PUT /admin/codex-gateway-borrow/config` owns one JSON setting, `codex_gateway_borrow_config`: `enabled`, `source_account_ids`, @@ -126,9 +128,14 @@ The original cache identity includes route cookie, proxy, Authorization, ChatGPT-Account-ID, UA, Originator, Version, business STATE and TLS profile. The host adds actual model and fingerprint policy revision. Trace IDs, native session headers and business tier are not additional cache rules. STATE changes -therefore require fresh validation. A failed target cools that qualification for -15 seconds; it does not discard a source candidate or force a replacement attempt. -After cooldown the next request may validate again; missing or expired candidates +therefore require fresh validation. A completed rejection or protocol failure +cools that qualification for 15 seconds. A transport interruption instead retains failed evidence with +`retry_on_next_request`: the next independent request starts a fresh complete +two-shot validation through the existing shared flight, without a negative-cache +rejection. It never retries an uncertain observation within the failed request, +admits an incomplete probe or replays a business generation. Neither failure +discards a source candidate or extends its lease. After a rejection cooldown +the next request may validate again; missing or expired candidates are acquired on demand. There is no renewal timer, exit rotation, ordinary fallback or business replay. Business WS keeps its existing continuation anchors. @@ -168,7 +175,9 @@ separate from existing `count` (dispatches) and `applied_count`. `dispatched`, `failure_stage`, client and gateway request IDs distinguish a local rejection from an inference failure. Original target reasons survive failure cooldowns; Ops classifies local borrowing rejection as gateway/routing with upstream status -zero, retaining the compatible client 503 code. Business evidence is displayed +zero, retaining the compatible client 503 code. The event also retains probe +shot/transport evidence, next-request recovery policy and whether a rejection +came from the qualification cache. Business evidence is displayed ahead of diagnostics. Candidate expiry never implies a failed check is usable. `POST /admin/codex-gateway-borrow/diagnose` streams an explicit fixed-account diff --git a/backend/internal/service/codex_gateway_borrow.go b/backend/internal/service/codex_gateway_borrow.go index decee424920a..fac15dee56b5 100644 --- a/backend/internal/service/codex_gateway_borrow.go +++ b/backend/internal/service/codex_gateway_borrow.go @@ -47,6 +47,7 @@ var ( // request failing upstream. Callers must surface it before ordinary account // failover, stream-timeout or health hooks. Unwrap preserves the specific cause. type CodexGatewayBorrowFailure struct { + CacheHit bool Revision uint64 CookieFingerprint string Cause error @@ -81,7 +82,7 @@ func IsCodexGatewayBorrowFailure(err error) bool { return errors.As(err, &failure) } -func borrowVerificationFailure(check codexGatewayBorrowTargetCheck, revision uint64) error { +func borrowVerificationFailure(check codexGatewayBorrowTargetCheck, revision uint64) *CodexGatewayBorrowFailure { result := check.result failure := &CodexGatewayBorrowFailure{Revision: revision, Cause: ErrCodexGatewayBorrowUnavailable, Stage: "target_validation", Reason: result.Reason, Detail: result.Error, Verification: &result, CookieFingerprint: check.cookieKey} if !check.retryAfter.IsZero() { @@ -199,6 +200,9 @@ type CodexGatewayBorrowSourceStatus struct { } type CodexGatewayBorrowVerification struct { + RetryOnNextRequest bool `json:"retry_on_next_request,omitempty"` + FailedShot string `json:"failed_shot,omitempty"` + FailureKind string `json:"failure_kind,omitempty"` MintTransportPhase string `json:"mint_transport_phase,omitempty"` ContinueTransportPhase string `json:"continue_transport_phase,omitempty"` MintTransportAttempts int `json:"mint_transport_attempts,omitempty"` @@ -555,6 +559,8 @@ func (s *CodexGatewayBorrowService) Status() CodexGatewayBorrowStatus { row.State = "ready" } else if c.validating { row.State = "validating" + } else if c.result.RetryOnNextRequest { + row.State = "retry_pending" } else if c.result.Success { row.State, row.Reason = "expired", "route_expired" } @@ -940,7 +946,7 @@ func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, m acquisition.Phase = "failed" } check := codexGatewayBorrowTargetCheck{acquisition: acquisition, validationID: validationID, identityRevision: identityRevision, policyRevision: policyRevision, key: key, cookieKey: borrowHash(candidate.cookie.Value), expires: candidate.expires, result: result} - if !result.Success { + if !result.Success && !result.RetryOnNextRequest { check.retryAfter = time.Now().Add(codexGatewayBorrowFailureWait) check.acquisition.RetryAfter = &check.retryAfter } @@ -1006,8 +1012,10 @@ func (s *CodexGatewayBorrowService) cachedTarget(rev uint64, key codexGatewayBor } return old, true, nil } - if !old.result.Success && now.Before(old.retryAfter) { - return old, false, borrowVerificationFailure(old, rev) + if !old.result.Success && !old.result.RetryOnNextRequest && now.Before(old.retryAfter) { + failure := borrowVerificationFailure(old, rev) + failure.CacheHit = true + return old, false, failure } return old, false, nil } diff --git a/backend/internal/service/codex_gateway_borrow_core_test.go b/backend/internal/service/codex_gateway_borrow_core_test.go index 78e5145f760b..57ba9555f114 100644 --- a/backend/internal/service/codex_gateway_borrow_core_test.go +++ b/backend/internal/service/codex_gateway_borrow_core_test.go @@ -284,7 +284,7 @@ func TestCodexGatewayBorrowTargetCacheModelAndFingerprintIsolation(t *testing.T) } func TestCodexGatewayBorrowTargetChangedStateRouteAndFailureCooldown(t *testing.T) { - for _, mode := range []string{"state", "cookie", "missing", "incomplete"} { + for _, mode := range []string{"state", "cookie", "missing", "incomplete", "upstream_failed"} { t.Run(mode, func(t *testing.T) { var calls int a := borrowCoreAccount(2) @@ -300,6 +300,9 @@ func TestCodexGatewayBorrowTargetChangedStateRouteAndFailureCooldown(t *testing. if mode == "incomplete" { return &http.Response{StatusCode: 200, Header: http.Header{"X-Codex-Turn-State": []string{"first"}}, Body: io.NopCloser(strings.NewReader("data: [DONE]\n\n"))}, nil } + if mode == "upstream_failed" { + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("data: {\"type\":\"response.failed\"}\n\n"))}, nil + } state := "first" if calls == 2 { state = "changed" @@ -314,10 +317,16 @@ func TestCodexGatewayBorrowTargetChangedStateRouteAndFailureCooldown(t *testing. initial := calls _, _, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) require.Error(t, err) - require.Equal(t, initial, calls, "failure stays in 15-second cooldown") status := s.Status() require.False(t, status.Targets[1].CacheValid) - require.NotNil(t, status.Targets[1].RetryAfter) + if mode == "incomplete" { + require.Equal(t, initial+1, calls, "each independent request revalidates the incomplete stream") + require.Nil(t, status.Targets[1].RetryAfter) + require.True(t, status.Targets[1].RetryOnNextRequest) + } else { + require.Equal(t, initial, calls, "rejection stays in 15-second cooldown") + require.NotNil(t, status.Targets[1].RetryAfter) + } }) } } diff --git a/backend/internal/service/codex_gateway_borrow_http.go b/backend/internal/service/codex_gateway_borrow_http.go index 9dba1a81bb63..baee156f5ab1 100644 --- a/backend/internal/service/codex_gateway_borrow_http.go +++ b/backend/internal/service/codex_gateway_borrow_http.go @@ -12,6 +12,12 @@ import ( const CodexGatewayBorrowPreparationFailureReason GatewayFailureReason = "codex_gateway_borrow_preparation_failed" +type CodexGatewayBorrowFailureEvidence struct { + CacheHit bool `json:"cache_hit"` + RetryAfter *time.Time `json:"retry_after,omitempty"` + Verification *CodexGatewayBorrowVerification `json:"verification,omitempty"` +} + // The existing failover owner may select another account. A local preparation // failure is not an inference attempt or evidence against the selected account. func NewCodexGatewayBorrowRequestFailure(_ error) *UpstreamFailoverError { @@ -61,6 +67,11 @@ func RecordCodexGatewayBorrowPreparationFailure(c *gin.Context, account *Account event.ProxyID, event.ProxyName = opsUpstreamProxyID(account), opsUpstreamProxyName(account) event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name } + var failure *CodexGatewayBorrowFailure + if errors.As(err, &failure) && failure.Verification != nil { + verification := *failure.Verification + event.BorrowQualification = &CodexGatewayBorrowFailureEvidence{CacheHit: failure.CacheHit, RetryAfter: failure.RetryAfter, Verification: &verification} + } appendOpsUpstreamError(c, event) } diff --git a/backend/internal/service/codex_gateway_borrow_probe.go b/backend/internal/service/codex_gateway_borrow_probe.go index 910a58572d5f..633def15e889 100644 --- a/backend/internal/service/codex_gateway_borrow_probe.go +++ b/backend/internal/service/codex_gateway_borrow_probe.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "strings" "time" @@ -41,6 +42,7 @@ type codexGatewayBorrowObservation struct { errorText string complete bool incomplete bool + interrupted bool modelMismatch bool cookies []*http.Cookie cookiesObservedAt time.Time @@ -267,7 +269,7 @@ func (s *CodexGatewayBorrowService) probeTarget(ctx context.Context, template *h return result } if err != nil || !borrowShotUsable(mint) { - result.Error = borrowObservationError(mint, err) + borrowTargetObservationFailure(ctx, &result, "mint", mint, err) return result } if mint.state == "" { @@ -299,7 +301,7 @@ func (s *CodexGatewayBorrowService) probeTarget(ctx context.Context, template *h return result } if err != nil || !borrowShotUsable(continued) { - result.Error = borrowObservationError(continued, err) + borrowTargetObservationFailure(ctx, &result, "continue", continued, err) return result } result.NewTicket = continued.state != "" && continued.state != mint.state @@ -315,6 +317,28 @@ func borrowShotUsable(shot codexGatewayBorrowObservation) bool { return shot.status == http.StatusOK && shot.complete && shot.errorText == "" } +// A transport interruption is inconclusive, not a reusable rejection. Keep its +// evidence, but let the next independent request run a fresh complete pair. +// Never retry this observation or replay an uncertain business generation here. +func borrowTargetObservationFailure(ctx context.Context, result *CodexGatewayBorrowVerification, phase string, shot codexGatewayBorrowObservation, err error) { + result.Error = borrowObservationError(shot, err) + result.FailedShot, result.FailureKind = phase, "response" + if ctx.Err() != nil || errors.Is(err, context.Canceled) { + result.FailureKind = "cancelled" + return + } + if shot.interrupted { + result.FailureKind = "stream_incomplete" + result.RetryOnNextRequest = true + } + var transport *codexBorrowTransportError + var network net.Error + if err != nil && (errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &transport) || errors.As(err, &network)) { + result.FailureKind = "transport" + result.RetryOnNextRequest = true + } +} + func borrowObservationError(shot codexGatewayBorrowObservation, err error) string { if shot.errorText != "" { return shot.errorText @@ -505,6 +529,7 @@ func (s *CodexGatewayBorrowService) sendObservation(req *http.Request, account * // business sender's early-terminal optimization for this criterion. validationErr := validateCodexGatewayBorrowPinnedProbeResponse(data) shot.complete = validationErr == nil + shot.interrupted = errors.Is(validationErr, errCodexBorrowProbeNoCompletion) || errors.Is(validationErr, errCodexBorrowProbeUnterminated) if validationErr == nil { shot.errorText = "" } else if shot.errorText == "" { diff --git a/backend/internal/service/codex_gateway_borrow_probe_terminal.go b/backend/internal/service/codex_gateway_borrow_probe_terminal.go index e893435535bf..52f46752650e 100644 --- a/backend/internal/service/codex_gateway_borrow_probe_terminal.go +++ b/backend/internal/service/codex_gateway_borrow_probe_terminal.go @@ -13,6 +13,11 @@ import ( "strings" ) +var ( + errCodexBorrowProbeNoCompletion = errors.New("probe did not complete successfully") + errCodexBorrowProbeUnterminated = errors.New("unterminated probe event") +) + // A clean EOF or [DONE] is not evidence of a successful Responses generation. func validateCodexGatewayBorrowPinnedProbeResponse(body []byte) error { completed := false @@ -85,11 +90,11 @@ func validateCodexGatewayBorrowPinnedProbeResponse(body []byte) error { } // A trailing unterminated event may be truncated, even if its JSON parses. if len(data) > 0 { - return errors.New("unterminated probe event") + return errCodexBorrowProbeUnterminated } } if !completed { - return errors.New("probe did not complete successfully") + return errCodexBorrowProbeNoCompletion } return nil } diff --git a/backend/internal/service/codex_gateway_borrow_transient_test.go b/backend/internal/service/codex_gateway_borrow_transient_test.go new file mode 100644 index 000000000000..9b0900035d10 --- /dev/null +++ b/backend/internal/service/codex_gateway_borrow_transient_test.go @@ -0,0 +1,195 @@ +package service + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// Exercise real net/http framing: a response advertises more bytes than it +// delivers. Neither a partial stream nor a completed event followed by a +// transport error may qualify the target. Only a new, complete pair can do so. +func TestCodexGatewayBorrowTransientEOFFreshPairOnNextRequest(t *testing.T) { + for _, tc := range []struct { + name string + shot int32 + completedBeforeEOF bool + cleanEOF bool + }{{"mint", 1, false, false}, {"continue", 2, false, false}, {"completed_then_eof", 2, true, false}, {"clean_eof_without_completion", 1, false, true}, {"unterminated_completion", 2, true, true}} { + t.Run(tc.name, func(t *testing.T) { + var calls atomic.Int32 + var statesMu sync.Mutex + var states []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := calls.Add(1) + statesMu.Lock() + states = append(states, r.Header.Get("X-Codex-Turn-State")) + statesMu.Unlock() + w.Header().Set("X-Codex-Turn-State", "probe-state") + w.Header().Set("Content-Type", "text/event-stream") + body := borrowCoreSSE("gpt-6.1-sol", "OK") + if n == tc.shot { + if !tc.completedBeforeEOF { + body = "data: {\"type\":\"response.created\"}\n\n" + } + if tc.cleanEOF && tc.completedBeforeEOF { + body = strings.TrimSuffix(body, "\n\n") + } + if !tc.cleanEOF { + w.Header().Set("Content-Length", fmt.Sprint(len(body)+100)) + } + } + _, _ = io.WriteString(w, body) + })) + defer server.Close() + targetURL, err := url.Parse(server.URL) + require.NoError(t, err) + account := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(r *http.Request, _ string, id int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + if id != 2 { + return nil, fmt.Errorf("unexpected source acquisition") + } + wire := r.Clone(r.Context()) + wire.URL = targetURL + return server.Client().Do(wire) + }, account) + expires := time.Now().Add(codexGatewayBorrowTTL) + borrowCoreCandidate(s, expires) + _, req, _, err := s.accountTemplate(context.Background(), 2, "gpt-6.1-sol") + require.NoError(t, err) + req.Header.Set("X-Codex-Turn-State", "business-state") + _, app, err := s.Apply(req, account, "gpt-6.1-sol", "", nil, false) + require.Nil(t, app) + var failure *CodexGatewayBorrowFailure + require.ErrorAs(t, err, &failure) + require.Equal(t, "target_probe_failed", failure.Reason) + if !tc.cleanEOF { + require.Contains(t, failure.Detail, "unexpected EOF") + } + require.False(t, failure.CacheHit) + require.Nil(t, failure.RetryAfter) + require.True(t, failure.Verification.RetryOnNextRequest) + expectedKind := "transport" + if tc.cleanEOF { + expectedKind = "stream_incomplete" + } + require.Equal(t, expectedKind, failure.Verification.FailureKind) + expectedShot := "mint" + if tc.shot == 2 { + expectedShot = "continue" + } + require.Equal(t, expectedShot, failure.Verification.FailedShot) + require.Equal(t, tc.shot, calls.Load(), "the failed request never replays an uncertain probe") + status := s.Status().Targets[1] + require.Equal(t, "retry_pending", status.State) + require.False(t, status.CacheValid) + require.False(t, status.Success) + require.Nil(t, status.RetryAfter) + + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + RecordCodexGatewayBorrowPreparationFailure(c, account, failure) + events, exists := c.Get(OpsUpstreamErrorsKey) + require.True(t, exists) + records, ok := events.([]*OpsUpstreamErrorEvent) + require.True(t, ok) + require.Len(t, records, 1) + evidence := records[0].BorrowQualification + require.NotNil(t, evidence) + require.False(t, evidence.CacheHit) + require.Equal(t, expectedShot, evidence.Verification.FailedShot) + + // No sleep, manual Verify, STATE deletion or cached-proof override. + wire, app, err := s.Apply(req, account, "gpt-6.1-sol", "", nil, false) + require.NoError(t, err) + require.True(t, app.Applied) + require.True(t, app.Verification.MintCompleted) + require.True(t, app.Verification.ContinueCompleted) + require.False(t, app.Verification.RetryOnNextRequest) + require.Equal(t, expires, app.ExpiresAt, "recovery does not extend the lease") + require.Equal(t, tc.shot+2, calls.Load(), "both new shots are required") + statesMu.Lock() + lastStates := append([]string(nil), states[len(states)-2:]...) + statesMu.Unlock() + require.Equal(t, []string{"", "probe-state"}, lastStates) + require.Equal(t, "business-state", wire.Header.Get("X-Codex-Turn-State")) + require.Equal(t, "business-state", req.Header.Get("X-Codex-Turn-State")) + require.True(t, s.Status().Targets[1].CacheValid) + _, _, err = s.Apply(req, account, "gpt-6.1-sol", "", nil, false) + require.NoError(t, err) + require.Equal(t, tc.shot+2, calls.Load(), "a fully qualified proof remains reusable") + }) + } +} + +func TestCodexGatewayBorrowTransientRecoveryStillRejectsChangedState(t *testing.T) { + var calls int + a := borrowCoreAccount(2) + s := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { + calls++ + if calls == 1 { + return nil, io.ErrUnexpectedEOF + } + state := "first" + if req.Header.Get("X-Codex-Turn-State") != "" { + state = "changed" + } + return borrowCoreResponse("gpt-6.1-sol", "OK", state), nil + }, a) + borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL)) + _, req, _, err := s.accountTemplate(context.Background(), 2, "gpt-6.1-sol") + require.NoError(t, err) + _, app, err := s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.Error(t, err) + require.Nil(t, app) + _, app, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + var failure *CodexGatewayBorrowFailure + require.ErrorAs(t, err, &failure) + require.Nil(t, app) + require.Equal(t, "target_state_changed", failure.Reason) + require.False(t, failure.Verification.RetryOnNextRequest) + require.NotNil(t, failure.RetryAfter) + require.False(t, failure.CacheHit) + _, app, err = s.Apply(req, a, "gpt-6.1-sol", "", nil, false) + require.ErrorAs(t, err, &failure) + require.Nil(t, app) + require.True(t, failure.CacheHit) + require.Equal(t, 3, calls, "a real rejection still cools down without new probes") + require.Equal(t, "rejected", s.Status().Targets[1].State) +} + +func TestCodexGatewayBorrowTransientClassificationNeverGrantsSuccess(t *testing.T) { + for _, tc := range []struct { + name string + err error + cancelled bool + retry bool + }{ + {"eof", io.ErrUnexpectedEOF, false, true}, + {"caller_cancelled", io.ErrUnexpectedEOF, true, false}, + {"protocol_incomplete", nil, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.cancelled { + cancel() + } + var result CodexGatewayBorrowVerification + borrowTargetObservationFailure(ctx, &result, "mint", codexGatewayBorrowObservation{status: 200, raw: strings.Repeat(" ", 3)}, tc.err) + require.False(t, result.Success) + require.Equal(t, tc.retry, result.RetryOnNextRequest) + }) + } +} diff --git a/backend/internal/service/ops_upstream_context.go b/backend/internal/service/ops_upstream_context.go index 7f9db2fd27e2..673f1e1f7818 100644 --- a/backend/internal/service/ops_upstream_context.go +++ b/backend/internal/service/ops_upstream_context.go @@ -454,6 +454,8 @@ type OpsUpstreamErrorEvent struct { // Signature recovery evidence is bounded and contains fingerprints only; // bookkeeping must not replace the corresponding upstream failure. SignatureRecovery *ClaudeSignatureRecoveryDiagnostic `json:"signature_recovery,omitempty"` + // Probe evidence belongs to preparation, not to the business upstream status. + BorrowQualification *CodexGatewayBorrowFailureEvidence `json:"borrow_qualification,omitempty"` // SkipMonitoring is request-local rule state. It is intentionally excluded // from persisted attempt JSON. The logger consults it only when this event is diff --git a/frontend/src/api/admin/codexGatewayBorrow.ts b/frontend/src/api/admin/codexGatewayBorrow.ts index e74c114730fe..956ec36946f5 100644 --- a/frontend/src/api/admin/codexGatewayBorrow.ts +++ b/frontend/src/api/admin/codexGatewayBorrow.ts @@ -27,6 +27,9 @@ export interface BorrowAcquisition { } export interface BorrowTargetStatus extends BorrowSourceStatus { + retry_on_next_request?: boolean + failed_shot?: string + failure_kind?: string acquisition?: BorrowAcquisition model: string cache_valid: boolean @@ -88,6 +91,9 @@ export interface CodexGatewayBorrowStatus { } export interface BorrowVerification { + retry_on_next_request?: boolean + failed_shot?: string + failure_kind?: string account_id: number model: string success: boolean diff --git a/frontend/src/composables/useCodexBorrowUI.ts b/frontend/src/composables/useCodexBorrowUI.ts index 976b68b69e18..c4934c72379f 100644 --- a/frontend/src/composables/useCodexBorrowUI.ts +++ b/frontend/src/composables/useCodexBorrowUI.ts @@ -75,11 +75,12 @@ export function useCodexBorrowClock() { return { now, serverClockKnown, serverOffsetMs, receiveServerTime, cacheUsable, remainingSeconds } } -export function borrowTargetState(target: BorrowTargetStatus | undefined, now: number): 'ready' | 'expired' | 'validating' | 'waiting' | 'rejected' { +export function borrowTargetState(target: BorrowTargetStatus | undefined, now: number): 'ready' | 'expired' | 'validating' | 'waiting' | 'rejected' | 'retry_pending' { if (!target) return 'waiting' if (target.cache_valid && (!target.expires_at || Date.parse(target.expires_at) > now)) return 'ready' if (target.state === 'expired' || ((target.cache_valid || target.state === 'ready' || target.state === 'valid') && !!target.expires_at && Date.parse(target.expires_at) <= now)) return 'expired' if (target.state === 'validating' || target.reason === 'validating') return 'validating' + if (target.state === 'retry_pending' && target.retry_on_next_request) return 'retry_pending' if (['waiting', 'missing', 'disabled'].includes(target.state) || ['not_prepared', 'not_verified', 'not_configured', 'disabled'].includes(target.reason)) return 'waiting' return 'rejected' } diff --git a/frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts b/frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts index 3661ccedcd2f..72bf431d814f 100644 --- a/frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts +++ b/frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts @@ -53,21 +53,21 @@ export default { borrowStatus: 'Borrow status', onDemandHint: 'Saving enabled settings prepares once. The next business request acquires and validates missing or expired routes. There is no renewal timer; viewing or refreshing never calls a model.', onDemandExpired: 'Expired; waiting for a request to acquire a route', - modelScope: 'Astra follows ranxi v2.10.3. Sol is a downstream model extension and is validated independently. Cookie and two-shot STATE checks do not prove model identity or capability.', + modelScope: 'Astra follows ranxi v2.10.5. Sol is a downstream model extension and is validated independently. Cookie and two-shot STATE checks do not prove model identity or capability.', statusTitle: 'Codex Borrow Status', statusDescription: 'Inspect borrow routes for the saved configuration, then prepare or revalidate when needed.', preparationState: 'Preparation progress', idle: 'No preparation in progress', - readyPairs: 'Usable account / model pairs', + readyPairs: 'Pairs with valid validation records', pairCount: '{ready} / {total}', currentSource: 'Current source', candidateExpired: 'Source route expired', - validationMeaning: '“Route validation passed” means the borrowing conditions are met. Inspect artwork quality on the Pelican Comparison page.', + validationMeaning: 'An unexpired source route does not qualify every request. A proof applies only to the same account, model and request identity. A STATE or identity change requires validation before business dispatch.', prepareRoutes: 'Prepare borrow routes', prepareRoutesHint: 'Prepare the saved sources and targets, collecting a source route and validating account / model pairs in two rounds when needed.', revalidatePair: 'Revalidate this account / model', - revalidateHint: 'Revalidate this account and model with two requests, preparing a source route first when needed.', - pollingHint: 'While preparation or validation runs, refresh read-only status every 5 seconds, then stop. Expiry updates locally in the browser.', + revalidateHint: 'Each row shows the latest validation for an account and model. Other sessions or changed requests may need fresh validation. Manual validation sends two probes.', + pollingHint: 'While visible and borrowing is enabled, read status every 5 seconds. Pause in the background and refresh on return. Reads never call a model or renew a lease.', technicalDetails: 'Technical details', rawState: 'Server state label', rawReason: 'Raw reason', @@ -78,9 +78,11 @@ export default { noConfiguredTargets: 'No target accounts and models are saved yet. Complete Borrow Settings first.', preparationPending: 'Preparing borrow routes', lineStates: { + retry_pending: 'Probe interrupted; revalidation pending', + stale: 'Status needs refreshing', waiting: 'Not prepared', validating: 'Preparing', - ready: 'Route validation passed', + ready: 'Latest validation passed', expired: 'Expired', rejected: 'Validation failed', }, diff --git a/frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts b/frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts index 14e77b65a4d0..5cad61dcaa4b 100644 --- a/frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts +++ b/frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts @@ -53,21 +53,21 @@ export default { borrowStatus: '借用状态', onDemandHint: '保存启用配置后准备一次；线路缺失或过期时，由下一次业务请求自动获取并验证。没有定时续期。查看或刷新页面不会调用模型。', onDemandExpired: '已过期，等待请求自动获取', - modelScope: 'Astra 按 ranxi v2.10.3 原版验证；Sol 为本站模型扩展，单独验证。验证只检查路由 Cookie 与两轮 STATE，不证明模型身份或能力。', + modelScope: 'Astra 按 ranxi v2.10.5 原版验证；Sol 为本站模型扩展，单独验证。验证只检查路由 Cookie 与两轮 STATE，不证明模型身份或能力。', statusTitle: 'Codex 借用状态', statusDescription: '查看已保存配置的借用线路，按需准备或重新验证。', preparationState: '准备进度', idle: '当前没有准备任务', - readyPairs: '可使用的账号／模型', + readyPairs: '有有效验证记录的账号／模型', pairCount: '{ready} / {total}', currentSource: '当前来源', candidateExpired: '来源线路已过期', - validationMeaning: '“线路验证通过”表示满足借用条件；作品质量请在鹈鹕对比页观察。', + validationMeaning: '来源线路未到期不等于当前请求已获准。验证结果仅适用于相同账号、模型和请求身份；STATE 或身份变化时会重新验证。只有验证通过才发送业务。', prepareRoutes: '准备借用线路', prepareRoutesHint: '准备已保存的来源与目标；需要时采集来源线路，并对账号／模型执行两轮验证。', revalidatePair: '重新验证该账号／模型', - revalidateHint: '重新验证这一账号与模型（两轮请求）；需要时先准备来源线路。', - pollingHint: '准备或验证期间每 5 秒只读刷新状态；结束后停止。有效期在浏览器本地更新。', + revalidateHint: '此处显示每个账号与模型最近一次验证，不能保证其他会话或变化后的请求也能复用。手动验证会发送两轮探针。', + pollingHint: '页面可见且借用启用时，每 5 秒只读刷新。切到后台暂停，返回时立即刷新；读取状态不会发模型请求或续租。', technicalDetails: '技术详情', rawState: '服务端状态标识', rawReason: '原始原因', @@ -78,9 +78,11 @@ export default { noConfiguredTargets: '尚未保存目标账号与模型，请先完成借用设置。', preparationPending: '正在准备借用线路', lineStates: { + retry_pending: '验证中断，待重验', + stale: '状态待刷新', waiting: '尚未准备', validating: '准备中', - ready: '线路验证通过', + ready: '最近一次验证通过', expired: '已过期', rejected: '验证失败', }, diff --git a/frontend/src/views/admin/CodexGatewayBorrowStatusView.vue b/frontend/src/views/admin/CodexGatewayBorrowStatusView.vue index 39d21f106db6..c75c0ffd24fe 100644 --- a/frontend/src/views/admin/CodexGatewayBorrowStatusView.vue +++ b/frontend/src/views/admin/CodexGatewayBorrowStatusView.vue @@ -32,7 +32,7 @@
{{ status.enabled ? t('admin.codexGatewayBorrow.on') : t('admin.codexGatewayBorrow.off') }}
-
{{ text('线路就绪情况', 'Route readiness') }}
+
{{ text('最近验证情况', 'Latest validation') }}
{{ preparationLabel }}
@@ -68,7 +68,8 @@ 
{{ status.acquisition.error }}
-

{{ t('admin.codexGatewayBorrow.validationMeaning') }}

+

{{ t('admin.codexGatewayBorrow.validationMeaning') }}

+

{{ text('状态尚未更新，以下为上次记录，不能确认当前可用。', 'Status is out of date. The last records below do not establish current availability.') }}

{{ t('admin.codexGatewayBorrow.modelScope') }}

{{ t('admin.codexGatewayBorrow.checkedAt') }}
{{ formatTime(target.checked_at) }}
@@ -188,6 +190,8 @@ const status = ref(null) const loading = ref(true) const refreshing = ref(false) const statusError = ref('') +const statusReadFailed = ref(false) +const pageVisible = ref(document.visibilityState !== 'hidden') const inventoryError = ref('') const actionBusy = ref('') const actionFailure = ref<{ message: string; detail: string } | null>(null) @@ -205,17 +209,22 @@ const targets = computed(() => { } )) }) -const readyCount = computed(() => status.value?.enabled ? targets.value.filter(target => cacheUsable(target)).length : 0) +const statusStale = computed(() => !!status.value && (statusReadFailed.value || now.value - Date.parse(status.value.generated_at) >= 15_000)) +const readyCount = computed(() => status.value?.enabled && !statusStale.value ? targets.value.filter(target => cacheUsable(target)).length : 0) const candidateRemaining = computed(() => remainingSeconds(status.value?.candidate?.expires_at)) const serverValidating = computed(() => targets.value.some(target => target.state === 'validating' || target.reason === 'validating')) -const canAct = computed(() => !!status.value?.enabled && !loading.value && !actionBusy.value && !status.value.preparing && !serverValidating.value) -const shouldPoll = computed(() => !!status.value?.preparing || !!actionBusy.value || serverValidating.value) +const canAct = computed(() => !!status.value?.enabled && !statusStale.value && !loading.value && !actionBusy.value && !status.value.preparing && !serverValidating.value) +// Business requests can invalidate a proof after this page first becomes idle. +// These reads never prepare a route or send a model request. +const shouldPoll = computed(() => pageVisible.value && (!!status.value?.enabled || !!status.value?.preparing || !!actionBusy.value)) const preparationLabel = computed(() => { + if (statusStale.value) return text('状态待刷新', 'Status needs refreshing') if (status.value?.preparing || actionBusy.value === 'prepare') return t('admin.codexGatewayBorrow.preparationPending') if (actionBusy.value || serverValidating.value) return t('admin.codexGatewayBorrow.validating') if (!status.value?.enabled || !targets.value.length) return text('未配置或未启用', 'Not configured or disabled') - if (readyCount.value === targets.value.length) return text('线路可用', 'Routes ready') - if (readyCount.value > 0) return text('部分线路可用', 'Some routes ready') + if (readyCount.value === targets.value.length) return text('最近验证均通过', 'Latest validations passed') + if (readyCount.value > 0) return text('部分最近验证通过', 'Some latest validations passed') + if (targets.value.some(target => targetState(target) === 'retry_pending')) return text('探针中断，等待下一请求重验', 'Probe interrupted; awaiting the next request') if (targets.value.some(target => targetState(target) === 'expired')) return t('admin.codexGatewayBorrow.onDemandExpired') return text('尚无可用线路，查看下方原因', 'No ready route; see reasons below') }) @@ -237,6 +246,7 @@ function formatTime(value?: string) { } function targetState(target: BorrowTargetStatus) { if (!status.value?.enabled) return 'waiting' + if (statusStale.value) return 'stale' const state = borrowTargetState(target, now.value) if (state !== 'ready' && actionBusy.value === pairKey(target.account_id, target.model)) return 'validating' if (state === 'waiting' && (status.value.preparing || actionBusy.value === 'prepare')) return 'validating' @@ -245,6 +255,7 @@ function targetState(target: BorrowTargetStatus) { function reasonLabel(target: BorrowTargetStatus) { if (!status.value?.enabled) return borrowReasonLabel('disabled', t) const state = targetState(target) + if (state === 'stale') return text('上次读取的结果，等待更新', 'Last observed result; awaiting an update') if (state === 'expired') return borrowReasonLabel('route_expired', t) if (state === 'validating' && (actionBusy.value || target.state === 'waiting')) return borrowReasonLabel('validating', t) return borrowReasonLabel(target.reason, t) @@ -252,7 +263,7 @@ function reasonLabel(target: BorrowTargetStatus) { function stateClass(state: string) { if (state === 'ready') return 'bg-emerald-50 text-emerald-800 dark:bg-emerald-950 dark:text-emerald-200' if (state === 'rejected') return 'bg-red-50 text-red-800 dark:bg-red-950 dark:text-red-200' - if (state === 'expired') return 'bg-amber-50 text-amber-800 dark:bg-amber-950 dark:text-amber-200' + if (['expired', 'retry_pending', 'stale'].includes(state)) return 'bg-amber-50 text-amber-800 dark:bg-amber-950 dark:text-amber-200' return 'bg-gray-100 text-gray-700 dark:bg-dark-700 dark:text-gray-200' } function receiveStatus(value: CodexGatewayBorrowStatus) { @@ -261,7 +272,8 @@ function receiveStatus(value: CodexGatewayBorrowStatus) { if (current && value.revision < current.revision) return if (current && value.revision === current.revision && Date.parse(value.generated_at) < Date.parse(current.generated_at)) return status.value = value - receiveServerTime(value.generated_at) + statusReadFailed.value = false + if (!current || value.generated_at !== current.generated_at) receiveServerTime(value.generated_at) } function clearPollTimer() { if (pollTimer !== undefined) clearTimeout(pollTimer) @@ -283,7 +295,12 @@ function refreshStatus(): Promise { statusError.value = '' statusRequest = (async () => { try { receiveStatus(await codexGatewayBorrowAPI.getStatus(lifecycle.signal)) } - catch (value) { if (!lifecycle.signal.aborted) statusError.value = extractApiErrorMessage(value, t('admin.codexGatewayBorrow.statusLoadFailed')) } + catch (value) { + if (!lifecycle.signal.aborted) { + statusReadFailed.value = true + statusError.value = extractApiErrorMessage(value, t('admin.codexGatewayBorrow.statusLoadFailed')) + } + } finally { refreshing.value = false statusRequest = undefined @@ -328,13 +345,19 @@ async function verify(target: BorrowTargetStatus) { } watch(shouldPoll, schedulePoll) +function visibilityChanged() { + pageVisible.value = document.visibilityState !== 'hidden' + if (pageVisible.value) void refreshStatus() + else clearPollTimer() +} onMounted(async () => { + document.addEventListener('visibilitychange', visibilityChanged) const outcomes = await Promise.allSettled([refreshStatus(), loadAccounts()]) if (lifecycle.signal.aborted) return if (outcomes[1].status === 'rejected') inventoryError.value = extractApiErrorMessage(outcomes[1].reason, t('admin.codexGatewayBorrow.inventoryLoadFailed')) loading.value = false }) -onBeforeUnmount(() => { lifecycle.abort(); clearPollTimer() }) +onBeforeUnmount(() => { lifecycle.abort(); clearPollTimer(); document.removeEventListener('visibilitychange', visibilityChanged) }) defineExpose({ refreshStatus }) diff --git a/frontend/src/views/admin/__tests__/CodexGatewayBorrowStatusView.spec.ts b/frontend/src/views/admin/__tests__/CodexGatewayBorrowStatusView.spec.ts index ec846842aafb..3a5092f4214c 100644 --- a/frontend/src/views/admin/__tests__/CodexGatewayBorrowStatusView.spec.ts +++ b/frontend/src/views/admin/__tests__/CodexGatewayBorrowStatusView.spec.ts @@ -73,10 +73,66 @@ beforeEach(() => { }) afterEach(() => { wrappers.splice(0).forEach(wrapper => wrapper.unmount()) + vi.restoreAllMocks() vi.useRealTimers() }) describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { + it('replaces an idle green result with interrupted validation before the route expires', async () => { + const initial = makeStatus() + const interrupted = makeStatus() + interrupted.targets[1] = { ...interrupted.targets[1], state: 'retry_pending', cache_valid: false, + reason: 'target_probe_failed', retry_on_next_request: true, failed_shot: 'continue', failure_kind: 'transport', error: 'unexpected EOF' } + mocks.getStatus.mockResolvedValueOnce(initial).mockImplementation(async () => ({ ...interrupted, generated_at: new Date().toISOString() })) + const wrapper = await mountView() + expect(wrapper.get('[data-test="borrow-status-2-gpt-6.1-sol"] [data-test="borrow-line-state"]').text()).toContain('lineStates.ready') + await vi.advanceTimersByTimeAsync(5000) + const row = wrapper.get('[data-test="borrow-status-2-gpt-6.1-sol"]') + expect(row.get('[data-test="borrow-line-state"]').text()).toContain('lineStates.retry_pending') + expect(row.get('[data-test="borrow-retry-policy"]').text()).toContain('完整验证') + expect(row.get('[data-test="borrow-line-remaining"]').text()).toContain('"seconds":55') + expect(wrapper.get('[data-test="borrow-ready-count"]').text()).toContain('"ready":1') + expect(wrapper.get('[data-test="borrow-validation-scope"]').text()).toContain('validationMeaning') + expect(mocks.prepare).not.toHaveBeenCalled() + expect(mocks.verify).not.toHaveBeenCalled() + }) + + it('removes current-readiness claims after a failed or stalled status read', async () => { + const pending = deferred() + mocks.getStatus.mockResolvedValueOnce(makeStatus()).mockRejectedValueOnce(new Error('status unavailable')).mockImplementationOnce(() => pending.promise) + const wrapper = await mountView() + await vi.advanceTimersByTimeAsync(5000) + expect(wrapper.get('[data-test="borrow-status-stale"]').text()).toContain('不能确认当前可用') + expect(wrapper.get('[data-test="borrow-ready-count"]').text()).toContain('"ready":0') + await vi.advanceTimersByTimeAsync(15_000) + expect(wrapper.get('[data-test="borrow-status-2-gpt-6.1-sol"] [data-test="borrow-line-state"]').text()).toContain('lineStates.stale') + pending.resolve(makeStatus()) + await flushPromises() + expect(wrapper.find('[data-test="borrow-status-stale"]').exists()).toBe(false) + expect(wrapper.get('[data-test="borrow-ready-count"]').text()).toContain('"ready":2') + expect(mocks.prepare).not.toHaveBeenCalled() + expect(mocks.verify).not.toHaveBeenCalled() + }) + + it('pauses reads while hidden and refreshes immediately on return without probing', async () => { + const visibility = vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('visible') + const wrapper = await mountView() + visibility.mockReturnValue('hidden') + document.dispatchEvent(new Event('visibilitychange')) + await vi.advanceTimersByTimeAsync(20_000) + expect(mocks.getStatus).toHaveBeenCalledOnce() + visibility.mockReturnValue('visible') + document.dispatchEvent(new Event('visibilitychange')) + await flushPromises() + expect(mocks.getStatus).toHaveBeenCalledTimes(2) + wrapper.unmount() + document.dispatchEvent(new Event('visibilitychange')) + await vi.advanceTimersByTimeAsync(20_000) + expect(mocks.getStatus).toHaveBeenCalledTimes(2) + expect(mocks.prepare).not.toHaveBeenCalled() + expect(mocks.verify).not.toHaveBeenCalled() + }) + it('mounts with saved status and local inventory only, and retains collapsed technical details in full', async () => { const fullError = `HTTP 403\n${'Complete synthetic upstream detail '.repeat(1800)}` const value = makeStatus() @@ -102,7 +158,7 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { expectNoUnrelatedCalls() }) - it('expires ready routes using the server-aligned local clock without polling or preparing', async () => { + it('expires routes locally while continuing read-only observation without preparing', async () => { const value = makeStatus() value.generated_at = new Date(startTime + 120_000).toISOString() value.candidate!.expires_at = new Date(startTime + 123_000).toISOString() @@ -117,13 +173,13 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { expect(wrapper.get('[data-test="borrow-status-2-gpt-6-astra"] [data-test="borrow-line-reason"]').text()).toContain('reasons.route_expired') expect(wrapper.get('[data-test="borrow-candidate"]').text()).toContain('candidateExpired') await vi.advanceTimersByTimeAsync(20_000) - expect(mocks.getStatus).toHaveBeenCalledOnce() + expect(mocks.getStatus).toHaveBeenCalledTimes(5) expect(mocks.prepare).not.toHaveBeenCalled() expect(mocks.verify).not.toHaveBeenCalled() expectNoUnrelatedCalls() }) - it('polls every 5 seconds while preparing, never overlaps a slow read, and stops when preparation ends', async () => { + it('polls every 5 seconds without overlapping slow reads and keeps observing after preparation', async () => { const slowRead = deferred() mocks.getStatus.mockResolvedValueOnce(makeStatus({ preparing: true })).mockImplementationOnce(() => slowRead.promise).mockImplementation(async () => makeStatus()) const wrapper = await mountView() @@ -138,9 +194,9 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { await flushPromises() await vi.advanceTimersByTimeAsync(5000) expect(mocks.getStatus).toHaveBeenCalledTimes(3) - expect(wrapper.get('[data-test="borrow-preparation-status"]').text()).toContain('部分线路可用') + expect(wrapper.get('[data-test="borrow-preparation-status"]').text()).toContain('部分最近验证通过') await vi.advanceTimersByTimeAsync(20_000) - expect(mocks.getStatus).toHaveBeenCalledTimes(3) + expect(mocks.getStatus).toHaveBeenCalledTimes(7) expect(mocks.prepare).not.toHaveBeenCalled() expect(mocks.verify).not.toHaveBeenCalled() expectNoUnrelatedCalls() @@ -173,12 +229,12 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { expect(wrapper.get('[data-test="borrow-action-failure"] details').attributes('open')).toBeUndefined() expect(mocks.getStatus).toHaveBeenCalledTimes(5) await vi.advanceTimersByTimeAsync(20_000) - expect(mocks.getStatus).toHaveBeenCalledTimes(5) + expect(mocks.getStatus).toHaveBeenCalledTimes(9) expect(mocks.verify).toHaveBeenCalledOnce() expectNoUnrelatedCalls() }) - it('polls server validation started in another tab and stops when its terminal status arrives', async () => { + it('observes external validation and keeps polling after its terminal status', async () => { const active = makeStatus() active.targets[3] = { ...active.targets[3], state: 'validating', reason: 'validating' } const terminal = makeStatus() @@ -193,11 +249,11 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { expect(mocks.getStatus).toHaveBeenCalledOnce() await vi.advanceTimersByTimeAsync(1) expect(mocks.getStatus).toHaveBeenCalledTimes(2) - expect(wrapper.get('[data-test="borrow-preparation-status"]').text()).toContain('部分线路可用') + expect(wrapper.get('[data-test="borrow-preparation-status"]').text()).toContain('部分最近验证通过') expect(wrapper.get('[data-test="borrow-status-3-gpt-6.1-sol"] [data-test="borrow-line-state"]').text()).toContain('lineStates.ready') expect(wrapper.get('[data-test="borrow-prepare"]').element.disabled).toBe(false) await vi.advanceTimersByTimeAsync(10_000) - expect(mocks.getStatus).toHaveBeenCalledTimes(2) + expect(mocks.getStatus).toHaveBeenCalledTimes(4) expect(mocks.prepare).not.toHaveBeenCalled() expect(mocks.verify).not.toHaveBeenCalled() expectNoUnrelatedCalls() @@ -256,7 +312,7 @@ describe('CodexGatewayBorrowStatusView request and expiry boundaries', () => { expect(wrapper.get('[data-test="borrow-status-2-gpt-6.1-sol"] [data-test="borrow-line-state"]').text()).toContain('lineStates.rejected') expect(wrapper.get('[data-test="borrow-ready-count"]').text()).toContain('"ready":1') await vi.advanceTimersByTimeAsync(10_000) - expect(mocks.getStatus).toHaveBeenCalledTimes(3) + expect(mocks.getStatus).toHaveBeenCalledTimes(5) expectNoUnrelatedCalls() })