Fix Codex borrow probe identity and blocked-request evidence

.downstream/native-domains.md

@@ -100,6 +100,15 @@ and has a 90-second budget. Each target/model is validated with two completed
 HTTP 200 streams, at most 45 seconds each: the first must return STATE, and
 the second may omit STATE or return the same value. Qualification includes
 the target identity, actual model, exit, client headers, STATE and TLS profile.
+Target probes preserve the template's native session/thread/window headers and
+generate only a fresh legacy `session_id` for each shot, matching ranxi's full
+probe contract. The effective service tier in the gateway-owned routing hint is
+also included in the probe body. Proofs include native identity and routing
+headers; per-request tracing IDs do not invalidate them. Manual templates use a
+stable synthetic window per account/model so cached-only generation can reuse
+their proof without changing actual client windows. The pinned upstream's
+HTTP borrowing is Astra-only: Sol is a downstream extension, independently
+validated against Sol rather than certified by an Astra pass.
 Source and target probes use separate HTTP/TLS pools. Business WS connections
 stay in the ordinary account pool, with fixed one-hour continuation anchors.
 
@@ -125,6 +134,13 @@ cancellation; that detail remains available as `read_error`. Captures are snapsh
 under a lock, and the usage index preserves the first terminal event.
 Diagnostic and business requests are distinguished. This index contains no request
 payloads or credential/cookie values; the existing administrator error records remain.
+Attempts now start before preparation: `attempt_count` and `blocked_count` are
+separate from existing `count` (dispatches) and `applied_count`. `dispatched`,
+`failure_stage`, client and gateway request IDs distinguish a local rejection
+from an inference failure. Original target reasons survive failure cooldowns;
+Ops classifies local borrowing rejection as gateway/routing with upstream status
+zero, retaining the compatible client 503 code. Business evidence is displayed
+ahead of diagnostics. Candidate expiry never implies a failed check is usable.
 
 `POST /admin/codex-gateway-borrow/diagnose` streams an explicit fixed-account
 comparison with `account_id`, `model`, `transport` (`http` or `ws`), and optional
@@ -138,6 +154,15 @@ completion, actual borrow application and reported model. Cancellation releases
 owned work and sockets. The UI retains the current run and can download JSON;
 there is no new table or migration. These observations do not prove intelligence
 or physical model identity. Normal token refresh remains permitted.
+Optional `scenario: codex_session` checks HTTP native client metadata, a fixed
+side-effect-free echo tool call, returned STATE and complete-history continuation.
+It defaults to the borrowed path to fit preparation and two business turns within
+the same eight-request bound. Optional `mode` selects `ordinary` or `borrowed`;
+omitting it for the original short scenario retains both-path comparison.
+Optional `service_tier` exercises the effective routing tier. Results include the
+specific verification (shape, actual model/tier, both completions, STATE lengths
+and route replacement), failure stage and tool-round-trip result. HTTP diagnostic
+history and STATE live only within that invocation, never in client sessions.
 
 Ranxi reference: v2.10.2 (`d3e43f2de33af9e987cffa511d76dfabbcd749da`). Its five
 fingerprint, gateway-cookie, target-probe and automatic-setup core files are

backend/internal/handler/admin/codex_gateway_borrow_handler.go

@@ -277,7 +277,7 @@ func (h *CodexGatewayBorrowHandler) Diagnose(c *gin.Context) {
 		response.BadRequest(c, err.Error())
 		return
 	}
-	middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_diagnostic", "account_id": request.AccountID, "model": request.Model, "transport": request.Transport})
+	middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_diagnostic", "account_id": request.AccountID, "model": request.Model, "transport": request.Transport, "scenario": request.Scenario, "mode": request.Mode, "service_tier": request.ServiceTier})
 	c.Header("Content-Type", "text/event-stream")
 	c.Header("X-Accel-Buffering", "no")
 	c.Status(http.StatusOK)

backend/internal/handler/ops_codex_borrow_test.go

@@ -0,0 +1,29 @@
+package handler
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/Wei-Shaw/sub2api/internal/service"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOpsCodexBorrowPreparationIsNotAnUpstream503(t *testing.T) {
+	c, _ := gin.CreateTestContext(httptest.NewRecorder())
+	c.Request = httptest.NewRequest("POST", "/v1/responses", nil)
+	err := &service.CodexGatewayBorrowFailure{Cause: service.ErrCodexGatewayBorrowUnavailable, Stage: "target_validation", Reason: "target_state_changed", Detail: "full diagnostic detail"}
+	service.RecordCodexGatewayBorrowPreparationFailure(c, &service.Account{ID: 2, Platform: "openai"}, err)
+	phase, limited, owner, source := classifyOpsErrorLog(c, "upstream_error", "no qualified Codex gateway borrow route is available", "CODEX_GATEWAY_BORROW_UNAVAILABLE", 503)
+	require.Equal(t, "routing", phase)
+	require.False(t, limited, "a platform failure still counts as a failed client request")
+	require.Equal(t, "platform", owner)
+	require.Equal(t, "gateway", source)
+	entry := &service.OpsInsertErrorLogInput{}
+	applyOpsUpstreamFieldsFromContext(c, entry)
+	require.NotNil(t, entry.UpstreamStatusCode)
+	require.Zero(t, *entry.UpstreamStatusCode)
+	require.Len(t, entry.UpstreamErrors, 1)
+	require.Equal(t, "target_state_changed", entry.UpstreamErrors[0].Reason)
+	require.Contains(t, *entry.UpstreamErrorDetail, "full diagnostic detail")
+}

backend/internal/handler/ops_error_logger.go

@@ -1796,7 +1796,7 @@ func applyOpsUpstreamErrorEvents(entry *service.OpsInsertErrorLogInput, events [
 	entry.UpstreamStatusCode = nil
 	entry.UpstreamErrorMessage = nil
 	entry.UpstreamErrorDetail = nil
-	if last.Stage == string(service.GatewayFailureStageAccountAuth) {
+	if last.Stage == string(service.GatewayFailureStageAccountAuth) || last.Kind == string(service.CodexGatewayBorrowPreparationFailureReason) {
 		code := 0
 		entry.UpstreamStatusCode = &code
 	} else if last.UpstreamStatusCode > 0 {
@@ -2272,6 +2272,9 @@ func normalizeOpsErrorType(errType string, code string) string {
 }
 
 func classifyOpsPhase(errType, message, code string) string {
+	if code == "CODEX_GATEWAY_BORROW_UNAVAILABLE" {
+		return "routing"
+	}
 	msg := strings.ToLower(message)
 	// Standardized phases: request|auth|account_auth|routing|upstream|network|internal
 	// Map billing/concurrency/response => request; scheduling => routing.
@@ -2324,6 +2327,11 @@ func classifyOpsSeverity(errType string, status int) string {
 }
 
 func classifyOpsErrorLog(c *gin.Context, errType, message, code string, status int) (phase string, isBusinessLimited bool, errorOwner string, errorSource string) {
+	if code == "CODEX_GATEWAY_BORROW_UNAVAILABLE" {
+		// A local qualification rejection is a real client failure, but no
+		// inference request was dispatched and no upstream HTTP 503 was seen.
+		return "routing", false, "platform", "gateway"
+	}
 	phase = classifyOpsPhase(errType, message, code)
 	routingCapacityLimited := isOpsRoutingCapacityLimited(c)
 	clientBusinessLimited := service.HasOpsClientBusinessLimited(c)

backend/internal/service/codex_gateway_borrow.go

@@ -46,13 +46,27 @@ var (
 // A local route qualification failure is distinct from an actual business
 // request failing upstream. Callers must surface it before ordinary account
 // failover, stream-timeout or health hooks. Unwrap preserves the specific cause.
-type CodexGatewayBorrowFailure struct{ Cause error }
+type CodexGatewayBorrowFailure struct {
+	Cause        error
+	Stage        string
+	Reason       string
+	Detail       string
+	RetryAfter   *time.Time
+	Verification *CodexGatewayBorrowVerification
+}
 
 func (e *CodexGatewayBorrowFailure) Error() string {
 	if e == nil || e.Cause == nil {
 		return "Codex gateway borrow qualification failed"
 	}
-	return "Codex gateway borrow qualification failed: " + e.Cause.Error()
+	detail := e.Cause.Error()
+	if e.Detail != "" {
+		detail += ": " + e.Detail
+	}
+	if e.Reason != "" {
+		detail = e.Stage + "/" + e.Reason + ": " + detail
+	}
+	return "Codex gateway borrow qualification failed: " + detail
 }
 func (e *CodexGatewayBorrowFailure) Unwrap() error {
 	if e == nil {
@@ -65,6 +79,33 @@ func IsCodexGatewayBorrowFailure(err error) bool {
 	return errors.As(err, &failure)
 }
 
+func borrowVerificationFailure(check codexGatewayBorrowTargetCheck) error {
+	result := check.result
+	failure := &CodexGatewayBorrowFailure{Cause: ErrCodexGatewayBorrowUnavailable, Stage: "target_validation", Reason: result.Reason, Detail: result.Error, Verification: &result}
+	if !check.retryAfter.IsZero() {
+		retry := check.retryAfter
+		failure.RetryAfter = &retry
+	}
+	return failure
+}
+
+func codexBorrowFailureStage(err error) (string, string) {
+	var failure *CodexGatewayBorrowFailure
+	if errors.As(err, &failure) && failure.Reason != "" {
+		return failure.Stage, failure.Reason
+	}
+	if errors.Is(err, ErrCodexGatewayBorrowChanged) {
+		return "qualification", "configuration_changed"
+	}
+	if errors.Is(err, context.Canceled) {
+		return "preparation", "cancelled"
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "preparation", "deadline_exceeded"
+	}
+	return "preparation", "borrow_unavailable"
+}
+
 // CodexGatewayBorrowProbeUpstream is deliberately a distinct injection type.
 // Its implementation must own separate clients, pools and TLS caches, including
 // separation between source acquisition and target observation.
@@ -155,18 +196,25 @@ type CodexGatewayBorrowSourceStatus struct {
 }
 
 type CodexGatewayBorrowVerification struct {
-	AccountID      int64      `json:"account_id"`
-	Model          string     `json:"model"`
-	Success        bool       `json:"success"`
-	Reason         string     `json:"reason"`
-	Error          string     `json:"error,omitempty"`
-	MintStatus     int        `json:"mint_status"`
-	ContinueStatus int        `json:"continue_status"`
-	Minted         bool       `json:"minted"`
-	NewTicket      bool       `json:"new_ticket"`
-	ReportedModel  string     `json:"reported_model,omitempty"`
-	CheckedAt      time.Time  `json:"checked_at"`
-	ExpiresAt      *time.Time `json:"expires_at,omitempty"`
+	RequestShape        string     `json:"request_shape,omitempty"`
+	ServiceTier         string     `json:"service_tier,omitempty"`
+	MintCompleted       bool       `json:"mint_completed"`
+	ContinueCompleted   bool       `json:"continue_completed"`
+	MintStateLength     int        `json:"mint_state_length"`
+	ContinueStateLength int        `json:"continue_state_length"`
+	RouteChanged        bool       `json:"route_changed"`
+	AccountID           int64      `json:"account_id"`
+	Model               string     `json:"model"`
+	Success             bool       `json:"success"`
+	Reason              string     `json:"reason"`
+	Error               string     `json:"error,omitempty"`
+	MintStatus          int        `json:"mint_status"`
+	ContinueStatus      int        `json:"continue_status"`
+	Minted              bool       `json:"minted"`
+	NewTicket           bool       `json:"new_ticket"`
+	ReportedModel       string     `json:"reported_model,omitempty"`
+	CheckedAt           time.Time  `json:"checked_at"`
+	ExpiresAt           *time.Time `json:"expires_at,omitempty"`
 }
 
 type CodexGatewayBorrowTargetStatus struct {
@@ -195,6 +243,7 @@ type CodexGatewayBorrowStatus struct {
 // Application contains only opaque identities. The selected cookie itself is
 // available solely in the cloned outbound request's Cookie header.
 type CodexGatewayBorrowApplication struct {
+	Verification      CodexGatewayBorrowVerification
 	Applied           bool
 	SourceAccountID   int64
 	ExpiresAt         time.Time
@@ -708,7 +757,14 @@ func (s *CodexGatewayBorrowService) prepareSource(ctx context.Context, rev uint6
 
 func borrowRequestFingerprint(req *http.Request, model, proxy string, profile *tlsfingerprint.Profile, cookie string) string {
 	tlsJSON, _ := json.Marshal(profile)
-	identity, _ := json.Marshal([]string{cookie, model, proxy, req.URL.EscapedPath(), req.Header.Get("Authorization"), req.Header.Get("ChatGPT-Account-ID"), req.Header.Get("User-Agent"), req.Header.Get("Originator"), req.Header.Get("Version"), req.Header.Get("X-Codex-Turn-State"), string(tlsJSON)})
+	values := []string{cookie, model, proxy, req.URL.EscapedPath(), req.Header.Get("Authorization"), req.Header.Get("ChatGPT-Account-ID"), req.Header.Get("User-Agent"), req.Header.Get("Originator"), req.Header.Get("Version"), req.Header.Get("X-Codex-Turn-State"), string(tlsJSON)}
+	// Include the native identity and routing policy preserved by the probe.
+	// Per-request tracing IDs and the probe's random legacy session_id are not
+	// qualification identities and must not defeat reuse or failure cooldowns.
+	for _, name := range []string{"session-id", "thread-id", "x-codex-window-id", "x-codex-installation-id", "x-codex-beta-features", openAICodexRoutingHintHeader} {
+		values = append(values, req.Header.Get(name))
+	}
+	identity, _ := json.Marshal(values)
 	return borrowHash(string(identity))
 }
 
@@ -724,6 +780,21 @@ func (s *CodexGatewayBorrowService) Apply(req *http.Request, account *Account, m
 	if err != nil && !IsCodexGatewayBorrowFailure(err) {
 		err = &CodexGatewayBorrowFailure{Cause: err}
 	}
+	var diagnostic *codexBorrowDiagnostic
+	if req != nil {
+		diagnostic = borrowDiagnosticFromContext(req.Context())
+	}
+	if d := diagnostic; d != nil {
+		if application != nil {
+			result := application.Verification
+			d.verification.Store(&result)
+		} else {
+			var failure *CodexGatewayBorrowFailure
+			if errors.As(err, &failure) {
+				d.verification.Store(failure.Verification)
+			}
+		}
+	}
 	return wire, application, err
 }
 
@@ -750,7 +821,7 @@ func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, m
 			return req, nil, ErrCodexGatewayBorrowUnavailable
 		}
 		if err := s.prepareSource(req.Context(), rev, revisionCtx); err != nil {
-			return req, nil, err
+			return req, nil, &CodexGatewayBorrowFailure{Cause: err, Stage: "source_preparation", Reason: "source_prepare_failed"}
 		}
 		s.mu.Lock()
 		candidate = s.candidate
@@ -768,7 +839,7 @@ func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, m
 			if err != nil {
 				return req, nil, err
 			}
-			return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires), nil
+			return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires, check.result), nil
 		}
 	}
 	if cachedOnly {
@@ -843,7 +914,7 @@ func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, m
 		s.qualifications[qualificationKey] = check
 		s.mu.Unlock()
 		if !result.Success {
-			return nil, ErrCodexGatewayBorrowUnavailable
+			return nil, borrowVerificationFailure(check)
 		}
 		return check, nil
 	})
@@ -854,7 +925,7 @@ func (s *CodexGatewayBorrowService) apply(req *http.Request, account *Account, m
 	if !ok {
 		return req, nil, ErrCodexGatewayBorrowUnavailable
 	}
-	return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires), nil
+	return borrowApplyCookie(req, candidate, check), borrowApplication(candidate, key, check.expires, check.result), nil
 }
 
 func (s *CodexGatewayBorrowService) candidateCurrentLocked(rev uint64, candidate *codexGatewayBorrowCandidate) bool {
@@ -882,13 +953,17 @@ func (s *CodexGatewayBorrowService) cachedTarget(rev uint64, key codexGatewayBor
 		return old, true, nil
 	}
 	if !old.result.Success && now.Before(old.retryAfter) {
-		return old, false, ErrCodexGatewayBorrowUnavailable
+		return old, false, borrowVerificationFailure(old)
 	}
 	return old, false, nil
 }
 
-func borrowApplication(candidate *codexGatewayBorrowCandidate, fingerprint string, expires time.Time) *CodexGatewayBorrowApplication {
-	return &CodexGatewayBorrowApplication{Applied: true, SourceAccountID: candidate.sourceID, ExpiresAt: expires, Fingerprint: fingerprint, CookieFingerprint: borrowHash(candidate.cookie.Value)}
+func borrowApplication(candidate *codexGatewayBorrowCandidate, fingerprint string, expires time.Time, verification ...CodexGatewayBorrowVerification) *CodexGatewayBorrowApplication {
+	app := &CodexGatewayBorrowApplication{Applied: true, SourceAccountID: candidate.sourceID, ExpiresAt: expires, Fingerprint: fingerprint, CookieFingerprint: borrowHash(candidate.cookie.Value)}
+	if len(verification) > 0 {
+		app.Verification = verification[0]
+	}
+	return app
 }
 
 func borrowApplyCookie(req *http.Request, candidate *codexGatewayBorrowCandidate, check codexGatewayBorrowTargetCheck) *http.Request {