Keep Codex source and STATE probes on the pinned plaintext protocol

.downstream/native-domains.md

@@ -100,6 +100,11 @@ and uses the ordinary OAuth account-test payload (default instructions, fixed
 short user prompt, medium effort), with its own configured normal connection
 policy. It does not inherit target-probe session/window headers, routing hints,
 encrypted-output options or connection-close behavior.
+Source acquisition and the target's two STATE shots preserve the pinned
+plaintext-JSON wire protocol. They do not enter the business zstd encoder after
+clearing old body headers. This leaves the saved compression setting and normal
+generation unchanged. Candidate status records `source_request_encoding`, and
+verification records the actual encoding of each shot.
 It has a 90-second budget. Each target/model is validated with two completed
 HTTP 200 streams, at most 45 seconds each: the first must return STATE, and
 the second may omit STATE or return the same value. Qualification includes
@@ -201,6 +206,10 @@ completion stay false. These comparisons never publish business qualifications
 or replace real usage observations. Source acquisition and all shots share the
 existing eight-request cap; the snapshot's expiry cancels remaining comparisons
 without silently selecting a different candidate. Page reads do not invoke it.
+Optional `comparison: encoding` keeps the same current body, tier, identity,
+cookie and exit, and compares plaintext probes with the configured business
+codec. It changes no global setting or business qualification; actual per-shot
+encoding distinguishes an enabled zstd comparison from two plaintext samples.
 
 Ranxi reference: v2.10.2 (`d3e43f2de33af9e987cffa511d76dfabbcd749da`). Its five
 fingerprint, gateway-cookie, target-probe and automatic-setup core files are

backend/internal/handler/admin/codex_gateway_borrow_handler.go

@@ -277,7 +277,7 @@ func (h *CodexGatewayBorrowHandler) Diagnose(c *gin.Context) {
 		response.BadRequest(c, err.Error())
 		return
 	}
-	middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_diagnostic", "account_id": request.AccountID, "model": request.Model, "transport": request.Transport, "scenario": request.Scenario, "mode": request.Mode, "service_tier": request.ServiceTier})
+	middleware.SetAuditExtra(c, map[string]any{"result": "codex_gateway_borrow_diagnostic", "account_id": request.AccountID, "model": request.Model, "transport": request.Transport, "scenario": request.Scenario, "comparison": request.Comparison, "mode": request.Mode, "service_tier": request.ServiceTier})
 	c.Header("Content-Type", "text/event-stream")
 	c.Header("X-Accel-Buffering", "no")
 	c.Status(http.StatusOK)

backend/internal/service/codex_gateway_borrow.go

@@ -181,10 +181,11 @@ func cloneCodexGatewayBorrowConfig(cfg CodexGatewayBorrowConfig) CodexGatewayBor
 }
 
 type CodexGatewayBorrowCandidateStatus struct {
-	SourceAccountID   int64     `json:"source_account_id"`
-	ExpiresAt         time.Time `json:"expires_at"`
-	RemainingSeconds  int64     `json:"remaining_seconds"`
-	CookieFingerprint string    `json:"cookie_fingerprint"`
+	SourceRequestEncoding string    `json:"source_request_encoding,omitempty"`
+	SourceAccountID       int64     `json:"source_account_id"`
+	ExpiresAt             time.Time `json:"expires_at"`
+	RemainingSeconds      int64     `json:"remaining_seconds"`
+	CookieFingerprint     string    `json:"cookie_fingerprint"`
 }
 
 type CodexGatewayBorrowSourceStatus struct {
@@ -198,25 +199,27 @@ type CodexGatewayBorrowSourceStatus struct {
 }
 
 type CodexGatewayBorrowVerification struct {
-	RequestShape        string     `json:"request_shape,omitempty"`
-	ServiceTier         string     `json:"service_tier,omitempty"`
-	MintCompleted       bool       `json:"mint_completed"`
-	ContinueCompleted   bool       `json:"continue_completed"`
-	MintStateLength     int        `json:"mint_state_length"`
-	ContinueStateLength int        `json:"continue_state_length"`
-	RouteChanged        bool       `json:"route_changed"`
-	AccountID           int64      `json:"account_id"`
-	Model               string     `json:"model"`
-	Success             bool       `json:"success"`
-	Reason              string     `json:"reason"`
-	Error               string     `json:"error,omitempty"`
-	MintStatus          int        `json:"mint_status"`
-	ContinueStatus      int        `json:"continue_status"`
-	Minted              bool       `json:"minted"`
-	NewTicket           bool       `json:"new_ticket"`
-	ReportedModel       string     `json:"reported_model,omitempty"`
-	CheckedAt           time.Time  `json:"checked_at"`
-	ExpiresAt           *time.Time `json:"expires_at,omitempty"`
+	MintRequestEncoding     string     `json:"mint_request_encoding,omitempty"`
+	ContinueRequestEncoding string     `json:"continue_request_encoding,omitempty"`
+	RequestShape            string     `json:"request_shape,omitempty"`
+	ServiceTier             string     `json:"service_tier,omitempty"`
+	MintCompleted           bool       `json:"mint_completed"`
+	ContinueCompleted       bool       `json:"continue_completed"`
+	MintStateLength         int        `json:"mint_state_length"`
+	ContinueStateLength     int        `json:"continue_state_length"`
+	RouteChanged            bool       `json:"route_changed"`
+	AccountID               int64      `json:"account_id"`
+	Model                   string     `json:"model"`
+	Success                 bool       `json:"success"`
+	Reason                  string     `json:"reason"`
+	Error                   string     `json:"error,omitempty"`
+	MintStatus              int        `json:"mint_status"`
+	ContinueStatus          int        `json:"continue_status"`
+	Minted                  bool       `json:"minted"`
+	NewTicket               bool       `json:"new_ticket"`
+	ReportedModel           string     `json:"reported_model,omitempty"`
+	CheckedAt               time.Time  `json:"checked_at"`
+	ExpiresAt               *time.Time `json:"expires_at,omitempty"`
 }
 
 type CodexGatewayBorrowTargetStatus struct {
@@ -254,9 +257,10 @@ type CodexGatewayBorrowApplication struct {
 }
 
 type codexGatewayBorrowCandidate struct {
-	cookie   http.Cookie
-	expires  time.Time
-	sourceID int64
+	sourceRequestEncoding string
+	cookie                http.Cookie
+	expires               time.Time
+	sourceID              int64
 }
 
 type codexGatewayBorrowTargetKey struct {
@@ -503,7 +507,7 @@ func (s *CodexGatewayBorrowService) Status() CodexGatewayBorrowStatus {
 	out := CodexGatewayBorrowStatus{Enabled: s.config.Enabled, Revision: s.revision, GeneratedAt: now, Preparing: s.preparing || s.prepareRuns > 0 || s.setup.State == "queued",
 		ObservedSince: s.observedSince, RecentUsage: []CodexGatewayBorrowUsage{}, Setup: s.setup, Config: cloneCodexGatewayBorrowConfig(s.config), Sources: []CodexGatewayBorrowSourceStatus{}, Targets: []CodexGatewayBorrowTargetStatus{}, ModelEfforts: codexGatewayBorrowModelEfforts()}
 	if c := s.candidate; c != nil {
-		out.Candidate = &CodexGatewayBorrowCandidateStatus{SourceAccountID: c.sourceID, ExpiresAt: c.expires, RemainingSeconds: borrowRemaining(c.expires, now), CookieFingerprint: borrowHash(c.cookie.Value)}
+		out.Candidate = &CodexGatewayBorrowCandidateStatus{SourceAccountID: c.sourceID, SourceRequestEncoding: c.sourceRequestEncoding, ExpiresAt: c.expires, RemainingSeconds: borrowRemaining(c.expires, now), CookieFingerprint: borrowHash(c.cookie.Value)}
 	}
 	for _, usage := range s.usage {
 		out.RecentUsage = append(out.RecentUsage, usage)

backend/internal/service/codex_gateway_borrow_diagnose.go

@@ -72,6 +72,7 @@ func consumeBorrowDiagnosticRequest(ctx context.Context) error {
 }
 
 type CodexBorrowDiagnosticRequest struct {
+	Comparison   string `json:"comparison,omitempty"`
 	Scenario     string `json:"scenario,omitempty"`
 	Mode         string `json:"mode,omitempty"`
 	ServiceTier  string `json:"service_tier,omitempty"`
@@ -128,6 +129,9 @@ func (s *CodexGatewayBorrowService) Diagnose(ctx context.Context, request CodexB
 	if request.Scenario == "probe_contract" && (request.Transport != "http" || request.Mode == "ordinary") {
 		return errors.New("probe_contract requires borrowed HTTP observations")
 	}
+	if request.Comparison != "" && (request.Scenario != "probe_contract" || (request.Comparison != "body" && request.Comparison != "encoding")) {
+		return errors.New("comparison must be body or encoding in probe_contract")
+	}
 	if request.ServiceTier != "" && request.ServiceTier != "default" && request.ServiceTier != OpenAIFastTierPriority && request.ServiceTier != OpenAIFastTierFlex && request.ServiceTier != OpenAIFastTierUltrafast {
 		return errors.New("unsupported service_tier")
 	}

backend/internal/service/codex_gateway_borrow_diagnose_contract.go

@@ -17,13 +17,16 @@ const (
 	codexBorrowProbeRanxiLiteral = "ranxi_literal_without_tier"
 	codexBorrowProbeRanxiTier    = "ranxi_literal_with_final_tier"
 	codexBorrowProbeCurrent      = "current_final_tier"
+	codexBorrowProbePlain        = "current_plaintext"
+	codexBorrowProbeConfigured   = "current_configured_encoding"
 )
 
 // These are observations of one fixed candidate, never qualifications usable
 // by business requests. The original pinned route probe selects Astra; rendering
 // a selected Sol model here is explicitly the downstream model extension.
 type CodexBorrowProbeContext struct {
 	SourceAccountID          int64     `json:"source_account_id"`
+	SourceRequestEncoding    string    `json:"source_request_encoding,omitempty"`
 	TargetProxyID            *int64    `json:"target_proxy_id"`
 	CookieFingerprint        string    `json:"cookie_fingerprint"`
 	CandidateExpiresAt       time.Time `json:"candidate_expires_at"`
@@ -97,6 +100,9 @@ func (s *CodexGatewayBorrowService) diagnoseProbeContract(ctx context.Context, r
 	if finalTier == "" {
 		shapes = []string{codexBorrowProbeRanxiLiteral, codexBorrowProbeCurrent}
 	}
+	if request.Comparison == "encoding" {
+		shapes = []string{codexBorrowProbePlain, codexBorrowProbeConfigured}
+	}
 	for _, shape := range shapes {
 		if err := ctx.Err(); err != nil {
 			return err
@@ -109,17 +115,21 @@ func (s *CodexGatewayBorrowService) diagnoseProbeContract(ctx context.Context, r
 			tier = ""
 		}
 		operation := withCodexBorrowServiceTier(context.WithValue(ctx, codexBorrowProbeShapeContextKey{}, shape), tier)
+		if request.Comparison == "encoding" {
+			operation = context.WithValue(operation, codexBorrowProbeConfiguredEncodingKey{}, shape == codexBorrowProbeConfigured)
+		}
 		variant := template.WithContext(operation)
 		started := time.Now()
 		verification := s.probeTarget(operation, variant, account, request.Model, proxy, nil, candidate)
 		body := borrowRanxiLiteralPayload(request.Model, tier)
-		if shape == codexBorrowProbeCurrent {
+		if shape == codexBorrowProbeCurrent || request.Comparison == "encoding" {
 			body = borrowObservationPayload(request.Model, "Reply with OK.", "", tier)
 		}
 		result := CodexBorrowDiagnosticResult{Scenario: "probe_contract", Mode: "borrowed", ProbeOnly: true,
 			Verification: &verification, DurationMS: time.Since(started).Milliseconds(), ReportedModel: verification.ReportedModel,
 			ProbeContext: &CodexBorrowProbeContext{SourceAccountID: candidate.sourceID, TargetProxyID: account.ProxyID,
-				CookieFingerprint: borrowHash(candidate.cookie.Value), CandidateExpiresAt: candidate.expires,
+				SourceRequestEncoding: candidate.sourceRequestEncoding,
+				CookieFingerprint:     borrowHash(candidate.cookie.Value), CandidateExpiresAt: candidate.expires,
 				TemplateFingerprint: identity, RequestedTier: request.ServiceTier, FinalTier: finalTier, RoutingHint: template.Header.Get(openAICodexRoutingHintHeader),
 				PlaintextBodyFingerprint: borrowHash(string(body)), OriginalAstraModel: request.Model == codexGatewayBorrowSourceModel}}
 		if !verification.Success {

backend/internal/service/codex_gateway_borrow_probe.go

@@ -26,6 +26,7 @@ const codexGatewayBorrowProbeMaxBody = 1 << 20
 const codexGatewayBorrowPelicanMaxBody = 32 << 20
 
 type codexGatewayBorrowObservation struct {
+	requestEncoding   string
 	status            int
 	state             string
 	model             string
@@ -125,6 +126,7 @@ func (s *CodexGatewayBorrowService) acquireSource(ctx context.Context, id int64)
 		return nil, "routing cookie missing, invalid, deleted or expired", errors.New("source_routing_cookie_unavailable")
 	}
 	candidate.sourceID = id
+	candidate.sourceRequestEncoding = shot.requestEncoding
 	return candidate, "", nil
 }
 
@@ -135,6 +137,7 @@ func (s *CodexGatewayBorrowService) acquireSource(ctx context.Context, id int64)
 // closed. The source keeps its own configured OpenAI transport and proxy.
 func borrowSourceObservationRequest(ctx context.Context, headers http.Header) (*http.Request, error) {
 	ctx = WithCodexGatewayBorrowObservation(WithHTTPUpstreamRedirectsDisabled(WithHTTPUpstreamProfile(ctx, HTTPUpstreamProfileCodexBorrowSource)))
+	ctx = context.WithValue(ctx, codexBorrowProbeConfiguredEncodingKey{}, false)
 	payload := createOpenAITestPayload(codexGatewayBorrowSourceModel, true, "Reply with OK only.")
 	payload["reasoning"] = map[string]string{"effort": "medium"}
 	body, err := json.Marshal(payload)
@@ -206,6 +209,7 @@ func borrowCandidateFromCookies(cookies []*http.Cookie, path string, now time.Ti
 }
 
 func (s *CodexGatewayBorrowService) probeTarget(ctx context.Context, template *http.Request, account *Account, model, proxy string, profile *tlsfingerprint.Profile, candidate *codexGatewayBorrowCandidate) CodexGatewayBorrowVerification {
+	ctx = withCodexBorrowPlainProbe(ctx)
 	expires := candidate.expires
 	result := CodexGatewayBorrowVerification{AccountID: account.ID, Model: model, CheckedAt: time.Now(), ExpiresAt: &expires, Reason: "target_probe_failed", RequestShape: "ranxi_full", ServiceTier: codexBorrowRequestServiceTier(template)}
 	if shape, _ := ctx.Value(codexBorrowProbeShapeContextKey{}).(string); shape != "" {
@@ -230,6 +234,7 @@ func (s *CodexGatewayBorrowService) probeTarget(ctx context.Context, template *h
 	}
 	mint, err := fire("", cookies)
 	result.MintStatus, result.ReportedModel = mint.status, mint.model
+	result.MintRequestEncoding = mint.requestEncoding
 	result.MintCompleted, result.MintStateLength = mint.complete, len(mint.state)
 	if err != nil || !borrowShotUsable(mint) {
 		result.Error = borrowObservationError(mint, err)
@@ -255,6 +260,7 @@ func (s *CodexGatewayBorrowService) probeTarget(ctx context.Context, template *h
 	}
 	continued, err := fire(mint.state, cookies)
 	result.ContinueStatus = continued.status
+	result.ContinueRequestEncoding = continued.requestEncoding
 	result.ContinueCompleted, result.ContinueStateLength = continued.complete, len(continued.state)
 	if continued.model != "" {
 		result.ReportedModel = continued.model
@@ -406,9 +412,17 @@ func (s *CodexGatewayBorrowService) sendObservation(req *http.Request, account *
 		}
 		defer releaseCapacity()
 	}
-	wire, err := prepareOpenAICodexWireRequest(req, account)
-	if err != nil {
-		return shot, err
+	wire := req
+	if configured, probe := req.Context().Value(codexBorrowProbeConfiguredEncodingKey{}).(bool); !probe || configured {
+		var err error
+		wire, err = prepareOpenAICodexWireRequest(req, account)
+		if err != nil {
+			return shot, err
+		}
+	}
+	shot.requestEncoding = strings.TrimSpace(wire.Header.Get("Content-Encoding"))
+	if shot.requestEncoding == "" {
+		shot.requestEncoding = "identity"
 	}
 	resp, err := s.upstream.DoWithTLS(wire, proxy, account.ID, account.Concurrency, profile)
 	if err != nil {

backend/internal/service/codex_gateway_borrow_probe_wire.go

@@ -0,0 +1,15 @@
+package service
+
+import "context"
+
+// The pinned source and two-shot STATE protocols send plaintext JSON. They are
+// independent of the user's business-request compression setting. Only an
+// explicit administrator encoding comparison may opt into the configured codec.
+type codexBorrowProbeConfiguredEncodingKey struct{}
+
+func withCodexBorrowPlainProbe(ctx context.Context) context.Context {
+	if _, set := ctx.Value(codexBorrowProbeConfiguredEncodingKey{}).(bool); set {
+		return ctx
+	}
+	return context.WithValue(ctx, codexBorrowProbeConfiguredEncodingKey{}, false)
+}

backend/internal/service/codex_gateway_borrow_probe_wire_test.go

@@ -0,0 +1,79 @@
+package service
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCodexBorrowProbesStayPlainWhileGenerationKeepsCompression(t *testing.T) {
+	oldCompression := codexRequestZstd.Load()
+	SetCodexRequestZstdEnabled(true)
+	defer SetCodexRequestZstdEnabled(oldCompression)
+	var sources, targets int
+	s := newBorrowCoreTest(t, func(req *http.Request, _ string, id int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) {
+		if id == 1 {
+			sources++
+			assert.Empty(t, req.Header.Get("Content-Encoding"), "the pinned source protocol sends plain JSON")
+			return borrowCoreResponse("gpt-6-astra", "OK", "source", "__oailb=source-cookie; Secure; Path=/; Max-Age=230"), nil
+		}
+		targets++
+		if targets <= 2 {
+			assert.Empty(t, req.Header.Get("Content-Encoding"), "a body helper must not silently recompress the two-shot probe")
+		} else {
+			assert.Equal(t, "zstd", req.Header.Get("Content-Encoding"), "ordinary generation retains the enabled runtime policy")
+		}
+		return borrowCoreResponse("gpt-6.1-sol", "OK", "stable-state"), nil
+	}, borrowCoreAccount(1), borrowCoreAccount(2))
+	_, err := s.Verify(context.Background(), 2, "gpt-6.1-sol")
+	require.NoError(t, err)
+	result, err := s.GeneratePelican(context.Background(), 2, "gpt-6.1-sol", "low")
+	require.NoError(t, err)
+	require.Equal(t, "complete", result.Status)
+	require.Equal(t, 1, sources)
+	require.Equal(t, 3, targets)
+	require.True(t, codexRequestZstd.Load(), "diagnostic protocol isolation cannot toggle the global setting")
+	require.Equal(t, "identity", s.Status().Candidate.SourceRequestEncoding)
+}
+
+func TestCodexBorrowEncodingComparisonChangesOnlyWireEncoding(t *testing.T) {
+	oldCompression := codexRequestZstd.Load()
+	SetCodexRequestZstdEnabled(true)
+	defer SetCodexRequestZstdEnabled(oldCompression)
+	var encodings []string
+	var bodies []string
+	s := newBorrowCoreTest(t, func(req *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) {
+		encodings = append(encodings, req.Header.Get("Content-Encoding"))
+		bodies = append(bodies, string(borrowCoreBody(t, req)))
+		return borrowCoreResponse("gpt-6.1-sol", "OK", "stable"), nil
+	}, borrowCoreAccount(2))
+	borrowCoreCandidate(s, time.Now().Add(codexGatewayBorrowTTL))
+	var results []*CodexBorrowDiagnosticResult
+	var requests int32
+	err := s.Diagnose(context.Background(), CodexBorrowDiagnosticRequest{Scenario: "probe_contract", Comparison: "encoding", AccountID: 2, Model: "gpt-6.1-sol", Transport: "http", ServiceTier: "priority"}, func(e CodexBorrowDiagnosticEvent) {
+		requests = e.Requests
+		if e.Result != nil {
+			results = append(results, e.Result)
+		}
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, 4, requests)
+	require.Equal(t, []string{"", "", "zstd", "zstd"}, encodings)
+	for _, body := range bodies {
+		require.Equal(t, bodies[0], body)
+	}
+	require.Len(t, results, 2)
+	require.Equal(t, results[0].ProbeContext.CookieFingerprint, results[1].ProbeContext.CookieFingerprint)
+	require.Equal(t, results[0].ProbeContext.PlaintextBodyFingerprint, results[1].ProbeContext.PlaintextBodyFingerprint)
+	require.Equal(t, "identity", results[0].Verification.MintRequestEncoding)
+	require.Equal(t, "identity", results[0].Verification.ContinueRequestEncoding)
+	require.Equal(t, "zstd", results[1].Verification.MintRequestEncoding)
+	require.Equal(t, "zstd", results[1].Verification.ContinueRequestEncoding)
+	require.True(t, codexRequestZstd.Load())
+	require.Empty(t, s.qualifications)
+}

backend/internal/service/codex_gateway_borrow_request_repair_test.go

@@ -26,6 +26,9 @@ func TestCodexBorrowNilRequestDoesNotCreateDiagnosticEvidence(t *testing.T) {
 // A synthetic upstream rotates STATE when those headers change, reproducing
 // the false rejection introduced by manufacturing a new native session per shot.
 func TestCodexBorrowProbePreservesNativeSessionAndTier(t *testing.T) {
+	oldCompression := codexRequestZstd.Load()
+	SetCodexRequestZstdEnabled(true)
+	defer SetCodexRequestZstdEnabled(oldCompression)
 	for _, model := range []string{"gpt-6-astra", "gpt-6.1-sol"} {
 		t.Run(model, func(t *testing.T) {
 			var shots []*http.Request