diff --git a/backend/cmd/server/wire_gen.go b/backend/cmd/server/wire_gen.go index 248743722c8a..344eac57723f 100644 --- a/backend/cmd/server/wire_gen.go +++ b/backend/cmd/server/wire_gen.go @@ -275,7 +275,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { errorPassthroughService := service.NewErrorPassthroughService(errorPassthroughRepository, errorPassthroughCache) errorPassthroughHandler := admin.NewErrorPassthroughHandler(errorPassthroughService) tlsFingerprintProfileHandler := admin.NewTLSFingerprintProfileHandler(tlsFingerprintProfileService) - pluginHandler := handler.ProvidePluginHandler(pluginManager) + pluginHandler := handler.ProvidePluginHandler(pluginManager, nativeFeatureBootstrap) adminAPIKeyHandler := admin.NewAdminAPIKeyHandler(adminService) scheduledTestPlanRepository := repository.NewScheduledTestPlanRepository(db) scheduledTestResultRepository := repository.NewScheduledTestResultRepository(db) @@ -359,7 +359,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { stepUpAuthMiddleware := middleware.NewStepUpAuthMiddleware(totpService, userService, settingService) engine := server.ProvideRouter(configConfig, handlers, jwtAuthMiddleware, optionalJWTAuthMiddleware, adminAuthMiddleware, apiKeyAuthMiddleware, auditLogMiddleware, stepUpAuthMiddleware, apiKeyService, subscriptionService, opsService, settingService, compositeRouteResolver, redisClient) httpServer := server.ProvideHTTPServer(configConfig, engine) - imageStudioGatewayExecutor := handler.NewImageStudioGatewayExecutor(openAIGatewayHandler, subscriptionService) + imageStudioGatewayExecutor := handler.NewImageStudioGatewayExecutor(openAIGatewayHandler, subscriptionService, opsService) imageStudioRuntime, err := handler.ProvideImageStudioRuntime(imageStudioRepository, imageStudioService, imageStudioArtifactStore, imageStudioGatewayExecutor) if err != nil { return nil, err diff --git a/backend/internal/codexruntime/tickets/cutover_test.go b/backend/internal/codexruntime/tickets/cutover_test.go index 4f3283b5496b..da712a157f14 100644 --- a/backend/internal/codexruntime/tickets/cutover_test.go +++ b/backend/internal/codexruntime/tickets/cutover_test.go @@ -104,6 +104,9 @@ func TestTicketAdmissionAndInjectionShareIdentityValidityAndScope(t *testing.T) qualification := &extensionv1.CodexRoutingQualification{Scope: testRoutingScope(account), Model: "gpt-6-astra", VerifiedAt: now, ExpiresAt: now.Add(time.Minute), Bundle: extensionv1.CodexRoutingBundleRef{Key: "bundle.test", Revision: 1, ExpiresAt: now.Add(time.Minute)}} request := extensionv1.SchedulingRequest{Account: account, Model: qualification.Model, Now: now} tc.change(qualification, &request) + if valid, problem := qualification.Valid(now, account.ID, account.Identity, "gpt-6-astra"), qualification.Problem(now, account.ID, account.Identity, "gpt-6-astra"); valid != (problem == "") { + t.Fatalf("Problem %q disagrees with Valid %t", problem, valid) + } rawState, _ := json.Marshal(State{Schema: 2, Identity: account.Identity, Qualification: qualification}) host := &memoryHost{account: account, state: map[string]extensionv1.StateResult{stateKey(account.ID, "gpt-6-astra"): {Found: true, Value: rawState}}} module := NewModule() @@ -128,6 +131,9 @@ func TestTicketAdmissionAndInjectionShareIdentityValidityAndScope(t *testing.T) if (injection.Code == "") != tc.allow { t.Fatalf("injection disagrees with admission: %+v", injection) } + if injection.Code == "ticket_missing" && !strings.HasPrefix(injection.Message, "the route qualification") { + t.Fatalf("ticket_missing lost its concrete reason: %+v", injection) + } if strings.Contains(string(injection.Payload), "routing_qualification") != tc.inject || strings.Contains(string(injection.Payload), "x-codex-turn-state") { t.Fatal("unexpected opaque-state injection or missing qualification reference") } diff --git a/backend/internal/codexruntime/tickets/module.go b/backend/internal/codexruntime/tickets/module.go index 21281dd26603..9ed67de7f2a5 100644 --- a/backend/internal/codexruntime/tickets/module.go +++ b/backend/internal/codexruntime/tickets/module.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/Wei-Shaw/sub2api/internal/codexruntime/profile" "github.com/Wei-Shaw/sub2api/internal/codexruntime/recovery" @@ -33,6 +34,10 @@ type Config struct { RequestZstd bool `json:"request_zstd"` } +// ErrStopping is returned while the module stops or applies a new +// configuration. It wraps context.Canceled for callers that treat it so. +var ErrStopping = fmt.Errorf("codex runtime is stopping or reloading: %w", context.Canceled) + type HostCaller interface { Call(context.Context, extensionv1.HostInvocation) (extensionv1.Result, error) } @@ -62,21 +67,36 @@ func (m *Module) SetHost(host HostCaller) { m.mu.Lock(); defer m.mu.Unlock(); m. func (m *Module) ValidateConfig(_ context.Context, raw json.RawMessage) (json.RawMessage, error) { cfg := Config{RoutingSchema: 2, FailClosed: true, Models: []string{"gpt-6-astra", "gpt-6-sol", "gpt-6-luna"}, RequestZstd: true} var fields map[string]json.RawMessage - if json.Unmarshal(raw, &fields) != nil || fields == nil || json.Unmarshal(raw, &cfg) != nil { - return nil, errors.New("invalid codex runtime configuration") + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, fmt.Errorf("invalid codex runtime configuration: %w", err) + } + if fields == nil { + return nil, errors.New("invalid codex runtime configuration: expected a JSON object") + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("invalid codex runtime configuration: %w", err) } + var nullKeys, unknownKeys []string for key, value := range fields { switch key { case "enabled", "fail_closed", "proxy_url", "proxy_protocol", "proxy_selection_id", "models", "request_zstd", "routing_schema": if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { - return nil, errors.New("null codex runtime setting") + nullKeys = append(nullKeys, key) } default: - return nil, errors.New("unknown codex runtime setting") + unknownKeys = append(unknownKeys, key) } } + if len(unknownKeys) > 0 { + slices.Sort(unknownKeys) + return nil, fmt.Errorf("unknown codex runtime setting: %s", strings.Join(unknownKeys, ", ")) + } + if len(nullKeys) > 0 { + slices.Sort(nullKeys) + return nil, fmt.Errorf("null codex runtime setting: %s", strings.Join(nullKeys, ", ")) + } if cfg.RoutingSchema < 0 || cfg.RoutingSchema > 2 { - return nil, errors.New("unsupported routing schema") + return nil, fmt.Errorf("unsupported routing schema %d (supported: 0, 1, 2)", cfg.RoutingSchema) } _, schemaPresent := fields["routing_schema"] if (!schemaPresent || cfg.RoutingSchema < 2) && len(cfg.Models) == 2 && slices.Contains(cfg.Models, "gpt-6-astra") && slices.Contains(cfg.Models, "gpt-5.6-sol") { @@ -84,15 +104,15 @@ func (m *Module) ValidateConfig(_ context.Context, raw json.RawMessage) (json.Ra } cfg.RoutingSchema = 2 if len(cfg.Models) == 0 || len(cfg.Models) > 32 { - return nil, errors.New("ticket model list required") + return nil, fmt.Errorf("ticket model list must contain 1 to 32 models (got %d)", len(cfg.Models)) } seen := make(map[string]bool, len(cfg.Models)) for _, model := range cfg.Models { if strings.TrimSpace(model) != model || model == "" || len(model) > 256 { - return nil, errors.New("invalid ticket model") + return nil, fmt.Errorf("invalid ticket model %q: a model must be 1 to 256 bytes without leading or trailing spaces", model) } if seen[model] { - return nil, errors.New("duplicate ticket model") + return nil, fmt.Errorf("duplicate ticket model %q", model) } seen[model] = true } @@ -289,7 +309,7 @@ func (m *Module) Invoke(ctx context.Context, in extensionv1.Invocation) (extensi m.mu.Lock() if !m.accepting { m.mu.Unlock() - return extensionv1.Result{}, context.Canceled + return extensionv1.Result{}, ErrStopping } host, cfg, epoch, work := m.host, m.config, m.epoch, m.work work.Add(1) @@ -340,8 +360,8 @@ func (m *Module) Invoke(ctx context.Context, in extensionv1.Invocation) (extensi Protocol string `json:"protocol"` ProxySelectionID string `json:"proxy_selection_id"` } - if json.Unmarshal(in.Payload, &req) != nil { - return extensionv1.Result{}, errors.New("invalid proxy test") + if err := json.Unmarshal(in.Payload, &req); err != nil { + return extensionv1.Result{}, fmt.Errorf("invalid proxy test: %w", err) } testCtx, testCancel := context.WithTimeout(ctx, 25*time.Second) defer testCancel() @@ -356,6 +376,10 @@ func (m *Module) Invoke(ctx context.Context, in extensionv1.Invocation) (extensi if result == nil { return extensionv1.Result{}, testErr } + if testErr != nil && result.FailureDetail == "" { + // The result is returned instead of the error; keep its original text. + result.FailureDetail = testErr.Error() + } value = result case in.Capability == extensionv1.CapabilityScheduling && in.Operation == "admit": var req extensionv1.SchedulingRequest @@ -388,21 +412,32 @@ func (m *Module) Invoke(ctx context.Context, in extensionv1.Invocation) (extensi var checked extensionv1.CodexRoutingProbeResult q := state.Qualification if checkErr := hostCall(ctx, host, extensionv1.HostCodexRoutingCheck, extensionv1.CodexRoutingQuery{AccountID: req.Account.ID, Model: req.Model, Transport: q.Scope.Transport, Bundle: &q.Bundle, Scope: &q.Scope}, &checked); checkErr != nil || !checked.Valid { + reason := checked.Observation.Code + if checked.Observation.Error != "" { + reason += ": " + checked.Observation.Error + } + if checkErr != nil { + reason = "routing check: " + checkErr.Error() + } + if reason == "" { + reason = "the host routing check rejected the recorded qualification" + } if state.Phase != "manual_running" && state.Phase != "pre_running" && state.Phase != "post_running" { + now := time.Now().UTC() state.Qualification, state.ExpiresAt = nil, nil state.LastCode = "routing_stale" + state.RevokedAt, state.RevocationReason = &now, reason if state.Phase != "stopped" && state.Enrolled { state.Phase = "needs_cookie_verification" - now := time.Now().UTC() state.NextAt = &now } _, _ = writeState(ctx, host, req.Account.ID, req.Model, state, revision) } - return extensionv1.Result{Code: "routing_stale"}, nil + return extensionv1.Result{Code: "routing_stale", Message: reason}, nil } injection.Qualification = q } else if cfg.FailClosed { - return extensionv1.Result{Code: "ticket_missing"}, nil + return extensionv1.Result{Code: "ticket_missing", Message: missingQualificationReason(state, req)}, nil } } value = injection @@ -488,6 +523,12 @@ func writeState(ctx context.Context, host HostCaller, id int64, model string, st observation := extensionv1.AccountObservation{Key: model, Kind: "codex_routing", State: state.Phase, ExpiresAt: state.ExpiresAt, NextAt: state.NextAt, CheckedAt: state.LastAttemptAt, Code: state.LastCode} if state.Observation != nil { observation.Count = state.Observation.StateLength + observation.HTTPStatus = state.Observation.HTTPStatus + observation.ResponseModel = truncateUTF8(state.Observation.ResponseModel, extensionv1.AccountObservationModelLimit) + observation.Message = truncateUTF8(state.Observation.Summary(), extensionv1.AccountObservationMessageLimit) + } + if state.RevocationReason != "" && state.LastCode == "routing_stale" { + observation.Message = truncateUTF8(state.RevocationReason, extensionv1.AccountObservationMessageLimit) } request.Projection = &extensionv1.AccountProjection{AccountID: id, Identity: state.Identity, Scheduling: []extensionv1.SchedulingConstraint{constraint}, Observations: []extensionv1.AccountObservation{observation}} } @@ -498,6 +539,43 @@ func writeState(ctx context.Context, host HostCaller, id int64, model string, st return result.Revision, err } +// missingQualificationReason explains a ticket_missing refusal with the +// routing phase, the last result code and that result's original text. +func missingQualificationReason(state State, req extensionv1.SchedulingRequest) string { + reason := state.Qualification.Problem(req.Now, req.Account.ID, req.Account.Identity, req.Model) + var context []string + if state.Phase != "" { + context = append(context, "routing phase "+state.Phase) + } + if state.LastCode != "" { + context = append(context, "last result "+state.LastCode) + } + if state.RevocationReason != "" { + context = append(context, "revoked: "+state.RevocationReason) + } + if state.Observation != nil { + if summary := state.Observation.Summary(); summary != "" { + context = append(context, summary) + } + } + if len(context) > 0 { + reason += " (" + strings.Join(context, "; ") + ")" + } + return reason +} + +// truncateUTF8 keeps at most limit bytes without splitting a UTF-8 sequence. +func truncateUTF8(text string, limit int) string { + if len(text) <= limit { + return text + } + cut := limit + for cut > 0 && !utf8.RuneStart(text[cut]) { + cut-- + } + return text[:cut] +} + func newSessionID() string { var value [16]byte _, _ = rand.Read(value[:]) diff --git a/backend/internal/codexruntime/tickets/module_test.go b/backend/internal/codexruntime/tickets/module_test.go index 27b1d67b5580..5058863b75bb 100644 --- a/backend/internal/codexruntime/tickets/module_test.go +++ b/backend/internal/codexruntime/tickets/module_test.go @@ -39,6 +39,12 @@ func TestInvalidConfigurationCannotStopCurrentTicketEpoch(t *testing.T) { t.Fatalf("invalid configuration changed ticket execution: %s", raw) } } + // The rejection names the concrete key or model. + for raw, want := range map[string]string{`{"extra":true}`: "unknown codex runtime setting: extra", `{"enabled":null}`: "null codex runtime setting: enabled", `{"models":["one","one"]}`: `duplicate ticket model "one"`} { + if _, err := module.ValidateConfig(context.Background(), []byte(raw)); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("configuration %s rejected without %q: %v", raw, want, err) + } + } } func TestTransportSettingDoesNotDependOnTicketAcquisitionSwitch(t *testing.T) { diff --git a/backend/internal/codexruntime/tickets/outcome.go b/backend/internal/codexruntime/tickets/outcome.go index c6af64f8abb6..3682c66fecff 100644 --- a/backend/internal/codexruntime/tickets/outcome.go +++ b/backend/internal/codexruntime/tickets/outcome.go @@ -9,35 +9,58 @@ import ( extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" ) +var outcomeMessages = map[string]string{ + "routing_verified": "业务出口响应完整、模型声明一致；本次未验证质量", "routing_candidate": "已取得候选路由材料，尚未通过业务出口验证", + "routing_model_mismatch": "上游模型声明不匹配，未发布路由资格", "routing_incomplete": "响应未完整结束或缺少模型声明，未发布路由资格", + "routing_capacity": "上游容量不足或流内限流，未完成路由验证", "routing_policy": "上游策略检查阻止请求，未完成路由验证", + "routing_cancelled": "客户端取消，本次响应未完成验证", "routing_legacy_retired": "旧 STATE 材料已丢弃，尚未获得路由资格", + "routing_cookie_missing": "响应未给出可用路由 Cookie", "routing_cookie_expired": "Cookie 已到期，需要重新验证", + "routing_transport": "采集或业务出口验证连接失败", "routing_stale": "账号主体、出口、指纹或 Cookie 代次已变化", + "routing_cookie_deleted": "上游已撤销路由 Cookie", "routing_budget_spent": "本次验证的固定调用预算已使用", + "routing_upstream": "上游拒绝路由验证请求", "routing_connection_unknown": "无法绑定经过验证的实际连接", + "routing_connection_expired": "已验证连接不可复用，需要重新验证", "routing_unavailable": "路由采集未能开始（账号出口范围不可用）", + "routing_persist": "路由 Cookie 记录保存失败", "routing_observation_unavailable": "无法登记本次路由观测", "routing_validation_unavailable": "验证请求未能完成", + "ticket_ready": "历史 STATE 采集记录，不代表当前路由资格", "ticket_skipped": "已有路由验证记录，本次跳过；本次未验证质量", "ticket_stopped": "路由自动续期已停止", + "ticket_disabled": "路由采集与验证已关闭", "ticket_model_invalid": "模型不在路由配置范围内", "ticket_proxy_missing": "尚未配置采集代理", + "ticket_missing": "该模型尚无有效路由资格", "ticket_operation_required": "手动采集缺少操作标识", + "ticket_busy": "账号和模型已有进行中的操作", "ticket_interrupted": "前次请求结果未确认，未重复发送", "ticket_not_due": "未到续期时间", + "ticket_attempt_in_progress": "该账号和模型的上一次尝试仍在进行", "ticket_ineligible": "仅 OpenAI OAuth/Setup Token 非影子账号支持路由采集", + "ticket_token": "无法取得账号访问令牌", "ticket_transport": "采集连接失败", "ticket_upstream": "上游拒绝采集请求", + "ticket_length": "历史 STATE 长度检查失败（该规则已停用）", "ticket_prefix": "历史 STATE 格式检查失败（该规则已停用）", "ticket_persist": "路由验证记录保存失败", + "ticket_stale": "账号主体或操作状态已变化", "ticket_canceled": "操作已取消", "proxy_connection_failed": "代理连接或证书验证失败", + "pinned_connection_failed": "固定证书验证失败", "invalid_proxy": "代理地址或协议不正确", +} + +// OutcomeMessage describes a result code. An unknown code is named as is, +// never replaced by another category's sentence. +func OutcomeMessage(code string) string { + if message := outcomeMessages[code]; message != "" { + return message + } + return "未归类的结果代码：" + code +} + +func responseHeaderText(headers []extensionv1.CodexRoutingHeader) string { + lines := make([]string, 0, len(headers)) + for _, header := range headers { + lines = append(lines, header.Name+": "+header.Value) + } + return strings.Join(lines, "\n") +} + +func outcomeFact(zh, en, value string) extensionv1.DisplayFact { + return extensionv1.DisplayFact{Label: map[string]string{"zh": zh, "en": en}, Value: value} +} + func (o Outcome) MarshalJSON() ([]byte, error) { type wire Outcome - message := map[string]string{ - "routing_verified": "业务出口响应完整、模型声明一致；本次未验证质量", "routing_candidate": "已取得候选路由材料，尚未通过业务出口验证", - "routing_model_mismatch": "上游模型声明不匹配，未发布路由资格", "routing_incomplete": "响应未完整结束或缺少模型声明，未发布路由资格", - "routing_capacity": "上游容量不足或流内限流，未完成路由验证", "routing_policy": "上游策略检查阻止请求，未完成路由验证", - "routing_cancelled": "客户端取消，本次响应未完成验证", "routing_legacy_retired": "旧 STATE 材料已丢弃，尚未获得路由资格", - "routing_cookie_missing": "响应未给出可用路由 Cookie", "routing_cookie_expired": "Cookie 已到期，需要重新验证", - "routing_transport": "采集或业务出口验证连接失败", "routing_stale": "账号主体、出口、指纹或 Cookie 代次已变化", - "routing_cookie_deleted": "上游已撤销路由 Cookie", "routing_budget_spent": "本次验证的固定调用预算已使用", - "routing_upstream": "上游拒绝路由验证请求", "routing_connection_unknown": "无法绑定经过验证的实际连接", - "ticket_ready": "历史 STATE 采集记录，不代表当前路由资格", "ticket_skipped": "已有路由验证记录，本次跳过；本次未验证质量", "ticket_stopped": "路由自动续期已停止", - "ticket_disabled": "路由采集与验证已关闭", "ticket_model_invalid": "模型不在路由配置范围内", "ticket_proxy_missing": "尚未配置采集代理", - "ticket_missing": "该模型尚无有效路由资格", - "ticket_busy": "账号和模型已有进行中的操作", "ticket_interrupted": "前次请求结果未确认，未重复发送", "ticket_not_due": "未到续期时间", - "ticket_token": "无法取得账号访问令牌", "ticket_transport": "采集连接失败", "ticket_upstream": "上游拒绝采集请求", - "ticket_length": "历史 STATE 长度检查失败（该规则已停用）", "ticket_prefix": "历史 STATE 格式检查失败（该规则已停用）", "ticket_persist": "路由验证记录保存失败", - "ticket_stale": "账号主体或操作状态已变化", "ticket_canceled": "操作已取消", "proxy_connection_failed": "代理连接或证书验证失败", - "pinned_connection_failed": "固定证书验证失败", "invalid_proxy": "代理地址或协议不正确", - }[o.Code] - if message == "" { - message = "操作未完成，请检查配置与账号状态" - } - facts := []extensionv1.DisplayFact{} + message := OutcomeMessage(o.Code) + facts := []extensionv1.DisplayFact{outcomeFact("结果代码", "Result code", o.Code)} if o.HTTPStatus > 0 { - facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "上游状态", "en": "Upstream status"}, Value: strconv.Itoa(o.HTTPStatus)}) + facts = append(facts, outcomeFact("上游状态", "Upstream status", strconv.Itoa(o.HTTPStatus))) } if o.ObservedLength > 0 { - facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "STATE 长度（仅观测）", "en": "STATE length (observation only)"}, Value: strconv.Itoa(o.ObservedLength)}) + facts = append(facts, outcomeFact("STATE 长度（仅观测）", "STATE length (observation only)", strconv.Itoa(o.ObservedLength))) } if observation := o.Observation; observation != nil { material, completed, matched := "未证实有效", "未证实完整", "未知" @@ -54,15 +77,46 @@ func (o Outcome) MarshalJSON() ([]byte, error) { if observation.ModelMatched { matched = "一致" } - facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "上游模型声明", "en": "Upstream model declaration"}, Value: observation.ResponseModel}) + facts = append(facts, outcomeFact("上游模型声明", "Upstream model declaration", observation.ResponseModel)) } facts = append(facts, - extensionv1.DisplayFact{Label: map[string]string{"zh": "路由材料", "en": "Routing material"}, Value: material}, - extensionv1.DisplayFact{Label: map[string]string{"zh": "完整响应", "en": "Response completion"}, Value: completed}, - extensionv1.DisplayFact{Label: map[string]string{"zh": "模型声明匹配", "en": "Model declaration match"}, Value: matched}) + outcomeFact("路由材料", "Routing material", material), + outcomeFact("完整响应", "Response completion", completed), + outcomeFact("模型声明匹配", "Model declaration match", matched)) + for _, detail := range []struct{ zh, en, value string }{ + {"阶段", "Stage", observation.Stage}, + {"传输", "Transport", observation.Transport}, + {"请求模型", "Requested model", observation.RequestedModel}, + {"推理强度", "Reasoning effort", observation.ReasoningEffort}, + {"错误原文", "Error", observation.Error}, + {"上游错误类型", "Upstream error type", observation.UpstreamErrorType}, + {"上游错误代码", "Upstream error code", observation.UpstreamErrorCode}, + {"上游错误信息", "Upstream error message", observation.UpstreamErrorMessage}, + {"上游错误参数", "Upstream error param", observation.UpstreamErrorParam}, + {"上游请求 ID", "Upstream request ID", observation.RequestID}, + {"CF-Ray", "CF-Ray", observation.CFRay}, + {"Cookie 名称", "Cookie names", strings.Join(observation.CookieNames, ", ")}, + {"STATE", "STATE", observation.State}, + {"上游响应头", "Upstream response headers", responseHeaderText(observation.ResponseHeaders)}, + {"未收录的响应头（超出 4 KiB）", "Response headers not kept (over 4 KiB)", strings.Join(observation.ResponseHeadersOmitted, ", ")}, + {"上游响应正文", "Upstream response body", observation.UpstreamBody}, + } { + if detail.value != "" { + facts = append(facts, outcomeFact(detail.zh, detail.en, detail.value)) + } + } + if observation.DurationMS > 0 { + facts = append(facts, outcomeFact("耗时（毫秒）", "Duration (ms)", strconv.FormatInt(observation.DurationMS, 10))) + } + if observation.CookieSent { + facts = append(facts, outcomeFact("已发送路由 Cookie", "Routing cookie sent", "是 / yes")) + } + if !observation.ObservedAt.IsZero() { + facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "观测时间", "en": "Observed at"}, Value: observation.ObservedAt.Format(time.RFC3339), Timestamp: true}) + } } if strings.HasPrefix(o.Code, "routing_") || o.Code == "ticket_skipped" || o.Code == "ticket_ready" { - facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "质量验证", "en": "Quality validation"}, Value: "本次未验证 / Not tested by this operation"}) + facts = append(facts, outcomeFact("质量验证", "Quality validation", "本次未验证 / Not tested by this operation")) } if o.ExpiresAt != nil { facts = append(facts, extensionv1.DisplayFact{Label: map[string]string{"zh": "路由记录有效期至", "en": "Routing record expires"}, Value: o.ExpiresAt.Format(time.RFC3339), Timestamp: true}) diff --git a/backend/internal/codexruntime/tickets/outcome_test.go b/backend/internal/codexruntime/tickets/outcome_test.go index 33c116b7cdf1..2eeb985a09bc 100644 --- a/backend/internal/codexruntime/tickets/outcome_test.go +++ b/backend/internal/codexruntime/tickets/outcome_test.go @@ -46,4 +46,15 @@ func TestRoutingOutcomeKeepsFailureCategories(t *testing.T) { t.Fatalf("failure category was lost: %s", raw) } } + // Unknown codes keep their name; the original error and upstream facts are shown. + observation := &extensionv1.CodexRoutingObservation{Code: "routing_new_case", Error: "dial tcp 192.0.2.1:8080: connect: connection refused", UpstreamErrorCode: "rate_limit_exceeded", UpstreamErrorMessage: "Rate limit reached", RequestID: "req_fixture", CFRay: "ray-fixture", UpstreamBody: `{"error":{"code":"rate_limit_exceeded"}}`} + raw, err := json.Marshal(Outcome{Code: "routing_new_case", HTTPStatus: 429, Observation: observation}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"未归类的结果代码：routing_new_case", "connection refused", "rate_limit_exceeded", "Rate limit reached", "req_fixture", "ray-fixture", `\"code\":\"rate_limit_exceeded\"`} { + if !strings.Contains(string(raw), want) { + t.Fatalf("outcome lost %q: %s", want, raw) + } + } } diff --git a/backend/internal/codexruntime/tickets/proxy.go b/backend/internal/codexruntime/tickets/proxy.go index 80ff51036703..7157029eab58 100644 --- a/backend/internal/codexruntime/tickets/proxy.go +++ b/backend/internal/codexruntime/tickets/proxy.go @@ -9,9 +9,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "net/http" "net/http/httptrace" + "net/url" + "strings" "sync" "time" @@ -19,6 +22,8 @@ import ( proxytransport "github.com/Wei-Shaw/sub2api/internal/proxytransport" ) +const proxyTestTarget = "https://chatgpt.com/backend-api/codex/responses" + type ProxyTrust struct { Certificates [][]byte `json:"certificates"` Fingerprint string `json:"fingerprint"` @@ -29,15 +34,37 @@ type ProxyStage struct { DurationMS int64 `json:"duration_ms"` Message string `json:"message,omitempty"` } + +// ProxyCertificate is one certificate presented during a proxy test handshake: +// the HTTPS proxy's own chain, the target chain, or a TLS-inspecting proxy's. +type ProxyCertificate struct { + ServerName string `json:"server_name,omitempty"` + Subject string `json:"subject"` + Issuer string `json:"issuer"` + SerialNumber string `json:"serial_number,omitempty"` + NotBefore time.Time `json:"not_before"` + NotAfter time.Time `json:"not_after"` + DNSNames []string `json:"dns_names,omitempty"` + SHA256 string `json:"sha256"` + SPKISHA256 string `json:"spki_sha256"` +} + +// A proxy test reports every stage with its original error text. FailureDetail +// is the complete error of the failed request; nothing is redacted because the +// administrator configured (and can read) the proxy URL itself. type ProxyResult struct { - Success bool `json:"success"` - NetworkReachable bool `json:"network_reachable"` - Protocol string `json:"protocol"` - HTTPStatus int `json:"http_status,omitempty"` - Code string `json:"code"` - Stages []ProxyStage `json:"stages"` - CertificateTrust string `json:"certificate_trust,omitempty"` - CertificateFingerprint string `json:"certificate_fingerprint,omitempty"` + Success bool `json:"success"` + NetworkReachable bool `json:"network_reachable"` + Protocol string `json:"protocol"` + HTTPStatus int `json:"http_status,omitempty"` + Code string `json:"code"` + Stages []ProxyStage `json:"stages"` + CertificateTrust string `json:"certificate_trust,omitempty"` + CertificateFingerprint string `json:"certificate_fingerprint,omitempty"` + Certificates []ProxyCertificate `json:"certificates,omitempty"` + CertificateError string `json:"certificate_error,omitempty"` + FailureDetail string `json:"failure_detail,omitempty"` + ProtocolSuggestion string `json:"protocol_suggestion,omitempty"` } func verifyProxyCertificate(state tls.ConnectionState, trust *ProxyTrust, learn bool) (*ProxyTrust, error) { @@ -54,62 +81,279 @@ func verifyProxyCertificate(state tls.ConnectionState, trust *ProxyTrust, learn intermediates.AddCert(cert) } opts := x509.VerifyOptions{DNSName: name, Intermediates: intermediates} - if _, err := leaf.Verify(opts); err == nil { + systemErr := func() error { _, err := leaf.Verify(opts); return err }() + if systemErr == nil { return nil, nil } if name != "chatgpt.com" { - return nil, errors.New("untrusted proxy TLS issuer") + return nil, fmt.Errorf("untrusted proxy TLS issuer for %s: %w", name, systemErr) } + var pinnedErr error if trust != nil && len(trust.Certificates) > 0 { roots := x509.NewCertPool() for _, der := range trust.Certificates { cert, err := x509.ParseCertificate(der) if err != nil { - return nil, err + return nil, fmt.Errorf("stored pinned proxy certificate cannot be parsed: %w", err) } roots.AddCert(cert) } opts.Roots = roots - if _, err := leaf.Verify(opts); err == nil { + if _, pinnedErr = leaf.Verify(opts); pinnedErr == nil { return trust, nil } } if !learn { - return nil, errors.New("proxy certificate changed or is untrusted") + if pinnedErr != nil { + return nil, fmt.Errorf("proxy certificate changed or is untrusted: system roots: %v; pinned certificate %s: %v", systemErr, trust.Fingerprint, pinnedErr) + } + return nil, fmt.Errorf("proxy certificate changed or is untrusted: system roots: %v; no pinned certificate", systemErr) } anchor := state.PeerCertificates[len(state.PeerCertificates)-1] - if time.Now().Before(anchor.NotBefore) || !time.Now().Before(anchor.NotAfter) { - return nil, errors.New("proxy certificate outside validity period") + if now := time.Now(); now.Before(anchor.NotBefore) || !now.Before(anchor.NotAfter) { + return nil, fmt.Errorf("proxy certificate outside validity period: %q valid from %s to %s", anchor.Subject.String(), anchor.NotBefore.UTC().Format(time.RFC3339), anchor.NotAfter.UTC().Format(time.RFC3339)) } roots := x509.NewCertPool() roots.AddCert(anchor) opts.Roots = roots if _, err := leaf.Verify(opts); err != nil { - return nil, err + return nil, fmt.Errorf("proxy certificate chain does not verify against its anchor %q: %w", anchor.Subject.String(), err) } fingerprint := sha256.Sum256(anchor.RawSubjectPublicKeyInfo) return &ProxyTrust{Certificates: [][]byte{anchor.Raw}, Fingerprint: hex.EncodeToString(fingerprint[:])}, nil } -func ticketProxyTransport(raw string, trust *ProxyTrust, learn bool, observe func(*ProxyTrust)) (*http.Transport, error) { +// proxyObserver collects stages and certificate facts from transport callbacks, +// which may run on dial goroutines. Results copy a snapshot, never the slices. +type proxyObserver struct { + mu sync.Mutex + start time.Time + prefix string + stages []ProxyStage + certificates []ProxyCertificate + certificateError string + connectStatus int + learned *ProxyTrust +} + +func newProxyObserver(prefix string) *proxyObserver { + return &proxyObserver{start: time.Now(), prefix: prefix} +} + +func (o *proxyObserver) add(stage ProxyStage) { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + stage.Name = o.prefix + stage.Name + if stage.DurationMS == 0 { + stage.DurationMS = time.Since(o.start).Milliseconds() + } + o.stages = append(o.stages, stage) +} + +func (o *proxyObserver) inspect(state tls.ConnectionState) { + if o == nil { + return + } + described := describeProxyCertificates(state.ServerName, state.PeerCertificates) + o.mu.Lock() + defer o.mu.Unlock() + for _, certificate := range described { + duplicate := false + for _, existing := range o.certificates { + if existing.ServerName == certificate.ServerName && existing.SHA256 == certificate.SHA256 { + duplicate = true + break + } + } + if !duplicate { + o.certificates = append(o.certificates, certificate) + } + } +} + +func (o *proxyObserver) verified(next *ProxyTrust, err error) { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if err != nil { + o.certificateError = err.Error() + return + } + o.learned = next +} + +func (o *proxyObserver) connectResponse(_ context.Context, _ *url.URL, _ *http.Request, response *http.Response) error { + o.mu.Lock() + o.connectStatus = response.StatusCode + o.mu.Unlock() + message := "HTTP " + response.Status + if challenge := response.Header.Get("Proxy-Authenticate"); challenge != "" { + message += " (Proxy-Authenticate: " + challenge + ")" + } + o.add(ProxyStage{Name: "connect", Success: response.StatusCode == http.StatusOK, Message: message}) + return nil +} + +func (o *proxyObserver) trace() *httptrace.ClientTrace { + return &httptrace.ClientTrace{ + DNSDone: func(info httptrace.DNSDoneInfo) { + message := "" + if info.Err != nil { + message = info.Err.Error() + } else { + addresses := make([]string, 0, len(info.Addrs)) + for _, address := range info.Addrs { + addresses = append(addresses, address.String()) + } + message = strings.Join(addresses, ", ") + } + o.add(ProxyStage{Name: "dns", Success: info.Err == nil, Message: message}) + }, + ConnectDone: func(network, addr string, err error) { + message := strings.TrimSpace(network + " " + addr) + if err != nil { + message += ": " + err.Error() + } + o.add(ProxyStage{Name: "tcp", Success: err == nil, Message: message}) + }, + TLSHandshakeDone: func(state tls.ConnectionState, err error) { + message := "" + if err != nil { + message = err.Error() + } else { + message = strings.TrimSpace(state.ServerName + " " + tls.VersionName(state.Version) + " " + tls.CipherSuiteName(state.CipherSuite) + " " + state.NegotiatedProtocol) + } + o.add(ProxyStage{Name: "tls", Success: err == nil, Message: message}) + }, + } +} + +// snapshot copies the observed facts into result and returns the trust learned +// by the last successful handshake. +func (o *proxyObserver) snapshot(result *ProxyResult) *ProxyTrust { + o.mu.Lock() + defer o.mu.Unlock() + result.Stages = append(result.Stages, o.stages...) + for _, certificate := range o.certificates { + duplicate := false + for _, existing := range result.Certificates { + if existing.ServerName == certificate.ServerName && existing.SHA256 == certificate.SHA256 { + duplicate = true + break + } + } + if !duplicate { + result.Certificates = append(result.Certificates, certificate) + } + } + if o.certificateError != "" { + result.CertificateError = o.certificateError + } + return o.learned +} + +func describeProxyCertificates(serverName string, chain []*x509.Certificate) []ProxyCertificate { + out := make([]ProxyCertificate, 0, len(chain)) + for _, cert := range chain { + if cert == nil { + continue + } + der := sha256.Sum256(cert.Raw) + spki := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + described := ProxyCertificate{ServerName: serverName, Subject: cert.Subject.String(), Issuer: cert.Issuer.String(), NotBefore: cert.NotBefore.UTC(), NotAfter: cert.NotAfter.UTC(), DNSNames: append([]string(nil), cert.DNSNames...), SHA256: hex.EncodeToString(der[:]), SPKISHA256: hex.EncodeToString(spki[:])} + if cert.SerialNumber != nil { + described.SerialNumber = cert.SerialNumber.Text(16) + } + out = append(out, described) + } + return out +} + +func ticketProxyTransport(raw string, trust *ProxyTrust, learn bool, observer *proxyObserver) (*http.Transport, error) { address, err := proxytransport.ParseEndpoint(raw) if err != nil { - return nil, errors.New("invalid proxy") + return nil, fmt.Errorf("invalid proxy: %w", err) } transport := &http.Transport{DialContext: (&net.Dialer{Timeout: 8 * time.Second}).DialContext, TLSHandshakeTimeout: 8 * time.Second, ResponseHeaderTimeout: 12 * time.Second, DisableKeepAlives: true, DisableCompression: true, ForceAttemptHTTP2: false, TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}} transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true, VerifyConnection: func(state tls.ConnectionState) error { // #nosec G402 -- hostname, time and constrained roots are verified explicitly. + observer.inspect(state) next, err := verifyProxyCertificate(state, trust, learn) - if err == nil && observe != nil { - observe(next) - } + observer.verified(next, err) return err }} + if observer != nil { + transport.OnProxyConnectResponse = observer.connectResponse + } if err := proxytransport.Configure(transport, address, nil); err != nil { return nil, err } return transport, nil } +// failureCode classifies a failed proxy request for the result catalog; the +// observed CONNECT status decides proxy authentication and rejection. +// FailureDetail always carries the unclassified original error. +func (o *proxyObserver) failureCode(err error) string { + o.mu.Lock() + status := o.connectStatus + o.mu.Unlock() + switch { + case status == http.StatusProxyAuthRequired: + return "ticket_proxy_auth" + case status != 0 && status != http.StatusOK: + return "ticket_proxy_connect" + } + return proxyFailureCode(err) +} + +// proxyFailureCode classifies a failed proxy request by its error alone. +func proxyFailureCode(err error) string { + if errors.Is(err, context.Canceled) { + return "ticket_canceled" + } + var timeout net.Error + if errors.Is(err, context.DeadlineExceeded) || errors.As(err, &timeout) && timeout.Timeout() { + return "ticket_timeout" + } + text := strings.ToLower(err.Error()) + switch { + case strings.Contains(text, "authentication failed"): // SOCKS5 username/password rejection + return "ticket_proxy_auth" + case strings.Contains(text, "certificate") || strings.Contains(text, "tls") || strings.Contains(text, "x509"): + return "ticket_proxy_tls" + case strings.Contains(text, "no such host") || strings.Contains(text, "name resolution"): + return "ticket_proxy_dns" + case strings.Contains(text, "malformed http response") || strings.Contains(text, "unexpected protocol") || strings.Contains(text, "socks version"): + return "ticket_proxy_protocol" + case strings.Contains(text, "connection refused") || strings.Contains(text, "connect rejected"): + return "ticket_proxy_connect" + case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): + return "ticket_proxy_eof" + case strings.Contains(text, "connection reset") || strings.Contains(text, "forcibly closed"): + return "ticket_proxy_reset" + } + return "proxy_connection_failed" +} + +func proxyProtocolSuggestion(scheme string, err error) string { + text := strings.ToLower(err.Error()) + var timeout net.Error + switch { + case strings.HasPrefix(scheme, "socks") && (errors.Is(err, context.DeadlineExceeded) || errors.As(err, &timeout) && timeout.Timeout()): + return "尝试确认该端口是否为HTTP CONNECT代理" + case scheme == "https" && strings.Contains(text, "first record does not look like a tls handshake"): + return "该端口未使用TLS，请尝试 http 协议" + case (scheme == "http" || scheme == "https") && strings.Contains(text, "malformed http response"): + return "该端口可能不是HTTP代理，请尝试 socks5 或 socks5h 协议" + } + return "" +} + func proxyStateKey(raw string) string { sum := sha256.Sum256([]byte(raw + "\x00chatgpt.com")) return hex.EncodeToString(sum[:]) @@ -118,65 +362,57 @@ func proxyStateKey(raw string) string { func loadProxyTrust(ctx context.Context, host HostCaller, raw string) (*ProxyTrust, int64, error) { var state extensionv1.StateResult if err := hostCall(ctx, host, extensionv1.HostStateRead, extensionv1.StateRequest{Namespace: "proxy-trust", Key: proxyStateKey(raw)}, &state); err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("read pinned proxy certificate: %w", err) } if !state.Found { return nil, 0, nil } var trust ProxyTrust if err := json.Unmarshal(state.Value, &trust); err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("decode pinned proxy certificate: %w", err) } return &trust, state.Revision, nil } +func noProxyRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + func prepareProxy(ctx context.Context, host HostCaller, raw string, explicitTest bool) (*http.Client, *ProxyResult, error) { address, err := proxytransport.ParseEndpoint(raw) result := &ProxyResult{Stages: []ProxyStage{}, Code: "invalid_proxy"} if err != nil { - return nil, result, errors.New("invalid proxy") + result.FailureDetail = err.Error() + return nil, result, fmt.Errorf("invalid proxy: %w", err) } normal := raw result.Protocol = address.Scheme trust, revision, err := loadProxyTrust(ctx, host, normal) if err != nil { + result.Code, result.FailureDetail = "proxy_trust_unavailable", err.Error() return nil, result, err } - var mu sync.Mutex - var observed *ProxyTrust - transport, err := ticketProxyTransport(normal, trust, explicitTest || trust == nil, func(next *ProxyTrust) { mu.Lock(); observed = next; mu.Unlock() }) + observer := newProxyObserver("") + transport, err := ticketProxyTransport(normal, trust, explicitTest || trust == nil, observer) if err != nil { + result.FailureDetail = err.Error() return nil, result, err } defer transport.CloseIdleConnections() - client := &http.Client{Transport: transport, Timeout: 25 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} - start := time.Now() - trace := &httptrace.ClientTrace{ - ConnectDone: func(_, _ string, err error) { - mu.Lock() - defer mu.Unlock() - result.Stages = append(result.Stages, ProxyStage{Name: "tcp", Success: err == nil, DurationMS: time.Since(start).Milliseconds()}) - }, - TLSHandshakeDone: func(_ tls.ConnectionState, err error) { - mu.Lock() - defer mu.Unlock() - result.Stages = append(result.Stages, ProxyStage{Name: "tls", Success: err == nil, DurationMS: time.Since(start).Milliseconds()}) - }, - } - request, _ := http.NewRequestWithContext(httptrace.WithClientTrace(ctx, trace), http.MethodHead, "https://chatgpt.com/backend-api/codex/responses", nil) + client := &http.Client{Transport: transport, Timeout: 25 * time.Second, CheckRedirect: noProxyRedirect} + request, _ := http.NewRequestWithContext(httptrace.WithClientTrace(ctx, observer.trace()), http.MethodHead, proxyTestTarget, nil) response, err := client.Do(request) if err != nil { - result.Code = "proxy_connection_failed" - return nil, result, errors.New("proxy connection or certificate validation failed") + observer.snapshot(result) + result.Code = observer.failureCode(err) + result.FailureDetail = err.Error() + result.ProtocolSuggestion = proxyProtocolSuggestion(address.Scheme, err) + return nil, result, fmt.Errorf("proxy connection or certificate validation failed: %w", err) } _ = response.Body.Close() + observer.add(ProxyStage{Name: "http", Success: response.StatusCode >= 200 && response.StatusCode < 400, Message: "HTTP " + response.Status}) + learned := observer.snapshot(result) result.HTTPStatus = response.StatusCode result.NetworkReachable = true result.Code = "target_http_status" - result.Stages = append(result.Stages, ProxyStage{Name: "http", Success: response.StatusCode >= 200 && response.StatusCode < 400, Message: fmt.Sprintf("HTTP %d", response.StatusCode)}) - mu.Lock() - learned := observed - mu.Unlock() result.CertificateTrust = "system" if learned != nil { result.CertificateTrust = "proxy_certificate_pinned" @@ -184,28 +420,50 @@ func prepareProxy(ctx context.Context, host HostCaller, raw string, explicitTest if trust == nil || trust.Fingerprint != learned.Fingerprint { rawTrust, _ := json.Marshal(learned) var saved extensionv1.StateResult - if err := hostCall(ctx, host, extensionv1.HostStateCompareSwap, extensionv1.StateRequest{Namespace: "proxy-trust", Key: proxyStateKey(normal), ExpectedRevision: revision, Value: rawTrust}, &saved); err != nil || !saved.Applied { - return nil, result, errors.New("proxy trust changed while preparing connection") + saveErr := hostCall(ctx, host, extensionv1.HostStateCompareSwap, extensionv1.StateRequest{Namespace: "proxy-trust", Key: proxyStateKey(normal), ExpectedRevision: revision, Value: rawTrust}, &saved) + if saveErr == nil && !saved.Applied { + saveErr = fmt.Errorf("pinned proxy certificate changed concurrently (expected revision %d, current revision %d)", revision, saved.Revision) + } + if saveErr != nil { + result.Code, result.FailureDetail = "proxy_trust_changed", saveErr.Error() + return nil, result, fmt.Errorf("proxy trust changed while preparing connection: %w", saveErr) } } } finalTransport, err := ticketProxyTransport(normal, learned, false, nil) if err != nil { + result.FailureDetail = err.Error() return nil, result, err } - finalClient := &http.Client{Transport: finalTransport, Timeout: 25 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + finalClient := &http.Client{Transport: finalTransport, Timeout: 25 * time.Second, CheckRedirect: noProxyRedirect} if explicitTest { - request, _ = http.NewRequestWithContext(ctx, http.MethodHead, "https://chatgpt.com/backend-api/codex/responses", nil) - response, err = finalClient.Do(request) + // The pinned check uses the same verifier as finalClient on its own + // transport, so its observer never touches the returned client. + checkObserver := newProxyObserver("pinned_") + checkTransport, err := ticketProxyTransport(normal, learned, false, checkObserver) + if err != nil { + finalTransport.CloseIdleConnections() + result.FailureDetail = err.Error() + return nil, result, err + } + started := time.Now() + request, _ = http.NewRequestWithContext(httptrace.WithClientTrace(ctx, checkObserver.trace()), http.MethodHead, proxyTestTarget, nil) + response, err = (&http.Client{Transport: checkTransport, Timeout: 25 * time.Second, CheckRedirect: noProxyRedirect}).Do(request) + checkTransport.CloseIdleConnections() + checkObserver.snapshot(result) if err != nil { finalTransport.CloseIdleConnections() + result.Success = false + // The pinned re-test failed after a successful first connection. result.Code = "pinned_connection_failed" - result.Stages = append(result.Stages, ProxyStage{Name: "pinned_connection", Success: false}) - return nil, result, errors.New("pinned proxy connection failed") + result.FailureDetail = err.Error() + result.ProtocolSuggestion = proxyProtocolSuggestion(address.Scheme, err) + result.Stages = append(result.Stages, ProxyStage{Name: "pinned_connection", Success: false, DurationMS: time.Since(started).Milliseconds(), Message: err.Error()}) + return nil, result, fmt.Errorf("pinned proxy connection failed: %w", err) } _ = response.Body.Close() result.HTTPStatus = response.StatusCode - result.Stages = append(result.Stages, ProxyStage{Name: "pinned_connection", Success: true, Message: fmt.Sprintf("HTTP %d", response.StatusCode)}) + result.Stages = append(result.Stages, ProxyStage{Name: "pinned_connection", Success: true, DurationMS: time.Since(started).Milliseconds(), Message: "HTTP " + response.Status}) } result.Success = result.HTTPStatus >= 200 && result.HTTPStatus < 400 if result.Success { diff --git a/backend/internal/codexruntime/tickets/proxy_test.go b/backend/internal/codexruntime/tickets/proxy_test.go index ca918109c16f..c2e94c16c46a 100644 --- a/backend/internal/codexruntime/tickets/proxy_test.go +++ b/backend/internal/codexruntime/tickets/proxy_test.go @@ -16,6 +16,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "testing" "time" @@ -133,11 +134,23 @@ func TestProxyPreflightAndHarvestUseSamePinnedTransport(t *testing.T) { })) t.Cleanup(proxy.Close) address, _ := url.Parse(proxy.URL) - address.User = url.UserPassword("test", "test") host := &memoryHost{state: make(map[string]extensionv1.StateResult), lease: make(map[string]int64)} clientAPI := testHostClient(t, host) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + // A rejected CONNECT keeps its original status and error text per stage. + _, rejected, err := prepareProxy(ctx, clientAPI, address.String(), true) + if err == nil || rejected == nil || rejected.Code != "ticket_proxy_auth" || !strings.Contains(rejected.FailureDetail, "Proxy Authentication Required") { + t.Fatalf("CONNECT 407 lost its original cause: %v %+v", err, rejected) + } + connectStage := false + for _, stage := range rejected.Stages { + connectStage = connectStage || stage.Name == "connect" && !stage.Success && strings.Contains(stage.Message, "407") + } + if !connectStage { + t.Fatalf("CONNECT stage lost its status: %+v", rejected.Stages) + } + address.User = url.UserPassword("test", "test") client, result, err := prepareProxy(ctx, clientAPI, address.String(), true) if err != nil { t.Fatal(err) @@ -146,6 +159,9 @@ func TestProxyPreflightAndHarvestUseSamePinnedTransport(t *testing.T) { if result.Success || !result.NetworkReachable || result.HTTPStatus != 405 || result.CertificateTrust != "proxy_certificate_pinned" { t.Fatalf("preflight confused transport success with inference success: %+v", result) } + if len(result.Certificates) == 0 || !strings.Contains(result.Certificates[0].Subject, "chatgpt.com") || result.Certificates[0].NotAfter.IsZero() || result.Certificates[0].SPKISHA256 != result.CertificateFingerprint { + t.Fatalf("certificate facts missing: %+v", result.Certificates) + } request, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", strings.NewReader(`{"model":"synthetic-model"}`)) if err != nil { t.Fatal(err) @@ -168,3 +184,52 @@ func TestProxyPreflightAndHarvestUseSamePinnedTransport(t *testing.T) { t.Fatal("OAuth credential appeared in a preflight") } } + +func TestProxyPinnedRetestFailureKeepsItsCode(t *testing.T) { + first, _ := syntheticCertificate(t, "chatgpt.com", false) + rotated, _ := syntheticCertificate(t, "chatgpt.com", false) + var handshakes atomic.Int32 + target := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusMethodNotAllowed) })) + target.TLS = &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + if handshakes.Add(1) == 1 { + return &first, nil + } + return &rotated, nil + }} + target.StartTLS() + t.Cleanup(target.Close) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hijacker, ok := w.(http.Hijacker) + if r.Method != http.MethodConnect || !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + upstream, err := net.Dial("tcp", target.Listener.Addr().String()) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + client, _, err := hijacker.Hijack() + if err != nil { + _ = upstream.Close() + return + } + _, _ = io.WriteString(client, "HTTP/1.1 200 Connection Established\r\n\r\n") + go func() { + defer func() { _ = client.Close() }() + defer func() { _ = upstream.Close() }() + go func() { _, _ = io.Copy(upstream, client) }() + _, _ = io.Copy(client, upstream) + }() + })) + t.Cleanup(proxy.Close) + host := &memoryHost{state: make(map[string]extensionv1.StateResult), lease: make(map[string]int64)} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + // The first connection learns the certificate; the pinned re-test sees a + // rotated one and must report pinned_connection_failed with its cause. + _, result, err := prepareProxy(ctx, testHostClient(t, host), proxy.URL, true) + if err == nil || result == nil || result.Code != "pinned_connection_failed" || result.FailureDetail == "" || result.CertificateError == "" { + t.Fatalf("pinned re-test failure lost its code: %v %+v", err, result) + } +} diff --git a/backend/internal/codexruntime/tickets/routing.go b/backend/internal/codexruntime/tickets/routing.go index 9e3f8b3a4612..3484f9fe882e 100644 --- a/backend/internal/codexruntime/tickets/routing.go +++ b/backend/internal/codexruntime/tickets/routing.go @@ -105,7 +105,13 @@ func (m *Module) harvest(ctx, epoch context.Context, host HostCaller, cfg Config observation := extensionv1.CodexRoutingObservation{Stage: "configuration", Code: "routing_unavailable", RequestedModel: req.Model, ObservedAt: now} var scope extensionv1.CodexRoutingScope var qualified *extensionv1.CodexRoutingQualification - if scopeErr := hostCall(ctx, host, extensionv1.HostCodexRoutingScope, extensionv1.CodexRoutingQuery{AccountID: account.ID, Transport: "http"}, &scope); scopeErr == nil && scope.Identity == account.Identity { + scopeErr := hostCall(ctx, host, extensionv1.HostCodexRoutingScope, extensionv1.CodexRoutingQuery{AccountID: account.ID, Transport: "http"}, &scope) + switch { + case scopeErr != nil: + observation.Error = "routing scope: " + scopeErr.Error() + case scope.Identity != account.Identity: + observation.Error = fmt.Sprintf("routing scope identity %s does not match account identity %s", scope.Identity, account.Identity) + default: query := extensionv1.CodexRoutingQuery{AccountID: account.ID, Model: req.Model, Transport: "http", Stage: "acquire", OperationID: req.OperationID, Scope: &scope} var candidate extensionv1.CodexRoutingProbeResult if err := hostCall(ctx, host, extensionv1.HostCodexRoutingProbe, query, &candidate); err == nil { @@ -121,21 +127,34 @@ func (m *Module) harvest(ctx, epoch context.Context, host HostCaller, cfg Config } } else { observation.Code = "routing_transport" + observation.Error = "verify: " + err.Error() } } } else { observation.Code = "routing_transport" + observation.Error = "acquire: " + err.Error() } } finish, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer stop() var current extensionv1.Account if err := hostCall(finish, host, extensionv1.HostAccountRead, extensionv1.AccountQuery{AccountID: account.ID}, ¤t); err != nil || current.Identity != account.Identity { - return Outcome{Code: "ticket_stale"}, nil + stale := observation + if err != nil { + stale.Error = appendObservationError(stale.Error, "re-read account: "+err.Error()) + } else { + stale.Error = appendObservationError(stale.Error, fmt.Sprintf("account identity changed from %s to %s during the operation", account.Identity, current.Identity)) + } + return Outcome{Code: "ticket_stale", Observation: &stale}, nil } if ctx.Err() != nil || epoch.Err() != nil { qualified = nil observation.Code = "ticket_canceled" + if ctx.Err() != nil { + observation.Error = appendObservationError(observation.Error, "operation: "+ctx.Err().Error()) + } else { + observation.Error = appendObservationError(observation.Error, "runtime stopped or reconfigured: "+epoch.Err().Error()) + } } state.completeRouting(time.Now().UTC(), qualified, observation) if _, err := writeState(finish, host, account.ID, req.Model, state, revision); err != nil { @@ -144,6 +163,13 @@ func (m *Module) harvest(ctx, epoch context.Context, host HostCaller, cfg Config return Outcome{Success: qualified != nil, Code: observation.Code, HTTPStatus: observation.HTTPStatus, ObservedLength: observation.StateLength, ExpiresAt: state.ExpiresAt, Observation: &observation}, nil } +func appendObservationError(existing, next string) string { + if existing == "" { + return next + } + return existing + "; " + next +} + func (m *Module) migrateRoutingStates(ctx context.Context, host HostCaller, cfg Config) { var accounts []extensionv1.Account if hostCall(ctx, host, extensionv1.HostAccountList, extensionv1.AccountQuery{Platform: "openai", IncludeInactive: true}, &accounts) != nil { diff --git a/backend/internal/codexruntime/tickets/routing_test.go b/backend/internal/codexruntime/tickets/routing_test.go index 1e8df6d1a1f9..0817bd138cb1 100644 --- a/backend/internal/codexruntime/tickets/routing_test.go +++ b/backend/internal/codexruntime/tickets/routing_test.go @@ -76,13 +76,16 @@ func TestRoutingStaleHostReferenceWithdrawsOnlyItsQualification(t *testing.T) { module.config.Enabled = true raw, _ = json.Marshal(extensionv1.SchedulingRequest{Account: account, Model: "gpt-6-astra", Now: now}) result, err := module.Invoke(context.Background(), extensionv1.Invocation{Capability: extensionv1.CapabilityRequest, Operation: "inject", Payload: raw}) - if err != nil || result.Code != "routing_stale" { - t.Fatal("stale bundle was accepted") + if err != nil || result.Code != "routing_stale" || result.Message == "" { + t.Fatalf("stale bundle was accepted or lost its reason: %+v %v", result, err) } var state State if json.Unmarshal(host.state[stateKey(7, "gpt-6-astra")].Value, &state) != nil || state.Qualification != nil || state.Phase != "needs_cookie_verification" || !state.Enrolled || state.Failures != 0 { t.Fatal("stale reference did not withdraw exactly the old qualification") } + if state.RevocationReason != result.Message || state.RevokedAt == nil || state.Observation != nil { + t.Fatalf("withdrawal must be recorded apart from the last real observation: %+v", state) + } } func TestRoutingDemandRearmsAnEnrolledDormantState(t *testing.T) { diff --git a/backend/internal/codexruntime/tickets/state.go b/backend/internal/codexruntime/tickets/state.go index c8a1adf3166b..4d7f33afb2dc 100644 --- a/backend/internal/codexruntime/tickets/state.go +++ b/backend/internal/codexruntime/tickets/state.go @@ -27,6 +27,10 @@ type State struct { ManualWasEnrolled bool `json:"manual_was_enrolled,omitempty"` LastCode string `json:"last_code,omitempty"` LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + // A host check that withdrew the qualification records its reason here; + // Observation keeps the last real upstream observation. + RevokedAt *time.Time `json:"revoked_at,omitempty"` + RevocationReason string `json:"revocation_reason,omitempty"` // Opaque only so every legacy representation can be discarded without // interpreting its length, timestamps or account-plan-specific format. Ticket json.RawMessage `json:"ticket,omitempty"` @@ -70,6 +74,7 @@ func (s *State) beginRouting(now time.Time, manual bool) error { func (s *State) completeRouting(now time.Time, q *extensionv1.CodexRoutingQualification, observation extensionv1.CodexRoutingObservation) { manual := s.Phase == "manual_running" s.Observation, s.LastCode = &observation, observation.Code + s.RevokedAt, s.RevocationReason = nil, "" if q != nil { s.Qualification, s.ExpiresAt = q, &q.ExpiresAt s.Enrolled, s.Failures, s.Phase = true, 0, "ready" diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 665c39b42ea1..ea321e9ba9e4 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -32,7 +32,7 @@ const ( // DefaultCSPPolicy is the default Content-Security-Policy with nonce support // __CSP_NONCE__ will be replaced with actual nonce at request time by the SecurityHeaders middleware -const DefaultCSPPolicy = "default-src 'self'; worker-src 'self' blob:; script-src 'self' __CSP_NONCE__ https://challenges.cloudflare.com https://*.alicdn.com https://static.cloudflareinsights.com https://turing.captcha.qcloud.com https://turing.captcha.gtimg.com https://ca.turing.captcha.qcloud.com https://global.turing.captcha.gtimg.com https://www.tycaptcha.com https://cloudcache.tencentcs.com https://*.stripe.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; style-src 'self' 'unsafe-inline' https://*.captcha.gtimg.com https://fonts.googleapis.com https://*.alicdn.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; img-src 'self' data: blob: https:; font-src 'self' data: https://fonts.gstatic.com; connect-src 'self' https://turing.captcha.qcloud.com https://www.tycaptcha.com https://rce.tencentrio.com https:; frame-src 'self' https://challenges.cloudflare.com https://turing.captcha.qcloud.com https://ca.turing.captcha.qcloud.com https://www.tycaptcha.com https://*.stripe.com https://checkout.airwallex.com https://checkout-demo.airwallex.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" +const DefaultCSPPolicy = "default-src 'self'; worker-src 'self' blob:; script-src 'self' __CSP_NONCE__ https://challenges.cloudflare.com https://*.alicdn.com https://static.cloudflareinsights.com https://turing.captcha.qcloud.com https://turing.captcha.gtimg.com https://ca.turing.captcha.qcloud.com https://global.turing.captcha.gtimg.com https://www.tycaptcha.com https://cloudcache.tencentcs.com https://*.stripe.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; style-src 'self' 'unsafe-inline' https://*.captcha.gtimg.com https://fonts.googleapis.com https://font.sec.miui.com https://*.alicdn.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; img-src 'self' data: blob: https:; font-src 'self' data: https://fonts.gstatic.com https://cdn-file.hyperos.mi.com; connect-src 'self' https://turing.captcha.qcloud.com https://www.tycaptcha.com https://rce.tencentrio.com https:; frame-src 'self' https://challenges.cloudflare.com https://turing.captcha.qcloud.com https://ca.turing.captcha.qcloud.com https://www.tycaptcha.com https://*.stripe.com https://checkout.airwallex.com https://checkout-demo.airwallex.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" // UMQ（用户消息队列）模式常量 const ( diff --git a/backend/internal/handler/admin/account_codex_import.go b/backend/internal/handler/admin/account_codex_import.go index 8f2092167592..90c1ea51c0da 100644 --- a/backend/internal/handler/admin/account_codex_import.go +++ b/backend/internal/handler/admin/account_codex_import.go @@ -118,7 +118,7 @@ type codexAccountIndex struct { func (h *AccountHandler) ImportCodexSession(c *gin.Context) { var req CodexSessionImportRequest if err := c.ShouldBindJSON(&req); err != nil { - response.BadRequest(c, "Invalid Codex session import request") + response.BadRequest(c, "Invalid Codex session import request: "+err.Error()) return } if err := service.ValidateOpenAIReasoningPolicyExtra(req.Extra); err != nil { @@ -127,7 +127,7 @@ func (h *AccountHandler) ImportCodexSession(c *gin.Context) { } entries, err := parseCodexSessionImportEntries(req) if err != nil { - response.BadRequest(c, "Invalid Codex session import content") + response.BadRequest(c, "Invalid Codex session import content: "+err.Error()) return } if len(entries) == 0 { diff --git a/backend/internal/handler/admin/account_codex_import_job_test.go b/backend/internal/handler/admin/account_codex_import_job_test.go index e6b0bc1564e3..7d8df8abbafe 100644 --- a/backend/internal/handler/admin/account_codex_import_job_test.go +++ b/backend/internal/handler/admin/account_codex_import_job_test.go @@ -246,9 +246,8 @@ func TestAccountJobCodexImportPreservesReplayRetryAndPayloadBoundary(t *testing. require.Equal(t, service.AccountJobItemStatusSucceeded, results[2].Status) require.Equal(t, 2, len(svc.createdAccounts)) require.Equal(t, 3, svc.createAttempts) - if strings.Contains(results[1].ErrorMessage+string(results[1].Metadata), tokens[1]) { - t.Fatal("synthetic backend error leaked a credential into the job result") - } + // The job result keeps the backend error text verbatim for administrators. + require.Equal(t, "synthetic backend failure: "+tokens[1], results[1].ErrorMessage) repo.failedSeeds = []service.AccountJobItemSeed{params.Items[1]} retry, replayed, err := jobs.RetryFailed(context.Background(), 1, accountJobTestActorID, "retry-fixture") require.NoError(t, err) diff --git a/backend/internal/handler/admin/account_codex_quality.go b/backend/internal/handler/admin/account_codex_quality.go index 4711acb3ee8e..69f03dc5c775 100644 --- a/backend/internal/handler/admin/account_codex_quality.go +++ b/backend/internal/handler/admin/account_codex_quality.go @@ -2,6 +2,7 @@ package admin import ( "context" + "fmt" "strconv" "github.com/Wei-Shaw/sub2api/internal/pkg/response" @@ -15,45 +16,71 @@ func codexQualityAdmin(c *gin.Context) (int64, int64, bool) { actor, ok := middleware.GetAuthSubjectFromContext(c) role, roleOK := middleware.GetUserRoleFromContext(c) id, err := strconv.ParseInt(c.Param("id"), 10, 64) - if !ok || !roleOK || role != "admin" || actor.UserID <= 0 || err != nil || id <= 0 { - response.Error(c, 403, "Codex quality run unavailable") - return 0, 0, false + switch { + case !ok || actor.UserID <= 0: + response.Error(c, 403, "Codex quality run unavailable: no authenticated administrator") + case !roleOK || role != "admin": + response.Error(c, 403, fmt.Sprintf("Codex quality run unavailable: role %q is not admin", role)) + case err != nil || id <= 0: + response.Error(c, 403, "Codex quality run unavailable: invalid account id "+strconv.Quote(c.Param("id"))) + default: + return actor.UserID, id, true } - return actor.UserID, id, true + return 0, 0, false } func writeCodexQualityResult(c *gin.Context, result *service.CodexQualityRunView, err error) { - if err != nil || result == nil { - response.Error(c, 409, "Codex quality run unavailable") + if err != nil { + response.Error(c, 409, "Codex quality run unavailable: "+err.Error()) + return + } + if result == nil { + response.Error(c, 409, "Codex quality run unavailable: the runtime returned no run") return } response.Success(c, result) } +func codexQualityKeyLookup(h *AccountHandler, actor int64) service.CodexQualityKeyLookup { + return func(ctx context.Context, keyID int64) (*service.APIKey, error) { + keys, _, err := h.adminService.GetUserAPIKeys(ctx, actor, 1, 1000, "", "") + if err != nil { + return nil, fmt.Errorf("list the API keys of administrator %d: %w", actor, err) + } + for index := range keys { + if keys[index].ID == keyID && keys[index].UserID == actor { + return &keys[index], nil + } + } + return nil, fmt.Errorf("API key %d is not one of the %d API keys of administrator %d", keyID, len(keys), actor) + } +} + func (h *AccountHandler) CreateCodexQualityRun(c *gin.Context) { actor, id, ok := codexQualityAdmin(c) if !ok { return } var req service.CodexQualityCreateRequest - if h.codexTicketGateway == nil || c.ShouldBindJSON(&req) != nil || req.APIKeyID <= 0 { - response.Error(c, 409, "Codex quality run unavailable") + if h.codexTicketGateway == nil { + response.Error(c, 409, "Codex quality run unavailable: Codex runtime unavailable") + return + } + if err := c.ShouldBindJSON(&req); err != nil { + response.Error(c, 409, "Codex quality run unavailable: invalid request: "+err.Error()) + return + } + if req.APIKeyID <= 0 { + response.Error(c, 409, "Codex quality run unavailable: api_key_id is required") return } // A global integration admin key resolves to a real administrator too. Only // that administrator's own existing downstream key can authorize this run. - keys, _, err := h.adminService.GetUserAPIKeys(c.Request.Context(), actor, 1, 1000, "", "") + key, err := codexQualityKeyLookup(h, actor)(c.Request.Context(), req.APIKeyID) if err != nil { - response.Error(c, 409, "Codex quality run unavailable") + response.Error(c, 409, "Codex quality run unavailable: "+err.Error()) return } - var key *service.APIKey - for index := range keys { - if keys[index].ID == req.APIKeyID && keys[index].UserID == actor { - key = &keys[index] - break - } - } result, err := h.codexTicketGateway.CreateCodexQualityRun(c.Request.Context(), actor, id, key, req) writeCodexQualityResult(c, result, err) } @@ -64,7 +91,7 @@ func (h *AccountHandler) ReadCodexQualityRun(c *gin.Context) { return } if h.codexTicketGateway == nil { - response.Error(c, 409, "Codex quality run unavailable") + response.Error(c, 409, "Codex quality run unavailable: Codex runtime unavailable") return } result, err := h.codexTicketGateway.ReadCodexQualityRun(c.Request.Context(), actor, id, c.Param("run_id")) @@ -79,23 +106,15 @@ func (h *AccountHandler) RenewCodexQualityRoute(c *gin.Context) { var req struct { OperationID string `json:"operation_id"` } - if h.codexTicketGateway == nil || c.ShouldBindJSON(&req) != nil { - response.Error(c, 409, "Codex quality run unavailable") + if h.codexTicketGateway == nil { + response.Error(c, 409, "Codex quality run unavailable: Codex runtime unavailable") return } - lookup := func(ctx context.Context, keyID int64) (*service.APIKey, error) { - keys, _, err := h.adminService.GetUserAPIKeys(ctx, actor, 1, 1000, "", "") - if err != nil { - return nil, service.ErrCodexQualityUnavailable - } - for index := range keys { - if keys[index].ID == keyID && keys[index].UserID == actor { - return &keys[index], nil - } - } - return nil, service.ErrCodexQualityUnavailable + if err := c.ShouldBindJSON(&req); err != nil { + response.Error(c, 409, "Codex quality run unavailable: invalid request: "+err.Error()) + return } - result, err := h.codexTicketGateway.RenewCodexQualityRoute(c.Request.Context(), actor, id, c.Param("run_id"), req.OperationID, lookup) + result, err := h.codexTicketGateway.RenewCodexQualityRoute(c.Request.Context(), actor, id, c.Param("run_id"), req.OperationID, codexQualityKeyLookup(h, actor)) writeCodexQualityResult(c, result, err) } @@ -105,7 +124,7 @@ func (h *AccountHandler) CloseCodexQualityRun(c *gin.Context) { return } if h.codexTicketGateway == nil { - response.Error(c, 409, "Codex quality run unavailable") + response.Error(c, 409, "Codex quality run unavailable: Codex runtime unavailable") return } result, err := h.codexTicketGateway.CloseCodexQualityRun(c.Request.Context(), actor, id, c.Param("run_id")) diff --git a/backend/internal/handler/admin/account_codex_routing.go b/backend/internal/handler/admin/account_codex_routing.go index 9975a2f109d2..f586c09b0570 100644 --- a/backend/internal/handler/admin/account_codex_routing.go +++ b/backend/internal/handler/admin/account_codex_routing.go @@ -21,7 +21,7 @@ func (h *AccountHandler) CodexFingerprint(c *gin.Context) { } view, err := h.codexTicketGateway.CodexFingerprint(c.Request.Context(), id) if err != nil { - response.Error(c, 503, "Codex fingerprint unavailable") + response.Error(c, 503, "Codex fingerprint unavailable: "+err.Error()) return } response.Success(c, view) @@ -31,8 +31,12 @@ func (h *AccountHandler) ValidateCodexRouting(c *gin.Context) { c.Header("Cache-Control", "no-store") id, err := strconv.ParseInt(c.Param("id"), 10, 64) var request service.CodexRoutingValidationRequest - if err != nil || id <= 0 || c.ShouldBindJSON(&request) != nil { - response.BadRequest(c, "Invalid Codex validation request") + if err != nil || id <= 0 { + response.BadRequest(c, "Invalid Codex validation request: account id "+strconv.Quote(c.Param("id"))) + return + } + if err := c.ShouldBindJSON(&request); err != nil { + response.BadRequest(c, "Invalid Codex validation request: "+err.Error()) return } if h.codexTicketGateway == nil { @@ -41,7 +45,7 @@ func (h *AccountHandler) ValidateCodexRouting(c *gin.Context) { } result, err := h.codexTicketGateway.ValidateCodexRouting(c.Request.Context(), id, request) if err != nil { - response.Error(c, 409, "Codex validation unavailable or budget already spent") + response.Error(c, 409, "Codex validation unavailable: "+err.Error()) return } response.Success(c, result) @@ -51,8 +55,12 @@ func (h *AccountHandler) SelectCodexProfile(c *gin.Context) { c.Header("Cache-Control", "no-store") id, err := strconv.ParseInt(c.Param("id"), 10, 64) var choice service.CodexProfileSelection - if err != nil || id <= 0 || c.ShouldBindJSON(&choice) != nil { - response.BadRequest(c, "Invalid profile selection") + if err != nil || id <= 0 { + response.BadRequest(c, "Invalid profile selection: account id "+strconv.Quote(c.Param("id"))) + return + } + if err := c.ShouldBindJSON(&choice); err != nil { + response.BadRequest(c, "Invalid profile selection: "+err.Error()) return } if h.codexTicketGateway == nil { @@ -60,7 +68,7 @@ func (h *AccountHandler) SelectCodexProfile(c *gin.Context) { return } if err := h.codexTicketGateway.SelectCodexProfile(c.Request.Context(), id, choice); err != nil { - response.Error(c, 409, "Account identity changed or profile unavailable") + response.Error(c, 409, "Profile not applied: "+err.Error()) return } response.Success(c, gin.H{"applied": true}) diff --git a/backend/internal/handler/admin/account_codex_stop_job.go b/backend/internal/handler/admin/account_codex_stop_job.go index b8f8d61f6497..299039e7002c 100644 --- a/backend/internal/handler/admin/account_codex_stop_job.go +++ b/backend/internal/handler/admin/account_codex_stop_job.go @@ -3,6 +3,7 @@ package admin import ( "context" "encoding/json" + "fmt" "strconv" "github.com/Wei-Shaw/sub2api/internal/pkg/response" @@ -22,8 +23,8 @@ func (h *AccountHandler) BatchStopCodexTicketRenewal(c *gin.Context) { func (h *AccountHandler) submitCodexTicketStop(c *gin.Context, single bool) { var req codexTicketHarvestRequest - if c.ShouldBindJSON(&req) != nil { - response.BadRequest(c, "请求格式不正确") + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, "请求格式不正确："+err.Error()) return } if single { @@ -51,12 +52,12 @@ func (h *AccountHandler) submitCodexTicketStop(c *gin.Context, single bool) { return } if len(accounts) != len(req.AccountIDs) { - response.BadRequest(c, "所选账号已发生变化") + response.BadRequest(c, fmt.Sprintf("所选账号已发生变化：请求 %d 个，找到 %d 个", len(req.AccountIDs), len(accounts))) return } for _, account := range accounts { if account.Platform != service.PlatformOpenAI || (account.Type != service.AccountTypeOAuth && account.Type != service.AccountTypeSetupToken) || account.IsShadow() { - response.BadRequest(c, "全部所选账号必须支持 Codex 路由续期") + response.BadRequest(c, fmt.Sprintf("全部所选账号必须支持 Codex 路由续期：账号 %d（%s）是 %s/%s，影子账号=%t", account.ID, account.Name, account.Platform, account.Type, account.IsShadow())) return } } @@ -76,23 +77,36 @@ func (h *AccountHandler) executeCodexTicketStop(ctx context.Context, raw json.Ra var metadata struct { Model string `json:"model_id"` } - if json.Unmarshal(raw, &req) != nil || json.Unmarshal(item.Metadata, &metadata) != nil || metadata.Model == "" { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return codexTicketJobFailure(item.ID, "payload_invalid", "decode the job payload: "+err.Error(), nil, false) + } + if err := json.Unmarshal(item.Metadata, &metadata); err != nil { + return codexTicketJobFailure(item.ID, "payload_invalid", "decode the item metadata: "+err.Error(), nil, false) + } + if metadata.Model == "" { + return codexTicketJobFailure(item.ID, "payload_invalid", "the item metadata has no model_id", nil, false) } id, ok := accountJobTarget(item) - if !ok || h.codexTicketGateway == nil { - return accountJobFailed(item.ID, "target_missing") + if !ok { + return codexTicketJobFailure(item.ID, "target_missing", "the job item has no target account", map[string]any{"model_id": metadata.Model}, false) + } + details := map[string]any{"account_id": id, "model_id": metadata.Model} + if h.codexTicketGateway == nil { + return codexTicketJobFailure(item.ID, "ticket_plugin_unavailable", "the Codex runtime gateway is not configured", details, false) } account, err := h.adminService.GetAccount(ctx, id) - if err != nil || account == nil || account.Platform != service.PlatformOpenAI || (account.Type != service.AccountTypeOAuth && account.Type != service.AccountTypeSetupToken) || account.IsShadow() { - return accountJobFailed(item.ID, "account_ineligible") + switch { + case err != nil: + return codexTicketJobFailure(item.ID, "account_ineligible", fmt.Sprintf("read account %d: %v", id, err), details, false) + case account == nil: + return codexTicketJobFailure(item.ID, "account_ineligible", fmt.Sprintf("account %d not found", id), details, false) + case account.Platform != service.PlatformOpenAI || (account.Type != service.AccountTypeOAuth && account.Type != service.AccountTypeSetupToken) || account.IsShadow(): + return codexTicketJobFailure(item.ID, "account_ineligible", fmt.Sprintf("account %d is %s/%s (shadow=%t); only OpenAI OAuth/setup-token accounts renew Codex routes", id, account.Platform, account.Type, account.IsShadow()), details, false) } if err := h.codexTicketGateway.StopCodexTicketRenewal(ctx, id, []string{metadata.Model}); err != nil { - failure := accountJobFailed(item.ID, "stop_failed") - if ctx.Err() != nil { - failure.Status = service.AccountJobItemStatusCanceled - } - return failure + details["error"] = err.Error() + return codexTicketJobFailure(item.ID, "stop_failed", err.Error(), details, ctx.Err() != nil) } - return accountJobSucceeded(item.ID, map[string]any{"account_id": id, "model_id": metadata.Model}) + details["code"], details["message"] = "ticket_stopped", service.CodexTicketFailure("ticket_stopped").Message + return accountJobSucceeded(item.ID, details) } diff --git a/backend/internal/handler/admin/account_codex_ticket.go b/backend/internal/handler/admin/account_codex_ticket.go index 15f5669b11d0..a20c584fd811 100644 --- a/backend/internal/handler/admin/account_codex_ticket.go +++ b/backend/internal/handler/admin/account_codex_ticket.go @@ -63,8 +63,8 @@ func (h *AccountHandler) BatchHarvestCodexTickets(c *gin.Context) { func (h *AccountHandler) submitCodexTicketHarvest(c *gin.Context, single bool) { var req codexTicketHarvestRequest - if c.ShouldBindJSON(&req) != nil { - response.BadRequest(c, "请求格式不正确") + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, "请求格式不正确："+err.Error()) return } if single { @@ -108,8 +108,8 @@ func (h *AccountHandler) StopCodexTicketRenewal(c *gin.Context) { return } var req codexTicketHarvestRequest - if c.ShouldBindJSON(&req) != nil { - response.BadRequest(c, "请求格式不正确") + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, "请求格式不正确："+err.Error()) return } models, err := h.ticketModels(req.Models) @@ -118,33 +118,90 @@ func (h *AccountHandler) StopCodexTicketRenewal(c *gin.Context) { return } if err = h.codexTicketGateway.StopCodexTicketRenewal(c.Request.Context(), id, models); err != nil { - response.InternalError(c, "停止续期失败") + response.InternalError(c, "停止续期失败："+err.Error()) return } response.Success(c, gin.H{"stopped": true}) } +// codexTicketJobFailure builds a failed item directly: ErrorCode keeps the +// result code and ErrorMessage the verbatim error text (shared job contract). +func codexTicketJobFailure(itemID int64, code, message string, metadata map[string]any, canceled bool) service.AccountJobExecutionResult { + if metadata == nil { + metadata = map[string]any{} + } + if _, exists := metadata["message"]; !exists { + metadata["message"] = message + } + raw, _ := json.Marshal(metadata) + result := service.AccountJobExecutionResult{ItemID: itemID, Status: service.AccountJobItemStatusFailed, Metadata: raw, ErrorCode: code, ErrorMessage: message} + if canceled { + result.Status = service.AccountJobItemStatusCanceled + } + return result +} + +// codexTicketJobMetadata keeps the complete harvest outcome. message and facts +// stay at the top level, where the operation views render them; the upstream +// body, STATE and response headers are stored once, in those facts. +func codexTicketJobMetadata(accountID int64, model string, result service.CodexTicketResult) map[string]any { + stored := result + stored.Facts = nil + if result.Observation != nil { + observation := *result.Observation + observation.UpstreamBody, observation.State, observation.ResponseHeaders, observation.ResponseHeadersOmitted = "", "", nil, nil + stored.Observation = &observation + } + message := result.Message + if detail := result.Detail(); detail != "" && detail != message { + if message == "" { + message = detail + } else { + message += "：" + detail + } + } + metadata := map[string]any{"account_id": accountID, "model_id": model, "code": result.Code, "message": message, "facts": result.Facts, "ticket_result": stored} + if result.HTTPStatus > 0 { + metadata["http_status"] = result.HTTPStatus + } + if result.ResponseModel != "" { + metadata["response_model"] = result.ResponseModel + } + if detail := result.Detail(); detail != "" { + metadata["error"] = detail + } + return metadata +} + func (h *AccountHandler) executeCodexTicketHarvest(ctx context.Context, job *service.AccountJob, raw json.RawMessage, item service.AccountJobItem) service.AccountJobExecutionResult { var req codexTicketHarvestRequest var meta struct { Model string `json:"model_id"` } - if json.Unmarshal(raw, &req) != nil || json.Unmarshal(item.Metadata, &meta) != nil || meta.Model == "" { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return codexTicketJobFailure(item.ID, "payload_invalid", "decode the job payload: "+err.Error(), nil, false) + } + if err := json.Unmarshal(item.Metadata, &meta); err != nil { + return codexTicketJobFailure(item.ID, "payload_invalid", "decode the item metadata: "+err.Error(), nil, false) + } + if meta.Model == "" { + return codexTicketJobFailure(item.ID, "payload_invalid", "the item metadata has no model_id", nil, false) } id, ok := accountJobTarget(item) - if !ok || h.codexTicketGateway == nil { - return accountJobFailed(item.ID, "target_missing") + if !ok { + return codexTicketJobFailure(item.ID, "target_missing", "the job item has no target account", map[string]any{"model_id": meta.Model}, false) + } + if h.codexTicketGateway == nil { + return codexTicketJobFailure(item.ID, "ticket_plugin_unavailable", "the Codex runtime gateway is not configured", map[string]any{"account_id": id, "model_id": meta.Model}, false) } result := h.codexTicketGateway.HarvestCodexTicket(ctx, id, meta.Model, fmt.Sprintf("job:%d:item:%d", job.ID, item.ID), job.ID, req.Force) - metadata := map[string]any{"account_id": id, "model_id": meta.Model, "ticket_result": result} + metadata := codexTicketJobMetadata(id, meta.Model, result) if result.Success { return accountJobSucceeded(item.ID, metadata) } - failure := accountJobFailed(item.ID, result.Code) - failure.Metadata, _ = json.Marshal(metadata) - if ctx.Err() != nil { - failure.Status = service.AccountJobItemStatusCanceled + message := result.Detail() + if message == "" { + message = result.Message } - return failure + return codexTicketJobFailure(item.ID, result.Code, message, metadata, ctx.Err() != nil) } diff --git a/backend/internal/handler/admin/account_codex_ticket_test.go b/backend/internal/handler/admin/account_codex_ticket_test.go index 8aa0dd8465bf..14b0041f74c0 100644 --- a/backend/internal/handler/admin/account_codex_ticket_test.go +++ b/backend/internal/handler/admin/account_codex_ticket_test.go @@ -1,9 +1,12 @@ package admin import ( + "encoding/json" + "strings" "testing" "github.com/Wei-Shaw/sub2api/internal/config" + extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/stretchr/testify/require" ) @@ -18,3 +21,24 @@ func TestAccountResponseCodexTicketsDoesNotUseLegacySettings(t *testing.T) { require.Empty(t, h.accountListResponseFromService(account).CodexTurnTickets) } } + +func TestCodexTicketJobResultKeepsRuntimeOutcome(t *testing.T) { + observation := &extensionv1.CodexRoutingObservation{Stage: "acquire", Code: "routing_upstream", HTTPStatus: 429, UpstreamErrorCode: "rate_limit_exceeded", UpstreamErrorMessage: "Rate limit reached", RequestID: "req_fixture", UpstreamBody: `{"error":{"code":"rate_limit_exceeded"}}`} + result := service.CodexTicketResult{Code: "routing_upstream", HTTPStatus: 429, Message: service.CodexTicketFailure("routing_upstream").Message, Observation: observation, + Facts: []extensionv1.DisplayFact{{Label: map[string]string{"en": "Upstream status"}, Value: "429"}, {Label: map[string]string{"en": "Upstream response body"}, Value: observation.UpstreamBody}}} + failure := codexTicketJobFailure(11, result.Code, result.Detail(), codexTicketJobMetadata(7, "gpt-6-astra", result), false) + require.Equal(t, service.AccountJobItemStatusFailed, failure.Status) + require.Equal(t, "routing_upstream", failure.ErrorCode, "the real routing code is kept") + require.Equal(t, "Rate limit reached (code=rate_limit_exceeded)", failure.ErrorMessage, "the verbatim upstream text is the item error") + require.NoError(t, service.ValidateAccountJobMetadata(failure.Metadata), "the job runtime accepts the complete outcome") + var stored map[string]any + require.NoError(t, json.Unmarshal(failure.Metadata, &stored)) + require.Equal(t, "routing_upstream", stored["code"]) + require.EqualValues(t, 429, stored["http_status"]) + require.Contains(t, stored["message"], result.Message) + require.Contains(t, stored["message"], "Rate limit reached") + require.Len(t, stored["facts"], 2) + require.Contains(t, string(failure.Metadata), "req_fixture") + require.Equal(t, 1, strings.Count(string(failure.Metadata), `rate_limit_exceeded\"}}`), "the upstream body is stored once") + require.NotContains(t, stored["message"], observation.UpstreamBody, "message never falls back to the body") +} diff --git a/backend/internal/handler/admin/account_data.go b/backend/internal/handler/admin/account_data.go index e5f4494d88f5..15cd94bddaaf 100644 --- a/backend/internal/handler/admin/account_data.go +++ b/backend/internal/handler/admin/account_data.go @@ -140,6 +140,8 @@ type DataImportItemResult struct { Code string `json:"code,omitempty"` Message string `json:"message,omitempty"` Error string `json:"error,omitempty"` + // MatchedAccountIDs lists the existing accounts an entry's identity matched. + MatchedAccountIDs []int64 `json:"matched_account_ids,omitempty"` } type DataImportPreviewResult struct { @@ -284,7 +286,7 @@ func (h *AccountHandler) ExportData(c *gin.Context) { Platform: acc.Platform, Type: acc.Type, Credentials: acc.Credentials, - Extra: service.RedactOpenAICodexTicketExtra(acc.Extra), + Extra: acc.Extra, ProxyKey: proxyKey, Concurrency: acc.Concurrency, Priority: acc.Priority, @@ -327,7 +329,7 @@ func (h *AccountHandler) ImportData(c *gin.Context) { func (h *AccountHandler) PreviewImportData(c *gin.Context) { var req DataImportRequest if err := c.ShouldBindJSON(&req); err != nil { - response.BadRequest(c, "Invalid request") + response.BadRequest(c, "Invalid request: "+err.Error()) return } if len(req.Data.Accounts) == 0 { diff --git a/backend/internal/handler/admin/account_data_handler_test.go b/backend/internal/handler/admin/account_data_handler_test.go index b096bd69a227..2af7c151887e 100644 --- a/backend/internal/handler/admin/account_data_handler_test.go +++ b/backend/internal/handler/admin/account_data_handler_test.go @@ -328,7 +328,8 @@ func TestImportDataReusesProxyAndSkipsDefaultGroup(t *testing.T) { require.True(t, adminSvc.createdAccounts[0].SkipDefaultGroupBind) } -func TestExportDataExcludesCodexTicketMaterial(t *testing.T) { +// The backup keeps every stored extra key, Codex ticket material included. +func TestExportDataKeepsCodexTicketMaterial(t *testing.T) { router, adminSvc := setupAccountDataRouter() extra := map[string]any{ "codex_turn_ticket:gpt-6-astra": map[string]any{"state": "private-ticket-blob", "length": 292}, @@ -342,10 +343,9 @@ func TestExportDataExcludesCodexTicketMaterial(t *testing.T) { var resp dataResponse require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) require.Len(t, resp.Data.Accounts, 1) - require.Equal(t, map[string]any{"ordinary": "retained"}, resp.Data.Accounts[0].Extra) + require.Equal(t, "retained", resp.Data.Accounts[0].Extra["ordinary"]) + require.Equal(t, "http://user:legacy-proxy-secret@proxy.example.com:8080", resp.Data.Accounts[0].Extra["codex_harvest_proxy_url"]) + require.Contains(t, resp.Data.Accounts[0].Extra, "codex_turn_ticket:gpt-6-astra") require.Equal(t, "backup-token", resp.Data.Accounts[0].Credentials["access_token"]) - require.NotContains(t, rec.Body.String(), "private-ticket-blob") - require.NotContains(t, rec.Body.String(), "legacy-proxy-secret") - require.Contains(t, extra, "codex_turn_ticket:gpt-6-astra") - require.Contains(t, extra, "codex_harvest_proxy_url") + require.Contains(t, rec.Body.String(), "private-ticket-blob") } diff --git a/backend/internal/handler/admin/account_data_import_decision.go b/backend/internal/handler/admin/account_data_import_decision.go index 449bbae157a0..e446bf3f613a 100644 --- a/backend/internal/handler/admin/account_data_import_decision.go +++ b/backend/internal/handler/admin/account_data_import_decision.go @@ -2,6 +2,9 @@ package admin import ( "context" + "fmt" + "strconv" + "strings" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" @@ -24,6 +27,10 @@ type dataImportDecision struct { AccountID *int64 Code string Message string + // ValidationError is the concrete reason an entry failed validation, and + // MatchedAccountIDs lists every existing account its identity matched. + ValidationError string + MatchedAccountIDs []int64 } func (d dataImportDecision) rejected() bool { return d.Action == dataImportActionReject } @@ -36,20 +43,40 @@ func dataImportMessage(code string) string { return message } +// dataImportDecisionMessage explains a decision with its own facts: the +// validation error of an invalid entry and the IDs an ambiguous identity matched. +func dataImportDecisionMessage(decision dataImportDecision) string { + switch decision.Code { + case dataImportCodePayloadInvalid: + if decision.ValidationError != "" { + return decision.ValidationError + } + case dataImportCodeIdentityConflict: + if len(decision.MatchedAccountIDs) > 0 { + ids := make([]string, 0, len(decision.MatchedAccountIDs)) + for _, id := range decision.MatchedAccountIDs { + ids = append(ids, strconv.FormatInt(id, 10)) + } + return fmt.Sprintf("%s: %s", dataImportMessage(decision.Code), strings.Join(ids, ", ")) + } + } + return dataImportMessage(decision.Code) +} + func rejectDataImportDecision(decision *dataImportDecision, code string) { decision.Action = dataImportActionReject decision.AccountID = nil decision.Code = code - decision.Message = dataImportMessage(code) + decision.Message = dataImportDecisionMessage(*decision) } func (h *AccountHandler) previewDataImport(ctx context.Context, req DataImportRequest) (DataImportPreviewResult, []dataImportDecision, error) { preview := DataImportPreviewResult{Items: make([]DataImportItemResult, 0, len(req.Data.Accounts))} if err := validateDataHeader(req.Data); err != nil { - return preview, nil, infraerrors.BadRequest("ACCOUNT_IMPORT_PAYLOAD_INVALID", "invalid account import payload") + return preview, nil, infraerrors.BadRequest("ACCOUNT_IMPORT_PAYLOAD_INVALID", "invalid account import payload: "+err.Error()) } if err := validateDataImportRequest(req); err != nil { - return preview, nil, infraerrors.BadRequest("ACCOUNT_IMPORT_SETTINGS_INVALID", "invalid account import settings") + return preview, nil, infraerrors.BadRequest("ACCOUNT_IMPORT_SETTINGS_INVALID", "invalid account import settings: "+err.Error()) } var identityIndex *dataIdentityIndex if previewState, prepared := dataImportPreviewStateFromContext(ctx); prepared { @@ -68,13 +95,17 @@ func (h *AccountHandler) previewDataImport(ctx context.Context, req DataImportRe item := req.Data.Accounts[index] enrichCredentialsFromIDToken(&item) decision := dataImportDecision{Account: item} - facts := extensionv1.AccountImportItemFacts{PayloadValid: validateDataAccountV2(item) == nil} + validationErr := validateDataAccountV2(item) + if validationErr != nil { + decision.ValidationError = validationErr.Error() + } + facts := extensionv1.AccountImportItemFacts{PayloadValid: validationErr == nil} keys := dataAccountIdentityKeys(item.Platform, item.Credentials, item.Extra) for _, match := range identityIndex.Find(keys) { - facts.Matches = append(facts.Matches, match.AccountID) - if len(facts.Matches) == 2 { - break - } // Two witnesses already prove ambiguity. + decision.MatchedAccountIDs = append(decision.MatchedAccountIDs, match.AccountID) + if len(facts.Matches) < 2 { + facts.Matches = append(facts.Matches, match.AccountID) + } // Two witnesses already prove ambiguity to the policy. } request.Items[index], decisions[index] = facts, decision } @@ -84,7 +115,8 @@ func (h *AccountHandler) previewDataImport(ctx context.Context, req DataImportRe } for index, plan := range plans { decision := &decisions[index] - decision.Action, decision.Code, decision.Message = plan.Action, plan.Code, dataImportMessage(plan.Code) + decision.Action, decision.Code = plan.Action, plan.Code + decision.Message = dataImportDecisionMessage(*decision) if plan.AccountID > 0 { id := plan.AccountID decision.AccountID = &id @@ -96,6 +128,7 @@ func (h *AccountHandler) previewDataImport(ctx context.Context, req DataImportRe preview.Items = append(preview.Items, DataImportItemResult{ Index: index, Name: decision.Account.Name, Action: decision.Action, AccountID: decision.AccountID, Code: decision.Code, Message: decision.Message, + MatchedAccountIDs: decision.MatchedAccountIDs, }) switch decision.Action { case dataImportActionCreate: diff --git a/backend/internal/handler/admin/account_data_import_job.go b/backend/internal/handler/admin/account_data_import_job.go index dec6ea9d8625..6fa75b9dfe6e 100644 --- a/backend/internal/handler/admin/account_data_import_job.go +++ b/backend/internal/handler/admin/account_data_import_job.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/Wei-Shaw/sub2api/internal/service" ) @@ -46,6 +47,8 @@ type dataImportJobState struct { taxonomy *dataImportTaxonomyResolver skipDefaultGroupBind bool identityIndex *dataIdentityIndex + // proxyErrors keeps what went wrong while importing the file's proxies. + proxyErrors []DataImportError } func dataImportJobStateFromContext(ctx context.Context) (*dataImportJobState, bool) { @@ -68,8 +71,11 @@ func (h *AccountHandler) PrepareAccountJob( return ctx, func() {}, nil } var req DataImportRequest - if json.Unmarshal(raw, &req) != nil || validateDataHeader(req.Data) != nil { - return ctx, nil, errors.New("invalid data import job payload") + if err := json.Unmarshal(raw, &req); err != nil { + return ctx, nil, fmt.Errorf("invalid data import job payload: %w", err) + } + if err := validateDataHeader(req.Data); err != nil { + return ctx, nil, fmt.Errorf("invalid data import job payload: %w", err) } if err := validateDataImportRequest(req); err != nil { return ctx, nil, err @@ -143,6 +149,7 @@ func (h *AccountHandler) PrepareAccountJob( taxonomy: taxonomy, skipDefaultGroupBind: skipDefaultGroupBind, identityIndex: identityIndex, + proxyErrors: dataImportProxyErrors(proxyResult.Errors), } preparedCtx := context.WithValue(ctx, dataImportReferenceCacheContextKey{}, referenceCache) preparedCtx = context.WithValue(preparedCtx, dataImportJobStateContextKey{}, state) @@ -153,6 +160,7 @@ func (h *AccountHandler) PrepareAccountJob( state.identityIndex = nil state.groups = nil state.taxonomy = nil + state.proxyErrors = nil referenceCache.groupIDs = nil referenceCache.proxyIDs = nil } @@ -170,6 +178,10 @@ func (s *dataImportJobState) currentDecision(index int) (dataImportDecision, err matches := s.identityIndex.Find(dataAccountIdentityKeys( decision.Account.Platform, decision.Account.Credentials, decision.Account.Extra, )) + decision.MatchedAccountIDs = nil + for _, match := range matches { + decision.MatchedAccountIDs = append(decision.MatchedAccountIDs, match.AccountID) + } switch len(matches) { case 0: decision.Action = dataImportActionCreate @@ -184,7 +196,7 @@ func (s *dataImportJobState) currentDecision(index int) (dataImportDecision, err rejectDataImportDecision(&decision, dataImportCodeIdentityConflict) return decision, nil } - decision.Message = dataImportMessage(decision.Code) + decision.Message = dataImportDecisionMessage(decision) return decision, nil } @@ -253,8 +265,11 @@ func (s *dataImportJobState) executeOne( err = errors.New("data import action is invalid") } if err != nil || account == nil { + if err == nil { + err = errors.New("data import did not return an account") + } itemResult.Code = dataImportCodeExecutionFailed - itemResult.Message = dataImportMessage(dataImportCodeExecutionFailed) + itemResult.Message = err.Error() itemResult.Error = itemResult.Message result.AccountFailed = 1 result.Items = append(result.Items, itemResult) diff --git a/backend/internal/handler/admin/account_data_v2_handler_test.go b/backend/internal/handler/admin/account_data_v2_handler_test.go index e345049da6ba..f487fc065471 100644 --- a/backend/internal/handler/admin/account_data_v2_handler_test.go +++ b/backend/internal/handler/admin/account_data_v2_handler_test.go @@ -536,3 +536,34 @@ func TestImportDataAllowsPartialSuccessWithPerItemResults(t *testing.T) { require.Equal(t, "failed", result.Items[1].Action) require.Contains(t, result.Items[1].Error, "synthetic create failure") } + +// The preview explains each rejected entry with its own facts: the concrete +// validation error and every existing account an ambiguous identity matched. +func TestPreviewDataImportKeepsRejectReasons(t *testing.T) { + svc := newDataV2AdminService() + svc.accounts = []service.Account{ + {ID: 11, Name: "First", Platform: service.PlatformOpenAI, Type: service.AccountTypeOAuth, + Credentials: map[string]any{"chatgpt_account_id": "workspace-1", "chatgpt_user_id": "user-1"}, Status: service.StatusActive}, + {ID: 12, Name: "Second", Platform: service.PlatformOpenAI, Type: service.AccountTypeOAuth, + Credentials: map[string]any{"chatgpt_account_id": "workspace-1", "chatgpt_user_id": "user-1"}, Status: service.StatusActive}, + } + invalid := testDataAccount("Invalid", "workspace-9", "user-9", "invalid@example.com") + invalid["tags"] = []any{" "} + raw, err := json.Marshal(map[string]any{"data": map[string]any{ + "type": dataType, "version": dataVersion, "proxies": []any{}, + "accounts": []any{testDataAccount("Ambiguous", "workspace-1", "user-1", "dup@example.com"), invalid}, + }}) + require.NoError(t, err) + var req DataImportRequest + require.NoError(t, json.Unmarshal(raw, &req)) + + preview, _, err := NewAccountHandler(svc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil).previewDataImport(context.Background(), req) + require.NoError(t, err) + require.Len(t, preview.Items, 2) + require.Equal(t, "Ambiguous", preview.Items[0].Name) + require.Equal(t, dataImportCodeIdentityConflict, preview.Items[0].Code) + require.Equal(t, []int64{11, 12}, preview.Items[0].MatchedAccountIDs) + require.Equal(t, "account identity matches multiple existing accounts: 11, 12", preview.Items[0].Message) + require.Equal(t, dataImportCodePayloadInvalid, preview.Items[1].Code) + require.Equal(t, "tag: name must contain between 1 and 100 characters", preview.Items[1].Message) +} diff --git a/backend/internal/handler/admin/account_job_execution.go b/backend/internal/handler/admin/account_job_execution.go index 7a18654840a4..d5593acca897 100644 --- a/backend/internal/handler/admin/account_job_execution.go +++ b/backend/internal/handler/admin/account_job_execution.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "strconv" "strings" @@ -44,7 +45,7 @@ func (h *AccountHandler) executeAccountJobItem(ctx context.Context, kind string, return accountJobFailed(item.ID, "target_missing") } if err := h.adminService.DeleteAccount(ctx, id); err != nil { - return accountJobFailed(item.ID, "delete_failed") + return accountJobFailedWithError(item.ID, "delete_failed", err) } return accountJobSucceeded(item.ID, map[string]any{"account_id": id}) @@ -55,7 +56,7 @@ func (h *AccountHandler) executeAccountJobItem(ctx context.Context, kind string, } account, err := h.adminService.ClearAccountError(ctx, id) if err != nil { - return accountJobFailed(item.ID, "clear_error_failed") + return accountJobFailedWithError(item.ID, "clear_error_failed", err) } if h.tokenCacheInvalidator != nil && account != nil && account.IsOAuth() { _ = h.tokenCacheInvalidator.InvalidateToken(ctx, account) @@ -69,23 +70,26 @@ func (h *AccountHandler) executeAccountJobItem(ctx context.Context, kind string, } account, err := h.adminService.GetAccount(ctx, id) if err != nil { - return accountJobFailed(item.ID, "account_not_found") + return accountJobFailedWithError(item.ID, "account_not_found", err) } _, warning, err := h.refreshSingleAccount(ctx, account) if err != nil { - return accountJobFailed(item.ID, "refresh_failed") + return accountJobFailedWithError(item.ID, "refresh_failed", err) } return accountJobSucceeded(item.ID, map[string]any{"account_id": id, "warning": warning}) case service.AccountJobKindBatchCreate: var payload batchCreateJobPayload - if json.Unmarshal(raw, &payload) != nil || item.Ordinal <= 0 || item.Ordinal > len(payload.Accounts) { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &payload); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) + } + if item.Ordinal <= 0 || item.Ordinal > len(payload.Accounts) { + return accountJobFailedWithError(item.ID, "payload_invalid", accountJobOrdinalError(item.Ordinal, len(payload.Accounts))) } request := payload.Accounts[item.Ordinal-1] created, err := h.createAccountJobAccount(ctx, request) if err != nil { - return accountJobFailed(item.ID, "create_failed") + return withAccountJobMetadata(accountJobFailedWithError(item.ID, "create_failed", err), map[string]any{"name": request.Name}) } h.scheduleOpenAIResponsesProbe(created) h.scheduleGrokImportProbe(created) @@ -94,11 +98,14 @@ func (h *AccountHandler) executeAccountJobItem(ctx context.Context, kind string, case service.AccountJobKindBatchUpdateCredentials: var req BatchUpdateCredentialsRequest id, ok := accountJobTarget(item) - if json.Unmarshal(raw, &req) != nil || !ok { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) + } + if !ok { + return accountJobFailedWithError(item.ID, "payload_invalid", errAccountJobTargetMissing) } if _, err := h.adminService.GetAccount(ctx, id); err != nil { - return accountJobFailed(item.ID, "account_not_found") + return accountJobFailedWithError(item.ID, "account_not_found", err) } updateCredentials := func(mutationCtx context.Context) (*service.Account, error) { current, getErr := h.adminService.GetAccount(mutationCtx, id) @@ -110,7 +117,7 @@ func (h *AccountHandler) executeAccountJobItem(ctx context.Context, kind string, return h.adminService.UpdateAccount(mutationCtx, id, &service.UpdateAccountInput{Credentials: credentials}) } if _, err := updateCredentials(ctx); err != nil { - return accountJobFailed(item.ID, "credentials_update_failed") + return accountJobFailedWithError(item.ID, "credentials_update_failed", err) } return accountJobSucceeded(item.ID, map[string]any{"account_id": id}) @@ -142,6 +149,12 @@ func accountJobTarget(item service.AccountJobItem) (int64, bool) { return returnValue, returnValue > 0 } +var errAccountJobTargetMissing = errors.New("account job item has no target account") + +func accountJobOrdinalError(ordinal, count int) error { + return fmt.Errorf("account job item ordinal %d is outside the %d submitted entries", ordinal, count) +} + func cloneAccountJobMap(input map[string]any) map[string]any { result := make(map[string]any, len(input)+1) for key, value := range input { @@ -178,8 +191,8 @@ func (h *AccountHandler) createAccountJobAccount(ctx context.Context, item Creat func (h *AccountHandler) executeBulkUpdateJob(ctx context.Context, raw json.RawMessage, item service.AccountJobItem) service.AccountJobExecutionResult { var req BulkUpdateAccountsRequest - if json.Unmarshal(raw, &req) != nil { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) } id, ok := accountJobTarget(item) if !ok { @@ -191,9 +204,11 @@ func (h *AccountHandler) executeBulkUpdateJob(ctx context.Context, raw json.RawM req.Filters = nil succeeded := 0 failed := 0 + var failures []error for _, id := range req.AccountIDs { if _, getErr := h.adminService.GetAccount(ctx, id); getErr != nil { failed++ + failures = append(failures, getErr) continue } apply := func(mutationCtx context.Context) (*service.Account, error) { @@ -208,12 +223,13 @@ func (h *AccountHandler) executeBulkUpdateJob(ctx context.Context, raw json.RawM } if _, err := apply(ctx); err != nil { failed++ + failures = append(failures, err) continue } succeeded++ } if failed > 0 { - return accountJobFailed(item.ID, "bulk_update_failed") + return accountJobFailedWithError(item.ID, "bulk_update_failed", errors.Join(failures...)) } metadata := map[string]any{"success": succeeded, "failed": failed} if id, ok := accountJobTarget(item); ok { @@ -283,8 +299,8 @@ func (h *AccountHandler) resolveAccountJobTargetIDs( func (h *AccountHandler) executeBulkTaxonomyJob(ctx context.Context, raw json.RawMessage, item service.AccountJobItem) service.AccountJobExecutionResult { var req bulkAccountTaxonomyRequest - if json.Unmarshal(raw, &req) != nil { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) } id, ok := accountJobTarget(item) if !ok { @@ -294,12 +310,12 @@ func (h *AccountHandler) executeBulkTaxonomyJob(ctx context.Context, raw json.Ra console, err := h.accountTaxonomyMutationService() if err != nil { - return accountJobFailed(item.ID, "taxonomy_unavailable") + return accountJobFailedWithError(item.ID, "taxonomy_unavailable", err) } updated := 0 for _, id := range ids { if _, getErr := h.adminService.GetAccount(ctx, id); getErr != nil { - return accountJobFailed(item.ID, "taxonomy_update_failed") + return accountJobFailedWithError(item.ID, "taxonomy_update_failed", getErr) } apply := func(mutationCtx context.Context) (*service.Account, error) { result, updateErr := console.BulkUpdateAccountTaxonomy(mutationCtx, service.BulkAccountTaxonomyInput{ @@ -315,7 +331,7 @@ func (h *AccountHandler) executeBulkTaxonomyJob(ctx context.Context, raw json.Ra return h.adminService.GetAccount(mutationCtx, id) } if _, err := apply(ctx); err != nil { - return accountJobFailed(item.ID, "taxonomy_update_failed") + return accountJobFailedWithError(item.ID, "taxonomy_update_failed", err) } updated++ } @@ -324,20 +340,20 @@ func (h *AccountHandler) executeBulkTaxonomyJob(ctx context.Context, raw json.Ra func (h *AccountHandler) executeRefreshTierJob(ctx context.Context, raw json.RawMessage, item service.AccountJobItem) service.AccountJobExecutionResult { var req BatchRefreshTierRequest - if json.Unmarshal(raw, &req) != nil { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) } var accounts []*service.Account if id, ok := accountJobTarget(item); ok { account, err := h.adminService.GetAccount(ctx, id) if err != nil { - return accountJobFailed(item.ID, "account_not_found") + return accountJobFailedWithError(item.ID, "account_not_found", err) } accounts = []*service.Account{account} } else { all, _, err := h.adminService.ListAccounts(ctx, 1, 10000, service.PlatformGemini, service.AccountTypeOAuth, "", "", 0, "", "name", "asc") if err != nil { - return accountJobFailed(item.ID, "account_list_failed") + return accountJobFailedWithError(item.ID, "account_list_failed", err) } for index := range all { accounts = append(accounts, &all[index]) @@ -350,10 +366,10 @@ func (h *AccountHandler) executeRefreshTierJob(ctx context.Context, raw json.Raw } _, extra, credentials, err := h.geminiOAuthService.RefreshAccountGoogleOneTier(ctx, account) if err != nil { - return accountJobFailed(item.ID, "refresh_tier_failed") + return accountJobFailedWithError(item.ID, "refresh_tier_failed", fmt.Errorf("account %d: %w", account.ID, err)) } if _, err = h.adminService.UpdateAccount(ctx, account.ID, &service.UpdateAccountInput{Credentials: credentials, Extra: extra}); err != nil { - return accountJobFailed(item.ID, "refresh_tier_update_failed") + return accountJobFailedWithError(item.ID, "refresh_tier_update_failed", fmt.Errorf("account %d: %w", account.ID, err)) } updated++ } @@ -365,36 +381,53 @@ func (h *AccountHandler) executeDataImportJob(ctx context.Context, raw json.RawM preparedState, prepared := dataImportJobStateFromContext(ctx) if prepared { req = preparedState.request - } else if json.Unmarshal(raw, &req) != nil || validateDataHeader(req.Data) != nil { - return accountJobFailed(item.ID, "payload_invalid") + } else { + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) + } + if err := validateDataHeader(req.Data); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) + } } if item.Ordinal <= 0 || item.Ordinal > len(req.Data.Accounts) { - return accountJobFailed(item.ID, "payload_invalid") + return accountJobFailedWithError(item.ID, "payload_invalid", accountJobOrdinalError(item.Ordinal, len(req.Data.Accounts))) } originalIndex := item.Ordinal - 1 + // Every result names its source entry. Proxy import problems belong to the + // whole file, so the first entry carries them. + metadata := map[string]any{"source_index": originalIndex, "name": req.Data.Accounts[originalIndex].Name} + if prepared && originalIndex == 0 && len(preparedState.proxyErrors) > 0 { + metadata["proxy_errors"] = preparedState.proxyErrors + } + failed := func(code string, err error) service.AccountJobExecutionResult { + return withAccountJobMetadata(accountJobFailedWithError(item.ID, code, err), metadata) + } var decisions []dataImportDecision if prepared { decisions = preparedState.decisions } else { _, currentDecisions, decisionErr := h.previewDataImport(ctx, req) if decisionErr != nil { - return accountJobFailed(item.ID, dataImportCodeExecutionFailed) + return failed(dataImportCodeExecutionFailed, decisionErr) } decisions = currentDecisions } if originalIndex >= len(decisions) { - return accountJobFailed(item.ID, dataImportCodeExecutionFailed) + return failed(dataImportCodeExecutionFailed, fmt.Errorf("data import decision %d is unavailable (%d decisions)", originalIndex, len(decisions))) } decision := decisions[originalIndex] if prepared { var decisionErr error decision, decisionErr = preparedState.currentDecision(originalIndex) if decisionErr != nil { - return accountJobFailed(item.ID, dataImportCodeExecutionFailed) + return failed(dataImportCodeExecutionFailed, decisionErr) } } + if len(decision.MatchedAccountIDs) > 0 { + metadata["matched_account_ids"] = decision.MatchedAccountIDs + } if decision.rejected() { - return accountJobFailed(item.ID, decision.Code) + return failed(decision.Code, errors.New(decision.Message)) } if !prepared { req.Data.Accounts = []DataAccount{decision.Account} @@ -410,12 +443,12 @@ func (h *AccountHandler) executeDataImportJob(ctx context.Context, raw json.RawM } if importErr != nil || result.AccountFailed > 0 || len(result.Items) != 1 || result.Items[0].AccountID == nil { if importErr == nil { - importErr = errors.New("data import item failed") + importErr = dataImportItemResultError(result) } return nil, result, importErr } if len(result.Items[0].Warnings) > 0 { - return imported, result, errors.New("data import item completed with an incomplete mutation") + return imported, result, fmt.Errorf("data import item completed with an incomplete mutation: %s", strings.Join(result.Items[0].Warnings, "; ")) } if imported != nil { return imported, result, nil @@ -430,44 +463,120 @@ func (h *AccountHandler) executeDataImportJob(ctx context.Context, raw json.RawM preparedState.recordCommittedAccount(importedAccount) h.scheduleGrokImportProbe(importedAccount) } - if err != nil || result.AccountFailed > 0 { - return accountJobFailed(item.ID, dataImportCodeExecutionFailed) + if !prepared && originalIndex == 0 { + if proxyErrors := dataImportProxyErrors(result.Errors); len(proxyErrors) > 0 { + metadata["proxy_errors"] = proxyErrors + } } - metadata := map[string]any{"source_index": originalIndex} if len(result.Items) == 1 { metadata["action"] = result.Items[0].Action if result.Items[0].AccountID != nil { metadata["account_id"] = *result.Items[0].AccountID } + if len(result.Items[0].Warnings) > 0 { + metadata["warnings"] = result.Items[0].Warnings + } + } + if err != nil || result.AccountFailed > 0 { + if err == nil { + err = dataImportItemResultError(result) + } + return failed(dataImportCodeExecutionFailed, err) } return accountJobSucceeded(item.ID, metadata) } +// dataImportItemResultError returns the recorded reason of a failed import +// result instead of a generic sentence. +func dataImportItemResultError(result DataImportResult) error { + if len(result.Items) == 1 && strings.TrimSpace(result.Items[0].Error) != "" { + return errors.New(result.Items[0].Error) + } + messages := make([]string, 0, len(result.Errors)) + for _, item := range result.Errors { + if item.Kind == "account" && strings.TrimSpace(item.Message) != "" { + messages = append(messages, item.Message) + } + } + if len(messages) > 0 { + return errors.New(strings.Join(messages, "; ")) + } + return fmt.Errorf("data import returned %d item results without an account", len(result.Items)) +} + +func dataImportProxyErrors(errs []DataImportError) []DataImportError { + proxies := make([]DataImportError, 0, len(errs)) + for _, item := range errs { + if item.Kind == "proxy" { + proxies = append(proxies, item) + } + } + return proxies +} + func (h *AccountHandler) executeCodexImportJob(ctx context.Context, raw json.RawMessage, item service.AccountJobItem) service.AccountJobExecutionResult { var req CodexSessionImportRequest - if json.Unmarshal(raw, &req) != nil { - return accountJobFailed(item.ID, "payload_invalid") + if err := json.Unmarshal(raw, &req); err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) } if err := service.ValidateOpenAILongContextBillingExtra(service.PlatformOpenAI, req.Extra); err != nil { - return accountJobFailed(item.ID, "payload_invalid") + return accountJobFailedWithError(item.ID, "payload_invalid", err) } if err := service.ValidateOpenAIReasoningPolicyExtra(req.Extra); err != nil { - return accountJobFailed(item.ID, "payload_invalid") + return accountJobFailedWithError(item.ID, "payload_invalid", err) } entries, err := parseCodexSessionImportEntries(req) - if err != nil || item.Ordinal <= 0 || item.Ordinal > len(entries) { - return accountJobFailed(item.ID, "payload_invalid") + if err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) } - result, err := h.importCodexSessions(ctx, req, []codexImportEntry{entries[item.Ordinal-1]}) - if err != nil || result.Failed > 0 { - return accountJobFailed(item.ID, "import_failed") + if item.Ordinal <= 0 || item.Ordinal > len(entries) { + return accountJobFailedWithError(item.ID, "payload_invalid", accountJobOrdinalError(item.Ordinal, len(entries))) } - metadata := map[string]any{"source_index": entries[item.Ordinal-1].Index} + entry := entries[item.Ordinal-1] + result, err := h.importCodexSessions(ctx, req, []codexImportEntry{entry}) + metadata := map[string]any{"source_index": entry.Index} if len(result.Items) == 1 { - metadata["action"] = result.Items[0].Action - if result.Items[0].AccountID > 0 { - metadata["account_id"] = result.Items[0].AccountID + imported := result.Items[0] + metadata["action"] = imported.Action + if imported.AccountID > 0 { + metadata["account_id"] = imported.AccountID + } + if imported.Name != "" { + metadata["name"] = imported.Name + } + if imported.Message != "" && imported.Action != "failed" { + metadata["message"] = imported.Message } } + if len(result.Warnings) > 0 { + warnings := make([]string, 0, len(result.Warnings)) + for _, warning := range result.Warnings { + warnings = append(warnings, warning.Message) + } + metadata["warnings"] = warnings + } + if err == nil && result.Failed > 0 { + err = codexImportResultError(result) + } + if err != nil { + return withAccountJobMetadata(accountJobFailedWithError(item.ID, "import_failed", err), metadata) + } return accountJobSucceeded(item.ID, metadata) } + +// codexImportResultError returns the recorded reason of a failed Codex entry. +func codexImportResultError(result CodexSessionImportResult) error { + if len(result.Items) == 1 && strings.TrimSpace(result.Items[0].Message) != "" { + return errors.New(result.Items[0].Message) + } + messages := make([]string, 0, len(result.Errors)) + for _, item := range result.Errors { + if strings.TrimSpace(item.Message) != "" { + messages = append(messages, item.Message) + } + } + if len(messages) > 0 { + return errors.New(strings.Join(messages, "; ")) + } + return fmt.Errorf("codex import reported %d failed entries", result.Failed) +} diff --git a/backend/internal/handler/admin/account_job_handler.go b/backend/internal/handler/admin/account_job_handler.go index 72e916832615..707bc037b6c5 100644 --- a/backend/internal/handler/admin/account_job_handler.go +++ b/backend/internal/handler/admin/account_job_handler.go @@ -272,8 +272,29 @@ func accountJobSucceeded(itemID int64, metadata any) service.AccountJobExecution return service.AccountJobExecutionResult{ItemID: itemID, Status: service.AccountJobItemStatusSucceeded, Metadata: raw} } +// accountJobFailed records a failure that has no underlying error text; the +// catalog sentence of code describes it. func accountJobFailed(itemID int64, code string) service.AccountJobExecutionResult { - code, message := service.NormalizeAccountJobFailure(code) + return accountJobFailedWithError(itemID, code, nil) +} + +// accountJobFailedWithError keeps code and the verbatim text of err for a +// failed item. Without an err the catalog sentence of code is used. +func accountJobFailedWithError(itemID int64, code string, err error) service.AccountJobExecutionResult { + message := "" + if err != nil { + message = err.Error() + } + code, message = service.AccountJobFailure(code, message) return service.AccountJobExecutionResult{ItemID: itemID, Status: service.AccountJobItemStatusFailed, Metadata: json.RawMessage(`{}`), ErrorCode: code, ErrorMessage: message} } + +// withAccountJobMetadata attaches result metadata, for example the source +// index or name of a failed item, without changing its status or error. +func withAccountJobMetadata(result service.AccountJobExecutionResult, metadata map[string]any) service.AccountJobExecutionResult { + if len(metadata) > 0 { + result.Metadata, _ = json.Marshal(metadata) + } + return result +} diff --git a/backend/internal/handler/admin/account_job_legacy_codex.go b/backend/internal/handler/admin/account_job_legacy_codex.go index b9351bc5ff6c..cc6676aeabd1 100644 --- a/backend/internal/handler/admin/account_job_legacy_codex.go +++ b/backend/internal/handler/admin/account_job_legacy_codex.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/Wei-Shaw/sub2api/internal/service" ) @@ -15,14 +16,20 @@ func (h *AccountHandler) executeLegacyCodexAccountJob(ctx context.Context, job * Metadata: json.RawMessage(`{}`), ErrorCode: "legacy_operation_unsupported", ErrorMessage: "The stored operation has no native executor"} } if err != nil { - return accountJobFailed(item.ID, "payload_invalid") + return accountJobFailedWithError(item.ID, "payload_invalid", err) } - if h.codexTicketGateway == nil || h.codexTicketGateway.ValidateLegacyCodexJobSource(ctx, intent.PluginID) != nil { - return accountJobFailed(item.ID, "legacy_source_unavailable") + if h.codexTicketGateway == nil { + return accountJobFailedWithError(item.ID, "legacy_source_unavailable", errors.New("codex ticket gateway is unavailable")) + } + if err := h.codexTicketGateway.ValidateLegacyCodexJobSource(ctx, intent.PluginID); err != nil { + return accountJobFailedWithError(item.ID, "legacy_source_unavailable", err) } models, err := h.ticketModels([]string{intent.Model}) - if err != nil || len(models) != 1 { - return accountJobFailed(item.ID, "payload_invalid") + if err != nil { + return accountJobFailedWithError(item.ID, "payload_invalid", err) + } + if len(models) != 1 { + return accountJobFailedWithError(item.ID, "payload_invalid", fmt.Errorf("stored model %q resolved to %d models", intent.Model, len(models))) } payload, _ := json.Marshal(codexTicketHarvestRequest{AccountIDs: []int64{intent.AccountID}, Models: models, Force: intent.Force}) // Only the original frozen target and single saved model reach the native diff --git a/backend/internal/handler/admin/codex_error_diagnostics.go b/backend/internal/handler/admin/codex_error_diagnostics.go index 7094bae67ff5..c4d4e1920887 100644 --- a/backend/internal/handler/admin/codex_error_diagnostics.go +++ b/backend/internal/handler/admin/codex_error_diagnostics.go @@ -9,12 +9,12 @@ import ( "github.com/gin-gonic/gin" ) -// ErrorDiagnostics is installed behind the existing administrator middleware -// and never exposes the underlying log document. +// CodexErrorDiagnostics is installed behind the existing administrator +// middleware and returns every continuation diagnostic stored on the error. func (h *AccountHandler) CodexErrorDiagnostics(ops *OpsHandler) gin.HandlerFunc { return func(c *gin.Context) { c.Header("Cache-Control", "private, no-store") - if h == nil || h.adminService == nil || ops == nil || ops.opsService == nil { + if h == nil || ops == nil || ops.opsService == nil { response.Error(c, http.StatusServiceUnavailable, "Diagnostics unavailable") return } @@ -28,6 +28,6 @@ func (h *AccountHandler) CodexErrorDiagnostics(ops *OpsHandler) gin.HandlerFunc response.ErrorFrom(c, err) return } - response.Success(c, service.ProjectCodexErrorDiagnostics(c.Request.Context(), detail, h.adminService.GetAccount)) + response.Success(c, service.ProjectCodexErrorDiagnostics(detail)) } } diff --git a/backend/internal/handler/admin/plugin_handler.go b/backend/internal/handler/admin/plugin_handler.go index b622b4fc6d48..b3f928c177f5 100644 --- a/backend/internal/handler/admin/plugin_handler.go +++ b/backend/internal/handler/admin/plugin_handler.go @@ -26,12 +26,32 @@ const pluginUISessionTTL = 30 * time.Minute // PluginHandler 提供插件安装、生命周期、配置和隔离 UI 资源接口。 type PluginHandler struct { manager *service.PluginManager + retired *service.NativeFeatureBootstrap } func NewPluginHandler(manager *service.PluginManager) *PluginHandler { return &PluginHandler{manager: manager} } +// SetRetiredPlugins attaches the read-only view of retired first-party +// installations, which the managed plugin list never contains. +func (h *PluginHandler) SetRetiredPlugins(retired *service.NativeFeatureBootstrap) { + h.retired = retired +} + +// ListRetired returns the retired first-party installations, their retirement +// receipt and decrypted saved configuration. It is read-only. +// GET /api/v1/admin/plugins/retired +func (h *PluginHandler) ListRetired(c *gin.Context) { + c.Header("Cache-Control", "private, no-store") + view, err := h.retired.RetiredPlugins(c.Request.Context()) + if err != nil { + response.ErrorFrom(c, err) + return + } + response.Success(c, view) +} + func (h *PluginHandler) List(c *gin.Context) { plugins, err := h.manager.List(c.Request.Context()) if err != nil { diff --git a/backend/internal/handler/admin/setting_codex_proxy_selection_test.go b/backend/internal/handler/admin/setting_codex_proxy_selection_test.go index 327bc21646b6..c370f6168626 100644 --- a/backend/internal/handler/admin/setting_codex_proxy_selection_test.go +++ b/backend/internal/handler/admin/setting_codex_proxy_selection_test.go @@ -29,9 +29,11 @@ func TestCodexProxyParseIsPureAndSelectionErrorsAreShared(t *testing.T) { parsed := call(h.ParseCodexTicketProxy, map[string]string{"proxy_url": input, "protocol": "https"}) require.Equal(t, http.StatusOK, parsed.Code) require.Equal(t, "no-store", parsed.Header().Get("Cache-Control")) - for _, value := range []string{"fixture-user", "fixture-password", "other-password", "http://"} { + for _, value := range []string{"fixture-password", "other-password", "http://"} { require.NotContains(t, parsed.Body.String(), value) } + // Candidates show the username so records differing only by it stay distinguishable. + require.Contains(t, parsed.Body.String(), `"username":"fixture-user"`) var envelope struct { Data proxytransport.ParseResult `json:"data"` } diff --git a/backend/internal/handler/admin/setting_codex_runtime.go b/backend/internal/handler/admin/setting_codex_runtime.go index 5aeb40b97f3f..ebba22c3a4a8 100644 --- a/backend/internal/handler/admin/setting_codex_runtime.go +++ b/backend/internal/handler/admin/setting_codex_runtime.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "strconv" @@ -34,7 +35,7 @@ func writeNativeCodexConfiguration(c *gin.Context, read func(context.Context) (j } raw, metadata, err := read(c.Request.Context()) if err != nil { - response.Error(c, 503, "Codex runtime is unavailable") + response.Error(c, 503, "Codex runtime is unavailable: "+err.Error()) return } c.Header(nativeCodexHostingModeHeader, "native") @@ -47,8 +48,12 @@ func writeNativeCodexConfiguration(c *gin.Context, read func(context.Context) (j func (h *SettingHandler) UpdateNativeCodexConfiguration(c *gin.Context) { c.Header("Cache-Control", "no-store") var raw json.RawMessage - if c.ShouldBindJSON(&raw) != nil || len(raw) > 1<<20 { - response.BadRequest(c, "Invalid Codex routing configuration") + if err := c.ShouldBindJSON(&raw); err != nil { + response.BadRequest(c, "Invalid Codex routing configuration: "+err.Error()) + return + } + if len(raw) > 1<<20 { + response.BadRequest(c, fmt.Sprintf("Invalid Codex routing configuration: %d bytes exceed the 1 MiB limit", len(raw))) return } normalized, err := service.NormalizeNativeCodexConfig(c.Request.Context(), raw) @@ -56,7 +61,7 @@ func (h *SettingHandler) UpdateNativeCodexConfiguration(c *gin.Context) { if writeCodexProxySelectionError(c, err) { return } - response.BadRequest(c, "Invalid Codex routing configuration") + response.BadRequest(c, "Invalid Codex routing configuration: "+err.Error()) return } if h.codexTicketGateway == nil { @@ -65,9 +70,9 @@ func (h *SettingHandler) UpdateNativeCodexConfiguration(c *gin.Context) { } if err := h.codexTicketGateway.UpdateNativeCodexConfiguration(c.Request.Context(), normalized, h.nativeCodexConfigEncryptor); err != nil { if errors.Is(err, service.ErrNativeCodexRuntimeChanged) { - response.Error(c, 409, "Codex runtime configuration changed; reload and retry") + response.Error(c, 409, "Codex runtime configuration changed; reload and retry: "+err.Error()) } else { - response.Error(c, 503, "Codex runtime configuration could not be applied") + response.Error(c, 503, "Codex runtime configuration could not be applied: "+err.Error()) } return } diff --git a/backend/internal/handler/admin/setting_codex_runtime_receipt_test.go b/backend/internal/handler/admin/setting_codex_runtime_receipt_test.go index 218836067a0a..f46143356191 100644 --- a/backend/internal/handler/admin/setting_codex_runtime_receipt_test.go +++ b/backend/internal/handler/admin/setting_codex_runtime_receipt_test.go @@ -73,11 +73,17 @@ func TestNativeCodexConfigurationHTTPReceiptUnavailableIsOpaque(t *testing.T) { } require.Equal(t, http.StatusServiceUnavailable, recorder.Code) require.Equal(t, "no-store", recorder.Header().Get("Cache-Control")) - require.JSONEq(t, `{"code":503,"message":"Codex runtime is unavailable"}`, recorder.Body.String()) + // The administrator sees the original cause; a partial snapshot and its + // receipt headers are still never returned. + cause := "native Codex runtime unavailable" + if name != "missing_runtime" { + cause = "synthetic-private-database-detail" + } + require.JSONEq(t, `{"code":503,"message":"Codex runtime is unavailable: `+cause+`"}`, recorder.Body.String()) for _, header := range [...]string{nativeCodexHostingModeHeader, nativeCodexConfigVersionHeader, nativeCodexConfigSHA256Header, nativeCodexRuntimeGenerationHeader} { require.Empty(t, recorder.Header().Values(header), "unavailable reads must not expose a receipt") } - require.NotContains(t, recorder.Body.String(), "synthetic-private") + require.NotContains(t, recorder.Body.String(), "synthetic-private-config") }) } } diff --git a/backend/internal/handler/admin/setting_codex_ticket_proxy.go b/backend/internal/handler/admin/setting_codex_ticket_proxy.go index 4e6ab83910ab..64c65cccc2e3 100644 --- a/backend/internal/handler/admin/setting_codex_ticket_proxy.go +++ b/backend/internal/handler/admin/setting_codex_ticket_proxy.go @@ -21,8 +21,8 @@ type codexTicketProxyInput struct { func (h *SettingHandler) ParseCodexTicketProxy(c *gin.Context) { c.Header("Cache-Control", "no-store") var req codexTicketProxyInput - if c.ShouldBindJSON(&req) != nil { - response.BadRequest(c, "请求格式不正确") + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, "请求格式不正确："+err.Error()) return } response.Success(c, proxytransport.Parse(req.ProxyURL, req.Protocol)) @@ -44,8 +44,8 @@ func (h *SettingHandler) SetCodexTicketGateway(gateway *service.OpenAIGatewaySer func (h *SettingHandler) TestCodexTicketProxy(c *gin.Context) { c.Header("Cache-Control", "no-store") var req codexTicketProxyInput - if c.ShouldBindJSON(&req) != nil { - response.BadRequest(c, "请求格式不正确") + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, "请求格式不正确："+err.Error()) return } normal, err := proxytransport.Resolve(req.ProxyURL, req.Protocol, req.ProxySelectionID) @@ -61,7 +61,7 @@ func (h *SettingHandler) TestCodexTicketProxy(c *gin.Context) { } result, err := h.codexTicketGateway.TestCodexTicketProxyWithProtocol(c.Request.Context(), normal, "") if err != nil { - response.InternalError(c, "无法完成代理测试") + response.InternalError(c, "无法完成代理测试："+err.Error()) return } response.Success(c, result) diff --git a/backend/internal/handler/admin/setting_handler.go b/backend/internal/handler/admin/setting_handler.go index 9885b892cee2..911d0e8fafbb 100644 --- a/backend/internal/handler/admin/setting_handler.go +++ b/backend/internal/handler/admin/setting_handler.go @@ -123,8 +123,11 @@ func (h *SettingHandler) GetImageToolsSettings(c *gin.Context) { func (h *SettingHandler) UpdateImageToolsSettings(c *gin.Context) { var req extensionv1.ImageToolsConfig raw, err := c.GetRawData() - if err != nil || service.DecodeSwitchSettings(raw, &req, "studio_enabled") != nil { - response.BadRequest(c, "Invalid image tools settings") + if err == nil { + err = service.DecodeSwitchSettings(raw, &req, "studio_enabled") + } + if err != nil { + response.BadRequest(c, "Invalid image tools settings: "+err.Error()) return } if err := h.settingService.UpdateImageToolsConfig(c.Request.Context(), req); err != nil { @@ -147,8 +150,11 @@ func (h *SettingHandler) GetObservabilitySettings(c *gin.Context) { func (h *SettingHandler) UpdateObservabilitySettings(c *gin.Context) { req := extensionv1.AdminObservabilityConfig{TelemetryEnabled: true, ThemeEnabled: true} raw, err := c.GetRawData() - if err != nil || service.DecodeSwitchSettings(raw, &req, "telemetry_enabled", "theme_enabled") != nil { - response.BadRequest(c, "Invalid observability settings") + if err == nil { + err = service.DecodeSwitchSettings(raw, &req, "telemetry_enabled", "theme_enabled") + } + if err != nil { + response.BadRequest(c, "Invalid observability settings: "+err.Error()) return } if err := h.settingService.UpdateAdminObservabilityConfig(c.Request.Context(), req); err != nil { @@ -479,7 +485,14 @@ func (h *SettingHandler) GetSettings(c *gin.Context) { payload.DefaultPlatformQuotas = platformQuotas } - response.Success(c, systemSettingsResponseData(payload, authSourceDefaults)) + data := systemSettingsResponseData(payload, authSourceDefaults) + // Deprecated compatibility values are preserved on save but have no public + // write field or runtime effect; they are echoed read-only. + data["deprecated_settings"] = map[string]any{ + service.SettingKeyOpenAIAPIKeyAlphaSearchResponsesBridgeEnabled: settings.OpenAIAPIKeyAlphaSearchResponsesBridgeEnabled, + service.SettingKeyOpenAIAPIKeyPromptCacheKeyNormalizationEnabled: settings.OpenAIAPIKeyPromptCacheKeyNormalizationEnabled, + } + response.Success(c, data) } // openaiFastPolicySettingsToDTO converts service -> dto for OpenAI fast policy. diff --git a/backend/internal/handler/dto/account_mapper_redact_test.go b/backend/internal/handler/dto/account_mapper_redact_test.go index 20c6dd37fdb5..39eb0f3ec361 100644 --- a/backend/internal/handler/dto/account_mapper_redact_test.go +++ b/backend/internal/handler/dto/account_mapper_redact_test.go @@ -95,7 +95,9 @@ func TestAccountFromServiceShallow_RedactsOllamaCloudManagedExtra(t *testing.T) require.Contains(t, src.Extra, service.OllamaCloudUsageSessionExtraKey) } -func TestAccountFromServiceShallow_RedactsCodexTurnTicketState(t *testing.T) { +// Administrators see the stored Codex ticket keys, including legacy ticket +// state and the retired account-level harvest proxy URL. +func TestAccountFromServiceShallow_KeepsCodexTurnTicketState(t *testing.T) { blob := "gAAAAA" + strings.Repeat("B", 286) src := &service.Account{ ID: 41, Platform: service.PlatformOpenAI, Type: service.AccountTypeOAuth, @@ -112,13 +114,11 @@ func TestAccountFromServiceShallow_RedactsCodexTurnTicketState(t *testing.T) { }, } got := AccountFromServiceShallow(src) - require.NotContains(t, got.Extra, "codex_turn_ticket:gpt-6-astra") + require.Contains(t, got.Extra, "codex_turn_ticket:gpt-6-astra") + require.Equal(t, "http://user:legacy-proxy-secret@proxy.example.com:8080", got.Extra["codex_harvest_proxy_url"]) raw, err := json.Marshal(got) require.NoError(t, err) - require.NotContains(t, string(raw), blob) - require.NotContains(t, string(raw), "legacy-proxy-secret") - require.NotContains(t, got.Extra, "codex_harvest_proxy_url") - require.Contains(t, src.Extra, "codex_harvest_proxy_url") + require.Contains(t, string(raw), blob) } func TestAccountFromServiceShallow_NilCredentialsOmitsStatus(t *testing.T) { diff --git a/backend/internal/handler/dto/group_mapper_profit_visibility_test.go b/backend/internal/handler/dto/group_mapper_profit_visibility_test.go index e0a214218076..4e1080c98bd0 100644 --- a/backend/internal/handler/dto/group_mapper_profit_visibility_test.go +++ b/backend/internal/handler/dto/group_mapper_profit_visibility_test.go @@ -60,12 +60,12 @@ func TestGroupFromServiceOmitsProfitControl(t *testing.T) { } } -// TestGroupFromServiceAdminReturnsDormantProfitControlFields keeps the legacy -// response shape without exposing dormant stored values. -func TestGroupFromServiceAdminReturnsDormantProfitControlFields(t *testing.T) { +// TestGroupFromServiceAdminReturnsStoredProfitControlFields shows administrators +// the stored legacy values of the retired profit control, read-only. +func TestGroupFromServiceAdminReturnsStoredProfitControlFields(t *testing.T) { admin := GroupFromServiceAdmin(profitControlServiceGroup()) - if admin.ProfitControlEnabled || admin.ProfitMinMargin != 0 || admin.ProfitSafetyBuffer != 0 { - t.Fatalf("管理员 DTO 必须把休眠利润字段固定为零值: %+v", admin) + if !admin.ProfitControlEnabled || admin.ProfitMinMargin != 0.3 || admin.ProfitSafetyBuffer != 0.05 { + t.Fatalf("管理员 DTO 必须原样返回库里的利润控制旧值: %+v", admin) } fields := marshalToMap(t, admin) for _, f := range profitControlJSONFields { diff --git a/backend/internal/handler/dto/mappers.go b/backend/internal/handler/dto/mappers.go index c97bb00b1498..3c03350c51db 100644 --- a/backend/internal/handler/dto/mappers.go +++ b/backend/internal/handler/dto/mappers.go @@ -110,6 +110,10 @@ func APIKeyFromService(k *service.APIKey) *APIKey { User: UserFromServiceShallow(k.User), Group: GroupFromServiceShallow(k.Group), } + if k.Purpose != "" && k.Purpose != service.APIKeyPurposeUser { + out.Purpose = k.Purpose + out.LeaseID = k.LeaseID + } if k.Window5hStart != nil && !service.IsWindowExpired(k.Window5hStart, service.RateLimitWindow5h) { t := k.Window5hStart.Add(service.RateLimitWindow5h) out.Reset5hAt = &t @@ -146,13 +150,15 @@ func GroupFromServiceAdmin(g *service.Group) *AdminGroup { if g == nil { return nil } + // Profit control is retired and has no runtime effect; the stored legacy + // values are returned read-only (writes are rejected). out := &AdminGroup{ Group: groupFromServiceBase(g), ForceOpenAIFast: g.ForceOpenAIFast, FreeOpenAIFast: g.FreeOpenAIFast, - ProfitControlEnabled: false, - ProfitMinMargin: 0, - ProfitSafetyBuffer: 0, + ProfitControlEnabled: g.ProfitControlEnabled, + ProfitMinMargin: g.ProfitMinMargin, + ProfitSafetyBuffer: g.ProfitSafetyBuffer, ModelPricing: g.ModelPricing, ModelRouting: g.ModelRouting, ModelRoutingEnabled: g.ModelRoutingEnabled, @@ -444,14 +450,12 @@ func redactAccountManagedExtra(extra map[string]any) map[string]any { } redacted := make(map[string]any, len(extra)) for key, value := range extra { - switch { - case key == service.OllamaCloudUsageSessionExtraKey, - key == service.OllamaCloudUsageAutoRefreshExtraKey, - key == service.OllamaCloudUsageSnapshotExtraKey, - key == service.OpenCodeGoUsageAutoRefreshExtraKey, - key == service.OpenCodeGoUsageSnapshotExtraKey: - continue - case service.IsOpenAICodexTicketPrivateExtraKey(key): + switch key { + case service.OllamaCloudUsageSessionExtraKey, + service.OllamaCloudUsageAutoRefreshExtraKey, + service.OllamaCloudUsageSnapshotExtraKey, + service.OpenCodeGoUsageAutoRefreshExtraKey, + service.OpenCodeGoUsageSnapshotExtraKey: continue default: redacted[key] = value diff --git a/backend/internal/handler/dto/types.go b/backend/internal/handler/dto/types.go index 7f57692a51eb..44e74075f559 100644 --- a/backend/internal/handler/dto/types.go +++ b/backend/internal/handler/dto/types.go @@ -88,6 +88,11 @@ type APIKey struct { User *User `json:"user,omitempty"` Group *Group `json:"group,omitempty"` + + // Purpose and LeaseID mark internal keys, such as the leased keys of the + // release acceptance runner. Ordinary user keys omit both. + Purpose string `json:"purpose,omitempty"` + LeaseID *string `json:"lease_id,omitempty"` } type Group struct { diff --git a/backend/internal/handler/gateway_handler.go b/backend/internal/handler/gateway_handler.go index 6e4d824575c8..261839e71ecd 100644 --- a/backend/internal/handler/gateway_handler.go +++ b/backend/internal/handler/gateway_handler.go @@ -1880,7 +1880,8 @@ func (h *GatewayHandler) handleConcurrencyError(c *gin.Context, err error, slotT func (h *GatewayHandler) handleFailoverExhausted(c *gin.Context, failoverErr *service.UpstreamFailoverError, platform string, streamStarted bool) { if failoverErr != nil && failoverErr.IsOpenAIModelNotSupported() { - service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedClientMessage, "") + // The client keeps the fixed terminal; Ops records the upstream text. + service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIFailoverUpstreamMessage(failoverErr), "") h.handleStreamingAwareError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedCode, service.OpenAIModelNotSupportedClientMessage, streamStarted) return } diff --git a/backend/internal/handler/image_studio_executor.go b/backend/internal/handler/image_studio_executor.go index 1bfaab73fd43..a0a9b1079acc 100644 --- a/backend/internal/handler/image_studio_executor.go +++ b/backend/internal/handler/image_studio_executor.go @@ -13,8 +13,10 @@ import ( "net/textproto" "strconv" "strings" + "sync" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/gin-gonic/gin" @@ -31,16 +33,94 @@ type imageStudioSubscriptionFinder interface { type ImageStudioGatewayExecutor struct { images imageStudioImagesInvoker subscriptions imageStudioSubscriptionFinder + ops *service.OpsService + + routerOnce sync.Once + router *gin.Engine } -func NewImageStudioGatewayExecutor(images imageStudioImagesInvoker, subscriptions *service.SubscriptionService) *ImageStudioGatewayExecutor { - return &ImageStudioGatewayExecutor{images: images, subscriptions: subscriptions} +func NewImageStudioGatewayExecutor(images imageStudioImagesInvoker, subscriptions *service.SubscriptionService, ops *service.OpsService) *ImageStudioGatewayExecutor { + return &ImageStudioGatewayExecutor{images: images, subscriptions: subscriptions, ops: ops} } func newImageStudioGatewayExecutorForTest(images imageStudioImagesInvoker, subscriptions imageStudioSubscriptionFinder) *ImageStudioGatewayExecutor { return &ImageStudioGatewayExecutor{images: images, subscriptions: subscriptions} } +// imageStudioGatewayCall carries one job item's authentication into the shared +// in-process router. +type imageStudioGatewayCall struct { + apiKey *service.APIKey + subscription *service.UserSubscription +} + +type imageStudioGatewayCallKey struct{} + +// gatewayRouter runs Image Studio requests through the Ops error logger in +// front of the images handler, as the public /v1/images routes do, so failed +// generations leave an Ops error record. +func (e *ImageStudioGatewayExecutor) gatewayRouter() *gin.Engine { + e.routerOnce.Do(func() { + router := gin.New() + router.Any("/*endpoint", OpsErrorLoggerMiddleware(e.ops), imageStudioGatewayAuth, e.images.Images) + e.router = router + }) + return e.router +} + +// applyImageStudioUsageOrigin records the Image Studio submitter's address and +// User-Agent on the usage row of an in-process gateway request. +func applyImageStudioUsageOrigin(c *gin.Context, snapshot *openAIUsageSnapshot) { + if c == nil || c.Request == nil || snapshot == nil { + return + } + origin, ok := service.ImageStudioRequestOriginFromContext(c.Request.Context()) + if !ok { + return + } + if userAgent := strings.TrimSpace(origin.UserAgent); userAgent != "" { + snapshot.userAgent = userAgent + } + if clientIP := strings.TrimSpace(origin.ClientIP); clientIP != "" { + snapshot.ipAddress = clientIP + } +} + +// applyImageStudioOpsOrigin records the Image Studio submitter's address and +// User-Agent on the Ops row of an in-process gateway request. +func applyImageStudioOpsOrigin(c *gin.Context, entry *service.OpsInsertErrorLogInput) { + if c == nil || c.Request == nil || entry == nil { + return + } + origin, ok := service.ImageStudioRequestOriginFromContext(c.Request.Context()) + if !ok { + return + } + if userAgent := strings.TrimSpace(origin.UserAgent); userAgent != "" { + entry.UserAgent = userAgent + } + if clientIP := strings.TrimSpace(origin.ClientIP); clientIP != "" { + entry.ClientIP = &clientIP + } +} + +func imageStudioGatewayAuth(c *gin.Context) { + call, _ := c.Request.Context().Value(imageStudioGatewayCallKey{}).(*imageStudioGatewayCall) + if call == nil || call.apiKey == nil || call.apiKey.User == nil { + c.AbortWithStatus(http.StatusServiceUnavailable) + return + } + c.Set(string(middleware2.ContextKeyAPIKey), call.apiKey) + c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{ + UserID: call.apiKey.UserID, Concurrency: call.apiKey.User.Concurrency, + }) + c.Set(string(middleware2.ContextKeyUserRole), call.apiKey.User.Role) + if call.subscription != nil { + c.Set(string(middleware2.ContextKeySubscription), call.subscription) + } + c.Next() +} + func (e *ImageStudioGatewayExecutor) Execute(ctx context.Context, request service.ImageStudioExecutionRequest) (*service.ImageStudioExecutionResult, error) { if e == nil || e.images == nil || request.APIKey == nil || request.APIKey.User == nil { return nil, errors.New("image gateway is unavailable") @@ -53,29 +133,34 @@ func (e *ImageStudioGatewayExecutor) Execute(ctx context.Context, request servic if err != nil { return nil, err } - httpRequest := httptest.NewRequest(http.MethodPost, path, body).WithContext(ctx) - httpRequest.Header.Set("Content-Type", contentType) - recorder := httptest.NewRecorder() - c, _ := gin.CreateTestContext(recorder) - c.Request = httpRequest - c.Set(string(middleware2.ContextKeyAPIKey), request.APIKey) - c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{ - UserID: request.APIKey.UserID, Concurrency: request.APIKey.User.Concurrency, - }) - c.Set(string(middleware2.ContextKeyUserRole), request.APIKey.User.Role) + call := &imageStudioGatewayCall{apiKey: request.APIKey} if request.APIKey.Group != nil && request.APIKey.Group.IsSubscriptionType() { if e.subscriptions == nil { return nil, errors.New("image subscription is unavailable") } subscription, lookupErr := e.subscriptions.GetActiveSubscription(ctx, request.APIKey.UserID, request.APIKey.Group.ID) - if lookupErr != nil || subscription == nil { + if lookupErr != nil { + return nil, fmt.Errorf("image subscription is unavailable: %w", lookupErr) + } + if subscription == nil { return nil, errors.New("image subscription is unavailable") } - c.Set(string(middleware2.ContextKeySubscription), subscription) + call.subscription = subscription } - e.images.Images(c) + // The correlation ID ties Ops and usage rows to this attempt of the job + // item; it is also the usage billing dedupe key, so it is new per attempt. + ctx = context.WithValue(ctx, ctxkey.ClientRequestID, service.NewImageStudioClientRequestID(request.Job.ID, request.Item.ID)) + ctx = context.WithValue(ctx, imageStudioGatewayCallKey{}, call) + // The job's submitter is recorded on usage and Ops rows from the context. + // It is not put on the request: the images handler forwards request + // headers such as User-Agent to API-key upstreams. + ctx = service.WithImageStudioRequestOrigin(ctx, request.Origin) + httpRequest := httptest.NewRequest(http.MethodPost, path, body).WithContext(ctx) + httpRequest.Header.Set("Content-Type", contentType) + recorder := httptest.NewRecorder() + e.gatewayRouter().ServeHTTP(recorder, httpRequest) if recorder.Code != http.StatusOK { - return nil, errors.New("image gateway request failed") + return nil, &service.ImageStudioGatewayError{StatusCode: recorder.Code, Body: recorder.Body.String()} } return decodeImageStudioGatewayResponse(recorder.Body.Bytes()) } @@ -150,20 +235,26 @@ func decodeImageStudioGatewayResponse(body []byte) (*service.ImageStudioExecutio RevisedPrompt string `json:"revised_prompt"` } `json:"data"` } - if err := json.Unmarshal(body, &payload); err != nil || len(payload.Data) != 1 || payload.Data[0].B64JSON == "" { - return nil, errors.New("image gateway returned an invalid result") + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("image gateway returned a result that is not JSON: %w; response: %s", err, service.ImageStudioLogText(string(body))) + } + if len(payload.Data) != 1 || payload.Data[0].B64JSON == "" { + return nil, fmt.Errorf("image gateway returned %d images instead of one base64 image; response: %s", len(payload.Data), service.ImageStudioLogText(string(body))) } encoded := payload.Data[0].B64JSON - if base64.StdEncoding.DecodedLen(len(encoded)) > service.ImageStudioMaxImageBytes { - return nil, errors.New("image gateway returned an oversized result") + if size := base64.StdEncoding.DecodedLen(len(encoded)); size > service.ImageStudioMaxImageBytes { + return nil, fmt.Errorf("image gateway returned an oversized image: about %d bytes, the limit is %d", size, service.ImageStudioMaxImageBytes) } data, err := base64.StdEncoding.DecodeString(encoded) - if err != nil || len(data) > service.ImageStudioMaxImageBytes { - return nil, errors.New("image gateway returned an invalid result") + if err != nil { + return nil, fmt.Errorf("image gateway returned an image that is not valid base64: %w", err) + } + if len(data) > service.ImageStudioMaxImageBytes { + return nil, fmt.Errorf("image gateway returned an oversized image: %d bytes, the limit is %d", len(data), service.ImageStudioMaxImageBytes) } contentType := imageStudioContentType(data) if contentType == "" { - return nil, errors.New("image gateway returned an invalid image") + return nil, fmt.Errorf("image gateway returned %d bytes that are not a PNG, JPEG or WebP image (first bytes %x)", len(data), data[:min(len(data), 12)]) } return &service.ImageStudioExecutionResult{ Data: data, ContentType: contentType, RevisedPrompt: strings.TrimSpace(payload.Data[0].RevisedPrompt), diff --git a/backend/internal/handler/image_studio_executor_test.go b/backend/internal/handler/image_studio_executor_test.go index 6b46c2b2f9ba..c63fcdbaed06 100644 --- a/backend/internal/handler/image_studio_executor_test.go +++ b/backend/internal/handler/image_studio_executor_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/gin-gonic/gin" @@ -90,17 +91,35 @@ func TestImageStudioGatewayExecutorUsesNativeNOneForEdit(t *testing.T) { require.Equal(t, "image/png", result.ContentType) } -func TestImageStudioGatewayExecutorDoesNotExposeUpstreamErrorBody(t *testing.T) { +// A failed generation keeps the gateway status and body for administrators +// (server log and Ops). The submitter's IP and User-Agent reach the usage and +// Ops rows from the context; the request headers, which the images handler +// forwards upstream, do not carry them. +func TestImageStudioGatewayExecutorKeepsGatewayStatusBodyAndOrigin(t *testing.T) { enableImageStudioPolicy(t) executor := newImageStudioGatewayExecutorForTest(imageStudioImagesInvokerFunc(func(c *gin.Context) { - c.JSON(http.StatusBadGateway, gin.H{"error": gin.H{"message": "private upstream body"}}) + require.Empty(t, c.GetHeader("User-Agent")) + require.NotEqual(t, "203.0.113.9", c.ClientIP()) + entry := &service.OpsInsertErrorLogInput{UserAgent: c.GetHeader("User-Agent")} + applyImageStudioOpsOrigin(c, entry) + require.Equal(t, "studio-browser/1.0", entry.UserAgent) + require.NotNil(t, entry.ClientIP) + require.Equal(t, "203.0.113.9", *entry.ClientIP) + snapshot := &openAIUsageSnapshot{} + applyImageStudioUsageOrigin(c, snapshot) + require.Equal(t, "studio-browser/1.0", snapshot.userAgent) + require.Equal(t, "203.0.113.9", snapshot.ipAddress) + c.JSON(http.StatusBadGateway, gin.H{"error": gin.H{"message": "upstream said no"}}) }), nil) _, err := executor.Execute(context.Background(), service.ImageStudioExecutionRequest{ Job: service.ImageStudioJob{UserID: 11, Mode: service.ImageStudioModeGenerate, Model: service.ImageStudioModelGPTImage2, Prompt: "draw"}, APIKey: imageStudioExecutorAPIKey(), + Origin: service.ImageStudioRequestOrigin{ClientIP: "203.0.113.9", UserAgent: "studio-browser/1.0"}, }) - require.Error(t, err) - require.NotContains(t, err.Error(), "private upstream body") + var gatewayErr *service.ImageStudioGatewayError + require.ErrorAs(t, err, &gatewayErr) + require.Equal(t, http.StatusBadGateway, gatewayErr.StatusCode) + require.JSONEq(t, `{"error":{"message":"upstream said no"}}`, gatewayErr.Body) } type imageStudioSubscriptionFinderStub struct { @@ -129,3 +148,44 @@ func TestImageStudioGatewayExecutorPreservesSubscriptionBillingContext(t *testin }) require.NoError(t, err) } + +// The client request ID is the usage billing dedupe key ("client:"+ID), so two +// executions of the same job item must not share it. +func TestImageStudioGatewayExecutorUsesNewBillingIDPerAttempt(t *testing.T) { + enableImageStudioPolicy(t) + var ids []string + executor := newImageStudioGatewayExecutorForTest(imageStudioImagesInvokerFunc(func(c *gin.Context) { + id, _ := c.Request.Context().Value(ctxkey.ClientRequestID).(string) + ids = append(ids, id) + c.JSON(http.StatusOK, gin.H{"data": []gin.H{{"b64_json": base64.StdEncoding.EncodeToString(imageStudioExecutorPNG())}}}) + }), nil) + request := service.ImageStudioExecutionRequest{ + Job: service.ImageStudioJob{ID: 41, UserID: 11, Mode: service.ImageStudioModeGenerate, Model: service.ImageStudioModelGPTImage2, Prompt: "draw"}, + Item: service.ImageStudioItem{ID: 7}, + APIKey: imageStudioExecutorAPIKey(), + } + for range 2 { + _, err := executor.Execute(context.Background(), request) + require.NoError(t, err) + } + require.Len(t, ids, 2) + require.NotEqual(t, ids[0], ids[1]) + for _, id := range ids { + require.True(t, strings.HasPrefix(id, service.ImageStudioClientRequestIDPrefix+"job-41-item-7-"), id) + require.LessOrEqual(t, len("client:"+id), 64) + } +} + +// Decode failures name the concrete reason; a response that is not an image +// result is quoted only up to 4 KiB. +func TestDecodeImageStudioGatewayResponseReportsConcreteReasons(t *testing.T) { + _, err := decodeImageStudioGatewayResponse([]byte("" + strings.Repeat("x", 64<<10))) + require.ErrorContains(t, err, "not JSON") + require.Less(t, len(err.Error()), 5<<10) + + _, err = decodeImageStudioGatewayResponse([]byte(`{"data":[{"b64_json":"not base64!"}]}`)) + require.ErrorContains(t, err, "not valid base64") + + _, err = decodeImageStudioGatewayResponse([]byte(`{"data":[{"b64_json":"` + base64.StdEncoding.EncodeToString([]byte("GIF89a-fixture")) + `"}]}`)) + require.ErrorContains(t, err, "not a PNG, JPEG or WebP image") +} diff --git a/backend/internal/handler/image_studio_job_handler.go b/backend/internal/handler/image_studio_job_handler.go index a51fa4dcd8fe..fc20e47e82f4 100644 --- a/backend/internal/handler/image_studio_job_handler.go +++ b/backend/internal/handler/image_studio_job_handler.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/Wei-Shaw/sub2api/internal/pkg/ip" "github.com/Wei-Shaw/sub2api/internal/pkg/response" middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware" "github.com/Wei-Shaw/sub2api/internal/service" @@ -88,7 +89,7 @@ func (h *ImageStudioJobHandler) Create(c *gin.Context) { imageStudioJobError(c, err) return } - job, err := h.studio.Create(c.Request.Context(), userID, service.ImageStudioCreateInput{ + job, err := h.studio.Create(imageStudioRequestContext(c), userID, service.ImageStudioCreateInput{ APIKeyID: apiKeyID, Mode: service.ImageStudioMode(c.PostForm("mode")), Model: c.PostForm("model"), @@ -180,7 +181,7 @@ func (h *ImageStudioJobHandler) Retry(c *gin.Context) { imageStudioJobError(c, err) return } - job, err := h.studio.Retry(c.Request.Context(), userID, jobID) + job, err := h.studio.Retry(imageStudioRequestContext(c), userID, jobID) if err != nil { imageStudioJobError(c, err) return @@ -188,6 +189,14 @@ func (h *ImageStudioJobHandler) Retry(c *gin.Context) { response.Accepted(c, job) } +// imageStudioRequestContext records the submitting client so the job's +// gateway requests carry its real IP address and User-Agent. +func imageStudioRequestContext(c *gin.Context) context.Context { + return service.WithImageStudioRequestOrigin(c.Request.Context(), service.ImageStudioRequestOrigin{ + ClientIP: ip.GetClientIP(c), UserAgent: c.GetHeader("User-Agent"), + }) +} + func (h *ImageStudioJobHandler) Artifact(c *gin.Context) { userID, jobID, ok := h.ownerAndJobID(c) if !ok { diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 25fd898bd827..911462aca198 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -473,6 +473,10 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { return } if err := h.gatewayService.AdmitCodexQualityRequest(c, apiKey, body, h.apiKeyService.GetByID); err != nil { + // The reason names users, groups, API keys and proxies, so the client + // keeps the fixed sentence; the server log and the Ops record carry it. + reqLog.Warn("openai.codex_quality_admission_rejected", zap.Error(err)) + setOpsLocalErrorDetail(c, err.Error()) h.errorResponse(c, http.StatusConflict, "codex_quality_unavailable", "Codex quality run unavailable") return } @@ -3893,7 +3897,9 @@ func (h *OpenAIGatewayHandler) handleConcurrencyError(c *gin.Context, err error, func (h *OpenAIGatewayHandler) handleOpenAINoAccountError(c *gin.Context, classification noAccountErrorClassification, streamStarted bool) { if classification.ErrType == service.OpenAIModelNotSupportedCode { - service.SetOpsUpstreamError(c, http.StatusBadRequest, classification.Message, "") + // No upstream was called for this request; record why instead of the + // client sentence. The status keeps the existing upstream attribution. + service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedNoAccountOpsMessage, "") h.handleStreamingAwareErrorWithCode( c, classification.Status, @@ -3943,7 +3949,9 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE if upstreamStatus <= 0 { upstreamStatus = http.StatusBadRequest } - service.SetOpsUpstreamError(c, upstreamStatus, service.OpenAIRequestRejectedClientMessage, "") + // The attempt event already carries the upstream text; never replace it + // with the client sentence. + service.SetOpsUpstreamError(c, upstreamStatus, service.OpenAIFailoverUpstreamMessage(failoverErr), "") // Record the semantic failure before the renderer can commit SSE HTTP // 200 or add its non-SLA fallback mark. This branch must precede the // generic ClientErrorCode JSON path, including pre-first-byte failures. @@ -3979,7 +3987,7 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE return } if failoverErr.IsOpenAIModelNotSupported() { - service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedClientMessage, "") + service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIFailoverUpstreamMessage(failoverErr), "") h.handleStreamingAwareErrorWithCode( c, http.StatusBadRequest, @@ -4000,11 +4008,13 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE if message == "" { message = service.OpenAIContinuationStateUnavailableClientMessage } + // An in-band response.failed arrives on HTTP 200; that transport status + // is not the upstream's error status. upstreamStatus := failoverErr.StatusCode - if upstreamStatus <= 0 { + if upstreamStatus < http.StatusBadRequest { upstreamStatus = statusCode } - service.SetOpsUpstreamError(c, upstreamStatus, message, "") + service.SetOpsUpstreamError(c, upstreamStatus, service.OpenAIFailoverUpstreamMessage(failoverErr), "") errType := "invalid_request_error" if statusCode >= http.StatusInternalServerError { errType = "server_error" @@ -4041,7 +4051,9 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE return } if failoverErr.IsOpenAIRefusalRecovery() { - service.SetOpsUpstreamError(c, http.StatusServiceUnavailable, failoverErr.ClientMessage, "") + // The refusal attempts carry their matched keyword and original text; + // the client sentence is not upstream evidence. + service.SetOpsUpstreamError(c, http.StatusServiceUnavailable, "", "") if failoverErr.IsOpenAICyberFailover() { h.handleStreamingAwareErrorWithDetails( c, @@ -4536,7 +4548,7 @@ func wsSelectionModelNotSupported( } func closeOpenAIWSModelNotSupported(c *gin.Context, conn *coderws.Conn) { - service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedClientMessage, "") + service.SetOpsUpstreamError(c, http.StatusBadRequest, service.OpenAIModelNotSupportedNoAccountOpsMessage, "") service.MarkOpsStreamFailure(c, service.OpenAIModelNotSupportedCode, service.OpenAIModelNotSupportedCode, service.OpenAIModelNotSupportedClientMessage, http.StatusBadRequest) writeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) if conn != nil { @@ -4561,10 +4573,17 @@ func closeOpenAIWSFailoverExhausted(c *gin.Context, conn *coderws.Conn, failover errorType = service.OpenAIModelNotSupportedCode errorCode = service.OpenAIModelNotSupportedCode message = service.OpenAIModelNotSupportedClientMessage + // The client receives the fixed terminal; Ops keeps the upstream text. + service.SetOpsUpstreamError(c, failoverErr.StatusCode, service.OpenAIFailoverUpstreamMessage(failoverErr), "") case failoverErr.IsOpenAIContinuationStateUnavailable(): intendedStatus = http.StatusBadRequest errorType = "invalid_request_error" message = service.OpenAIContinuationStateUnavailableClientMessage + upstreamStatus := failoverErr.StatusCode + if upstreamStatus < http.StatusBadRequest { + upstreamStatus = intendedStatus + } + service.SetOpsUpstreamError(c, upstreamStatus, service.OpenAIFailoverUpstreamMessage(failoverErr), "") case failoverErr.Stage == service.GatewayFailureStageAccountAuth: intendedStatus = http.StatusServiceUnavailable errorType = "api_error" diff --git a/backend/internal/handler/openai_images.go b/backend/internal/handler/openai_images.go index 6d6b69fcd980..0c09327311eb 100644 --- a/backend/internal/handler/openai_images.go +++ b/backend/internal/handler/openai_images.go @@ -428,6 +428,7 @@ func (h *OpenAIGatewayHandler) Images(c *gin.Context) { requestPayloadHash = service.HashUsageRequestPayload([]byte(parsed.StickySessionSeed())) } usageSnapshot := snapshotOpenAIUsageMetadataWithHash(c, apiKey, account, subscription, channelMapping, requestModel, result, requestPayloadHash) + applyImageStudioUsageOrigin(c, usageSnapshot) usageSnapshot.upstreamEndpoint = GetUpstreamEndpoint(c, account.Platform) usageInput := usageSnapshot.Input(result, h.apiKeyService, service.OpenAIPricingAtFromContext(c.Request.Context())) usageAPIKeyID := usageInput.APIKey.ID diff --git a/backend/internal/handler/openai_request_rejected_test.go b/backend/internal/handler/openai_request_rejected_test.go index 2928c9825f8d..0328974dcebe 100644 --- a/backend/internal/handler/openai_request_rejected_test.go +++ b/backend/internal/handler/openai_request_rejected_test.go @@ -4,9 +4,11 @@ package handler import ( "net/http" + "net/http/httptest" "strings" "testing" + middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" @@ -90,6 +92,11 @@ func TestOpenAIGatewayHandlerResponses_RequestRejectedStopsAndReportsSemantic400 require.Equal(t, service.OpenAIRequestRejectedCode, mark.Code) require.Len(t, mark.UpstreamErrors, 1) require.Equal(t, "request_rejected", mark.UpstreamErrors[0].Kind) + // The client never sees the upstream text; the Ops record keeps it + // and the handler no longer overwrites it with the client sentence. + upstreamMessage := gjson.Get(rejection.body, "error.message").String() + require.Equal(t, upstreamMessage, mark.UpstreamErrors[0].Message) + require.Equal(t, upstreamMessage, mark.UpstreamMessage) require.NotNil(t, mark.UpstreamErrors[0].ContinuationDiagnostic) require.Equal(t, rejection.classification, mark.UpstreamErrors[0].ContinuationDiagnostic.Classification) if format.stream { @@ -147,3 +154,43 @@ func TestOpenAIGatewayHandler_CompactRequestRejectedBeforeKeepaliveUsesJSON(t *t require.Equal(t, service.OpenAIRequestRejectedCode, gjson.GetBytes(rec.Body.Bytes(), "error.code").String()) require.NotContains(t, rec.Body.String(), "event:") } + +// The admission reason names users, groups, API keys and proxies: the client +// keeps the fixed sentence and status, the Ops record carries the reason. +func TestOpenAIGatewayHandlerResponses_CodexQualityAdmissionReasonOnlyInOps(t *testing.T) { + gin.SetMode(gin.TestMode) + setupOpsErrorLogTestQueue(t, 2) + handler := newOpenAIResponsesFailoverTestHandler(t, &openAIResponsesGenericBadRequestUpstream{}) + ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + groupID := int64(3131) + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{ + ID: 99, GroupID: &groupID, + Group: &service.Group{ID: groupID, Platform: service.PlatformOpenAI}, + User: &service.User{ID: 100}, + }) + c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{UserID: 100}) + c.Next() + }) + router.Use(OpsErrorLoggerMiddleware(ops)) + router.POST("/v1/responses", handler.Responses) + + req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"gpt-5.1","stream":true,"input":"hello"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(service.CodexQualityGrantHeader, "not-a-grant") + req.Header.Set(service.CodexQualityTrialHeader, "not-a-trial") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + require.Equal(t, http.StatusConflict, rec.Code) + require.JSONEq(t, `{"error":{"type":"codex_quality_unavailable","message":"Codex quality run unavailable"}}`, rec.Body.String()) + require.Equal(t, int64(1), OpsErrorLogQueueLength()) + job := <-opsErrorLogQueue + require.Equal(t, http.StatusConflict, job.entry.StatusCode) + require.Equal(t, "Codex quality run unavailable", job.entry.ErrorMessage, "users can read error_message in their error-request view") + require.NotNil(t, job.entry.UpstreamErrorDetail) + require.True(t, strings.HasPrefix(*job.entry.UpstreamErrorDetail, "local: "+service.ErrCodexQualityUnavailable.Error()+": "), + "the admin-only detail names the concrete admission reason: %s", *job.entry.UpstreamErrorDetail) + require.Contains(t, *job.entry.UpstreamErrorDetail, service.CodexQualityGrantHeader) +} diff --git a/backend/internal/handler/ops_error_logger.go b/backend/internal/handler/ops_error_logger.go index 713eede5427d..5311d7b07912 100644 --- a/backend/internal/handler/ops_error_logger.go +++ b/backend/internal/handler/ops_error_logger.go @@ -37,6 +37,10 @@ const ( opsUpstreamModelKey = service.OpsUpstreamModelKey opsRequestTypeKey = "ops_request_type" opsDedicatedEntryEnqueuedKey = "ops_dedicated_entry_enqueued" + // opsLocalErrorDetailKey keeps the concrete reason of a local rejection + // whose client response is a fixed sentence. Only the stored Ops message + // reads it; classification and error filters use the client message. + opsLocalErrorDetailKey = "ops_local_error_detail" // 错误过滤匹配常量 — shouldSkipOpsErrorLog 和错误分类共用 opsErrContextCanceled = "context canceled" @@ -486,6 +490,71 @@ func setOpsSelectedAccount(c *gin.Context, accountID int64, platform ...string) } } +// setOpsLocalErrorDetail records, for administrators only, why a local check +// rejected a request whose client response stays a fixed sentence. +func setOpsLocalErrorDetail(c *gin.Context, detail string) { + if c == nil { + return + } + if detail = strings.TrimSpace(detail); detail != "" { + c.Set(opsLocalErrorDetailKey, truncateString(detail, 2048)) + } +} + +// applyOpsLocalErrorDetail stores the local reason in upstream_error_detail, +// prefixed "local:". It runs after classification, and error_message keeps the +// client's sentence: the user error view returns error_message and error_body +// but never upstream_error_detail. +func applyOpsLocalErrorDetail(c *gin.Context, entry *service.OpsInsertErrorLogInput) { + if c == nil || entry == nil { + return + } + detail := strings.TrimSpace(c.GetString(opsLocalErrorDetailKey)) + if detail == "" { + return + } + local := "local: " + detail + if entry.UpstreamErrorDetail != nil && strings.TrimSpace(*entry.UpstreamErrorDetail) != "" { + local += "\n" + *entry.UpstreamErrorDetail + } + entry.UpstreamErrorDetail = &local +} + +// attachOpsRefusalRecoveryEvidence puts the evidence of HTTP refusal rewrites +// and prompt retries in front of the attempts of a request that finally +// failed. It is never classification input: the top-level upstream fields +// keep coming from the attempts, and the last attempt stays last. +func attachOpsRefusalRecoveryEvidence(c *gin.Context, entry *service.OpsInsertErrorLogInput) { + evidence := service.OpsRefusalRecoveryEvents(c) + if entry == nil || len(evidence) == 0 { + return + } + merged := make([]*service.OpsUpstreamErrorEvent, 0, len(evidence)+len(entry.UpstreamErrors)) + merged = append(merged, evidence...) + merged = append(merged, entry.UpstreamErrors...) + entry.UpstreamErrors = merged +} + +// isOpsGatewayFixedClientMessage reports whether a client-visible message is a +// sentence the gateway writes in place of an upstream error text. +func isOpsGatewayFixedClientMessage(message string) bool { + message = strings.TrimSpace(message) + if service.IsGatewayFixedClientMessage(message) { + return true + } + switch message { + case cyberSessionBlockedClientMsg, + "Upstream authentication failed, please contact administrator", + "Upstream access forbidden, please contact administrator", + "Upstream rate limit exceeded, please retry later", + "Upstream service overloaded, please retry later", + "Upstream service temporarily unavailable", + "Upstream request failed": + return true + } + return false +} + func markOpsRoutingCapacityLimited(c *gin.Context) { if c == nil { return @@ -1257,7 +1326,10 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc { applyOpsLatencyFieldsFromContext(c, entry) applyOpsUpstreamFieldsFromContext(c, entry) if parsed.StreamFailure { - if message := strings.TrimSpace(parsed.Message); message != "" { + // The terminal text becomes the upstream message (upstream v0.2.8) + // unless it is a sentence the gateway wrote in place of the + // upstream's text; then the recorded upstream message stays. + if message := strings.TrimSpace(parsed.Message); message != "" && !isOpsGatewayFixedClientMessage(message) { entry.UpstreamErrorMessage = &message } // The SSE terminal determines the client-visible semantic status, not @@ -1270,6 +1342,8 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc { } } suppressOpsUpstreamAttributionForLocalModelConfiguration(c, entry) + applyOpsLocalErrorDetail(c, entry) + attachOpsRefusalRecoveryEvidence(c, entry) if apiKey != nil { entry.APIKeyID = &apiKey.ID @@ -1293,6 +1367,7 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc { entry.ClientIP = &clientIP } + applyImageStudioOpsOrigin(c, entry) enqueueOpsErrorLog(ops, entry) } } @@ -1409,6 +1484,7 @@ func logOpsRecoveredUpstream(c *gin.Context, ops *service.OpsService, finalStatu entry.ClientIP = &clientIP } applyOpsLatencyFieldsFromContext(c, entry) + applyImageStudioOpsOrigin(c, entry) enqueueOpsErrorLog(ops, entry) } @@ -1579,6 +1655,9 @@ func logOpsStreamErrorValue(c *gin.Context, ops *service.OpsService, wireStatus if streamErr.Turn > 0 && !streamErr.RequestScoped { applyOpsStreamErrorSnapshot(entry, streamErr) } + if streamErr.Turn == 0 && !streamErr.RequestScoped { + attachOpsRefusalRecoveryEvidence(c, entry) + } if apiKey != nil { entry.APIKeyID = &apiKey.ID @@ -1598,6 +1677,7 @@ func logOpsStreamErrorValue(c *gin.Context, ops *service.OpsService, wireStatus entry.ClientIP = &clientIP } + applyImageStudioOpsOrigin(c, entry) enqueueOpsErrorLog(ops, entry) } diff --git a/backend/internal/handler/ops_error_logger_test.go b/backend/internal/handler/ops_error_logger_test.go index d75077761849..db56f64ddb38 100644 --- a/backend/internal/handler/ops_error_logger_test.go +++ b/backend/internal/handler/ops_error_logger_test.go @@ -2,6 +2,7 @@ package handler import ( "context" + "encoding/json" "net/http" "net/http/httptest" "strings" @@ -565,6 +566,121 @@ func TestOpsErrorLoggerMiddleware_StreamFailureKeepsTerminalSemanticsAndAttemptS require.Equal(t, "input exceeds the context window", *job.entry.UpstreamErrorMessage) } +// A terminal carrying a gateway sentence is not the upstream's text: the +// recorded upstream message stays while error_message is the client sentence. +func TestOpsErrorLoggerMiddleware_StreamFailureGatewaySentenceKeepsUpstreamMessage(t *testing.T) { + setupOpsErrorLogTestQueue(t, 2) + gin.SetMode(gin.TestMode) + + ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + router := gin.New() + router.Use(OpsErrorLoggerMiddleware(ops)) + router.POST("/v1/responses", func(c *gin.Context) { + service.SetOpsUpstreamError(c, http.StatusBadRequest, "Previous response with id 'resp_1' not found.", "") + c.Status(http.StatusOK) + _, _ = c.Writer.WriteString("event: response.failed\n") + _, _ = c.Writer.WriteString(`data: {"type":"response.failed","response":{"error":{"type":"invalid_request_error","code":"continuation_state_unavailable","message":"` + service.OpenAIContinuationStateUnavailableClientMessage + `"}}}`) + _, _ = c.Writer.WriteString("\n\n") + }) + + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/responses", nil)) + + require.Equal(t, int64(1), OpsErrorLogQueueLength()) + job := <-opsErrorLogQueue + require.Equal(t, service.OpenAIContinuationStateUnavailableClientMessage, job.entry.ErrorMessage) + require.NotNil(t, job.entry.UpstreamErrorMessage) + require.Equal(t, "Previous response with id 'resp_1' not found.", *job.entry.UpstreamErrorMessage) +} + +// A local rejection keeps the client's sentence in error_message; the reason +// sits in upstream_error_detail, which the user error view never returns. +func TestOpsErrorLoggerMiddleware_LocalErrorDetailStaysAdminOnly(t *testing.T) { + setupOpsErrorLogTestQueue(t, 2) + gin.SetMode(gin.TestMode) + + const reason = "quality run r1 belongs to API key 7 of user 8 in group 9" + ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + router := gin.New() + router.Use(OpsErrorLoggerMiddleware(ops)) + router.POST("/v1/responses", func(c *gin.Context) { + setOpsLocalErrorDetail(c, reason) + c.JSON(http.StatusConflict, gin.H{"error": gin.H{"type": "codex_quality_unavailable", "message": "Codex quality run unavailable"}}) + }) + + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/responses", nil)) + + require.JSONEq(t, `{"error":{"type":"codex_quality_unavailable","message":"Codex quality run unavailable"}}`, recorder.Body.String()) + require.Equal(t, int64(1), OpsErrorLogQueueLength()) + job := <-opsErrorLogQueue + require.Equal(t, "Codex quality run unavailable", job.entry.ErrorMessage) + require.NotEqual(t, "upstream", job.entry.ErrorPhase) + require.Nil(t, job.entry.UpstreamErrorMessage) + require.NotNil(t, job.entry.UpstreamErrorDetail) + require.Equal(t, "local: "+reason, *job.entry.UpstreamErrorDetail) + + stored := &service.OpsErrorLogDetail{ + OpsErrorLog: service.OpsErrorLog{ + Phase: job.entry.ErrorPhase, + Type: job.entry.ErrorType, + StatusCode: job.entry.StatusCode, + Message: job.entry.ErrorMessage, + }, + ErrorBody: job.entry.ErrorBody, + UpstreamErrorDetail: *job.entry.UpstreamErrorDetail, + } + userView, err := json.Marshal(service.ToUserErrorRequestDetail(stored)) + require.NoError(t, err) + require.NotContains(t, string(userView), "API key 7") + require.Contains(t, string(userView), "Codex quality run unavailable") +} + +// A refusal the gateway rewrote or retried away is evidence, not an upstream +// error: a request that succeeds leaves no Ops row, and a request that +// finally fails carries the evidence without turning it into its upstream +// message. +func TestOpsErrorLoggerMiddleware_RefusalEvidenceOnlyOnFailedRequests(t *testing.T) { + gin.SetMode(gin.TestMode) + evidence := []*service.OpsUpstreamErrorEvent{{ + Kind: "refusal_recovery", + Reason: "rewritten", + Message: `upstream refusal matched keyword "I cannot" (rewritten, transport=http)`, + Detail: "I cannot help with that request.", + }} + for _, tc := range []struct { + name string + status int + rows int64 + }{{"rewritten_success", http.StatusOK, 0}, {"retries_exhausted", http.StatusBadGateway, 1}} { + t.Run(tc.name, func(t *testing.T) { + setupOpsErrorLogTestQueue(t, 2) + ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + router := gin.New() + router.Use(OpsErrorLoggerMiddleware(ops)) + router.POST("/v1/responses", func(c *gin.Context) { + c.Set(service.OpsRefusalRecoveryEventsKey, evidence) + if tc.status >= http.StatusBadRequest { + c.JSON(tc.status, gin.H{"error": gin.H{"type": "upstream_error", "message": "Temporary upstream failure"}}) + return + } + c.JSON(http.StatusOK, gin.H{"id": "resp_1", "status": "completed"}) + }) + + router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/responses", nil)) + + require.Equal(t, tc.rows, OpsErrorLogQueueLength()) + if tc.rows == 0 { + return + } + job := <-opsErrorLogQueue + require.Nil(t, job.entry.UpstreamErrorMessage) + require.NotNil(t, job.entry.UpstreamErrorsJSON) + require.Contains(t, *job.entry.UpstreamErrorsJSON, `"kind":"refusal_recovery"`) + }) + } +} + func TestOpsErrorLoggerMiddleware_PrefersContextRequestID(t *testing.T) { setupOpsErrorLogTestQueue(t, 2) gin.SetMode(gin.TestMode) diff --git a/backend/internal/handler/wire.go b/backend/internal/handler/wire.go index 209c10b2d40f..f41888adeb12 100644 --- a/backend/internal/handler/wire.go +++ b/backend/internal/handler/wire.go @@ -321,8 +321,9 @@ func ProvideAccountJobRuntime(jobs *service.AccountJobService, accountHandler *a return runtime, nil } -func ProvidePluginHandler(manager *service.PluginManager) *admin.PluginHandler { +func ProvidePluginHandler(manager *service.PluginManager, retired *service.NativeFeatureBootstrap) *admin.PluginHandler { handler := admin.NewPluginHandler(manager) + handler.SetRetiredPlugins(retired) return handler } diff --git a/backend/internal/nativeapi/api.go b/backend/internal/nativeapi/api.go index e96f927feb42..9a5a43afa240 100644 --- a/backend/internal/nativeapi/api.go +++ b/backend/internal/nativeapi/api.go @@ -197,15 +197,26 @@ type AccountProjection struct { Observations []AccountObservation `json:"observations,omitempty"` } +// AccountObservationMessageLimit bounds Message because the projection is +// copied into the scheduler snapshot of every account. The complete text stays +// in the domain state (Codex: fingerprint panel and job results). +const AccountObservationMessageLimit = 1024 + +// AccountObservationModelLimit bounds ResponseModel like other model names. +const AccountObservationModelLimit = 256 + type AccountObservation struct { - Key string `json:"key"` - Kind string `json:"kind"` - State string `json:"state"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` - NextAt *time.Time `json:"next_at,omitempty"` - CheckedAt *time.Time `json:"checked_at,omitempty"` - Code string `json:"code,omitempty"` - Count int `json:"count,omitempty"` + Key string `json:"key"` + Kind string `json:"kind"` + State string `json:"state"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + NextAt *time.Time `json:"next_at,omitempty"` + CheckedAt *time.Time `json:"checked_at,omitempty"` + Code string `json:"code,omitempty"` + Count int `json:"count,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` + ResponseModel string `json:"response_model,omitempty"` + Message string `json:"message,omitempty"` } // Invocation carries one declared capability and a domain operation. The host diff --git a/backend/internal/nativeapi/catalog.go b/backend/internal/nativeapi/catalog.go index 75fd84e952ef..c3d69a1aec33 100644 --- a/backend/internal/nativeapi/catalog.go +++ b/backend/internal/nativeapi/catalog.go @@ -40,6 +40,6 @@ type OfficialModelContextCapacity struct { NormalizationBasis string `json:"normalization_basis,omitempty"` Conditions string `json:"conditions,omitempty"` Reference *ModelContextCapacityReference `json:"reference,omitempty"` - MatchHosts []string `json:"-"` - MatchAccountModes []string `json:"-"` + MatchHosts []string `json:"match_hosts,omitempty"` + MatchAccountModes []string `json:"match_account_modes,omitempty"` } diff --git a/backend/internal/nativeapi/codex_routing.go b/backend/internal/nativeapi/codex_routing.go index 93ccbdb57627..58c90d1a7e6f 100644 --- a/backend/internal/nativeapi/codex_routing.go +++ b/backend/internal/nativeapi/codex_routing.go @@ -1,6 +1,10 @@ package nativeapi -import "time" +import ( + "fmt" + "strings" + "time" +) // Routing operations are named, account-scoped host IO. They never accept a // destination URL, arbitrary request body, credential, or arbitrary Cookie. @@ -22,6 +26,10 @@ type CodexRoutingScope struct { RouteEvidence string `json:"route_evidence"` ConnectionLeaseID string `json:"connection_lease_id,omitempty"` Transport string `json:"transport"` + // AccountProxyID records which account proxy the RouteHash was computed + // with, so administrators can see the proxy behind a qualification. It is + // informational only; SameOwner keeps comparing the hashes. + AccountProxyID int64 `json:"account_proxy_id,omitempty"` } // SameOwner intentionally excludes account.updated_at: writing a projection @@ -49,23 +57,76 @@ type CodexRoutingQuery struct { Scope *CodexRoutingScope `json:"scope,omitempty"` } -// All observation fields are safe for an authenticated read-only account view. -// Cookie and STATE values, tokens, proxy URLs and response text are excluded. +// CodexRoutingUpstreamBodyLimit bounds the upstream error body kept with one +// observation (latest attempt per account and model). +const CodexRoutingUpstreamBodyLimit = 8 << 10 + +// CodexRoutingHeaderLimit bounds the upstream response headers kept with one +// probe observation. Each kept value is complete; values that no longer fit +// are listed by name in ResponseHeadersOmitted. +const CodexRoutingHeaderLimit = 4 << 10 + +type CodexRoutingHeader struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// An observation is shown to administrators as recorded: Error is the original +// Go error text, the upstream_* fields and UpstreamBody carry the upstream's own +// error object and (bounded) body, RequestID/CFRay identify the upstream request. +// Probe and validation observations also keep the raw STATE value and every +// upstream response header (Set-Cookie included) within CodexRoutingHeaderLimit. type CodexRoutingObservation struct { - Stage string `json:"stage"` - Code string `json:"code"` - HTTPStatus int `json:"http_status,omitempty"` - RequestedModel string `json:"requested_model,omitempty"` - ReasoningEffort string `json:"reasoning_effort,omitempty"` - ResponseModel string `json:"response_model,omitempty"` - Completed bool `json:"completed"` - ModelMatched bool `json:"model_matched"` - StateLength int `json:"state_length,omitempty"` - CookieNames []string `json:"cookie_names,omitempty"` - ObservedAt time.Time `json:"observed_at"` - DurationMS int64 `json:"duration_ms,omitempty"` - Transport string `json:"transport,omitempty"` - CookieSent bool `json:"cookie_sent"` + Stage string `json:"stage"` + Code string `json:"code"` + HTTPStatus int `json:"http_status,omitempty"` + RequestedModel string `json:"requested_model,omitempty"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` + ResponseModel string `json:"response_model,omitempty"` + Completed bool `json:"completed"` + ModelMatched bool `json:"model_matched"` + StateLength int `json:"state_length,omitempty"` + CookieNames []string `json:"cookie_names,omitempty"` + ObservedAt time.Time `json:"observed_at"` + DurationMS int64 `json:"duration_ms,omitempty"` + Transport string `json:"transport,omitempty"` + CookieSent bool `json:"cookie_sent"` + Error string `json:"error,omitempty"` + UpstreamErrorType string `json:"upstream_error_type,omitempty"` + UpstreamErrorCode string `json:"upstream_error_code,omitempty"` + UpstreamErrorMessage string `json:"upstream_error_message,omitempty"` + UpstreamErrorParam string `json:"upstream_error_param,omitempty"` + UpstreamBody string `json:"upstream_body,omitempty"` + RequestID string `json:"request_id,omitempty"` + CFRay string `json:"cf_ray,omitempty"` + // State is the raw x-codex-turn-state value returned by the upstream. + State string `json:"state,omitempty"` + ResponseHeaders []CodexRoutingHeader `json:"response_headers,omitempty"` + ResponseHeadersOmitted []string `json:"response_headers_omitted,omitempty"` +} + +// Summary returns the original failure text of an observation: the transport or +// host error, otherwise the upstream error object. It never falls back to the +// upstream body, which stays in UpstreamBody only. +func (o CodexRoutingObservation) Summary() string { + if o.Error != "" { + return o.Error + } + var fields []string + for _, field := range [][2]string{{"type", o.UpstreamErrorType}, {"code", o.UpstreamErrorCode}, {"param", o.UpstreamErrorParam}} { + if field[1] != "" { + fields = append(fields, field[0]+"="+field[1]) + } + } + switch { + case o.UpstreamErrorMessage != "" && len(fields) > 0: + return o.UpstreamErrorMessage + " (" + strings.Join(fields, ", ") + ")" + case o.UpstreamErrorMessage != "": + return o.UpstreamErrorMessage + case len(fields) > 0: + return strings.Join(fields, ", ") + } + return "" } type CodexRoutingProbeResult struct { @@ -89,6 +150,32 @@ func (q *CodexRoutingQualification) Valid(now time.Time, id int64, identity, mod now.Before(q.ExpiresAt) && !q.ExpiresAt.After(q.Bundle.ExpiresAt) && !q.ExpiresAt.After(q.VerifiedAt.Add(CodexRoutingMaxAge)) } +// Problem names the first condition that makes Valid false, or returns "". +func (q *CodexRoutingQualification) Problem(now time.Time, id int64, identity, model string) string { + format := func(at time.Time) string { return at.UTC().Format(time.RFC3339) } + switch { + case q == nil: + return "no route qualification is recorded" + case q.Scope.AccountID != id: + return fmt.Sprintf("the route qualification belongs to account %d, not %d", q.Scope.AccountID, id) + case q.Scope.Identity != identity: + return fmt.Sprintf("the route qualification belongs to credential owner %s, the account is now %s", q.Scope.Identity, identity) + case q.Model != model: + return fmt.Sprintf("the route qualification is for model %q, not %q", q.Model, model) + case q.Bundle.Key == "" || q.Bundle.Revision <= 0: + return "the route qualification names no route bundle" + case q.VerifiedAt.IsZero() || q.VerifiedAt.After(now): + return fmt.Sprintf("the route qualification has an invalid verification time %s", format(q.VerifiedAt)) + case !now.Before(q.ExpiresAt): + return fmt.Sprintf("the route qualification expired at %s", format(q.ExpiresAt)) + case q.ExpiresAt.After(q.Bundle.ExpiresAt): + return fmt.Sprintf("the route qualification (until %s) outlives its bundle (until %s)", format(q.ExpiresAt), format(q.Bundle.ExpiresAt)) + case q.ExpiresAt.After(q.VerifiedAt.Add(CodexRoutingMaxAge)): + return fmt.Sprintf("the route qualification (until %s) exceeds the %s cookie age after verification at %s", format(q.ExpiresAt), CodexRoutingMaxAge, format(q.VerifiedAt)) + } + return "" +} + // The host applies only a reference to a host-verified private bundle. Generic // plugin header mutations continue to reject Cookie and Set-Cookie. type CodexRoutingInjection struct { diff --git a/backend/internal/proxytransport/address.go b/backend/internal/proxytransport/address.go index 7b025ad855bc..49f09fd76403 100644 --- a/backend/internal/proxytransport/address.go +++ b/backend/internal/proxytransport/address.go @@ -14,13 +14,16 @@ import ( const parserVersion = "ticket-proxy-v2" -// Candidate deliberately has no exported credential or full address field. -// Selection is resolved by parsing the original draft again, never a client URL. +// Candidate shows the recognized protocol, host, port and username (so records +// that differ only by username stay distinguishable); the password and the +// full address are not echoed. Selection is resolved by parsing the original +// draft again, never a client URL. type Candidate struct { SelectionID string `json:"selection_id"` Protocol string `json:"protocol"` Host string `json:"host"` Port int `json:"port"` + Username string `json:"username,omitempty"` UsernameMasked string `json:"username_masked,omitempty"` SourceLine int `json:"source_line"` SourceColumn int `json:"source_column"` @@ -299,7 +302,7 @@ func candidate(raw, format string, line int) (Candidate, error) { if u.User != nil { masked = "***" } - return Candidate{Protocol: u.Scheme, Host: u.Hostname(), Port: effectivePort(u), UsernameMasked: masked, SourceLine: line, SourceColumn: 1, Format: format, endpoint: raw, identity: identity}, nil + return Candidate{Protocol: u.Scheme, Host: u.Hostname(), Port: effectivePort(u), Username: user, UsernameMasked: masked, SourceLine: line, SourceColumn: 1, Format: format, endpoint: raw, identity: identity}, nil } func fieldsCandidate(fields []string, protocol, format string, line int, reverse bool) (Candidate, error) { diff --git a/backend/internal/proxytransport/address_test.go b/backend/internal/proxytransport/address_test.go index 222606cce6bc..78a1a342738a 100644 --- a/backend/internal/proxytransport/address_test.go +++ b/backend/internal/proxytransport/address_test.go @@ -90,11 +90,15 @@ func TestProxyCandidatesAmbiguityDedupeSelectionAndRedaction(t *testing.T) { t.Fatal("dedupe must include complete credentials and effective endpoint") } encoded, _ := json.Marshal(parsed) - for _, secret := range []string{"fixture-user", "fixture-password", "different-password", "http://"} { + for _, secret := range []string{"fixture-password", "different-password", "http://"} { if strings.Contains(string(encoded), secret) { - t.Fatal("candidate serialization leaked credential material") + t.Fatal("candidate serialization leaked a password or the full address") } } + // Administrators see the username so candidates that differ only by it stay distinguishable. + if parsed.Candidates[0].Username != "fixture-user" || !strings.Contains(string(encoded), `"username":"fixture-user"`) { + t.Fatalf("candidate username is not shown: %s", encoded) + } preserved, err := Resolve(raw, "socks5", parsed.Candidates[0].SelectionID) if err != nil || preserved != strings.Split(raw, "\n")[0] { t.Fatal("existing explicit URI representation changed") diff --git a/backend/internal/repository/account_job_connection_integration_test.go b/backend/internal/repository/account_job_connection_integration_test.go index 618c19554ad0..f8d7c784b026 100644 --- a/backend/internal/repository/account_job_connection_integration_test.go +++ b/backend/internal/repository/account_job_connection_integration_test.go @@ -53,7 +53,8 @@ func TestAccountJobStoppedRetryAndMonitorFailure(t *testing.T) { _, seeds, _, _, err := repo.FailedItemSeeds(ctx, job.ID, user.ID) require.NoError(t, err) require.Len(t, seeds, 1) - require.Contains(t, string(seeds[0].Metadata), `"account_id": 7`) + // The failed item was seeded with {}; account_id came from its result. + require.JSONEq(t, `{}`, string(seeds[0].Metadata)) jobs := service.NewAccountJobService(repo, nil) view, err := jobs.Get(ctx, job.ID) require.NoError(t, err) @@ -77,3 +78,65 @@ func TestAccountJobStoppedRetryAndMonitorFailure(t *testing.T) { require.NoError(t, err) require.False(t, view.RetryEligible) } + +// Error text longer than 512 characters, invalid UTF-8 and NUL characters are +// stored (as U+FFFD where PostgreSQL rejects them), and a retry is seeded with +// the original item keys only. +func TestAccountJobStoresLongErrorTextAndRetriesSeedKeys(t *testing.T) { + ctx := context.Background() + user := mustCreateUser(t, testEntClient(t), &service.User{Email: "job-error-text-" + uuid.NewString() + "@example.com", PasswordHash: "fixture"}) + repo := &accountJobRepository{db: integrationDB} + var jobID int64 + t.Cleanup(func() { + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM admin_account_jobs WHERE id=$1", jobID) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id=$1", user.ID) + }) + target := int64(7) + job, _, err := repo.Create(ctx, service.CreateAccountJobParams{ + CreatedBy: user.ID, Kind: service.AccountJobKindCodexTicketHarvest, IdempotencyKey: uuid.NewString(), + RequestHash: strings.Repeat("c", 64), PayloadCipher: "encrypted-fixture", PayloadExpires: time.Now().Add(time.Hour), + Metadata: json.RawMessage(`{"label":"a\u0000b"}`), + Items: []service.AccountJobItemSeed{ + {Ordinal: 1, TargetAccountID: &target, Metadata: json.RawMessage(`{"account_id":7,"model_id":"gpt-6"}`)}, + {Ordinal: 2, TargetAccountID: &target, Metadata: json.RawMessage(`{"account_id":7,"model_id":"gpt-6-mini"}`)}, + }, + Attempt: 1, + }) + require.NoError(t, err) + jobID = job.ID + require.JSONEq(t, `{"label":"a\uFFFDb"}`, string(job.Metadata)) + _, err = integrationDB.ExecContext(ctx, "UPDATE admin_account_jobs SET status='running' WHERE id=$1", job.ID) + require.NoError(t, err) + items, err := repo.ReservePendingItems(ctx, job.ID, 100) + require.NoError(t, err) + require.Len(t, items, 2) + + raw := strings.Repeat("x", 998) + "\xff\x00" + want := strings.Repeat("x", 998) + "\uFFFD\uFFFD" + metadata := json.RawMessage("{\"account_id\":7,\"model_id\":\"gpt-6\",\"body\":\"a\\u0000b\",\"proxy_errors\":[\"bad \xff byte\"],\"warnings\":[\"w\"]}") + require.NoError(t, repo.CompleteItems(ctx, job.ID, []service.AccountJobExecutionResult{{ + ItemID: items[0].ID, Status: service.AccountJobItemStatusFailed, ErrorCode: "ticket_upstream_error", ErrorMessage: raw, Metadata: metadata, + }})) + finished, err := repo.Finish(ctx, job.ID, service.AccountJobCodeCancelCheckFailed, raw) + require.NoError(t, err) + require.Equal(t, service.AccountJobStatusFailed, finished.Status) + require.Equal(t, want, finished.ErrorMessage) + + stored, err := repo.ListItems(ctx, job.ID, "", 1, 10) + require.NoError(t, err) + require.Len(t, stored.Items, 2) + require.Equal(t, "ticket_upstream_error", stored.Items[0].ErrorCode) + require.Equal(t, want, stored.Items[0].ErrorMessage) + var got map[string]any + require.NoError(t, json.Unmarshal(stored.Items[0].Metadata, &got)) + require.Equal(t, "a\uFFFDb", got["body"]) + require.Equal(t, []any{"bad \uFFFD byte"}, got["proxy_errors"]) + require.Equal(t, []any{"body", "proxy_errors", "warnings"}, got["result_keys"]) + require.Equal(t, want, stored.Items[1].ErrorMessage, "remaining items keep the job error text") + + _, seeds, _, _, err := repo.FailedItemSeeds(ctx, job.ID, user.ID) + require.NoError(t, err) + require.Len(t, seeds, 2) + require.JSONEq(t, `{"account_id":7,"model_id":"gpt-6"}`, string(seeds[0].Metadata)) + require.JSONEq(t, `{"account_id":7,"model_id":"gpt-6-mini"}`, string(seeds[1].Metadata)) +} diff --git a/backend/internal/repository/account_job_repo.go b/backend/internal/repository/account_job_repo.go index 33488ec91630..877426440978 100644 --- a/backend/internal/repository/account_job_repo.go +++ b/backend/internal/repository/account_job_repo.go @@ -1,6 +1,7 @@ package repository import ( + "bytes" "context" "database/sql" "encoding/json" @@ -49,6 +50,10 @@ const accountJobSelectColumns = `id, created_by, kind, idempotency_key, request_ const accountJobItemSelectColumns = `id, job_id, ordinal, action, target_account_id, status, metadata, error_code, error_message, started_at, finished_at, created_at, updated_at` +// accountJobResultKeysField is the item metadata key that lists the keys a +// completed result added to the seed metadata. +const accountJobResultKeysField = "result_keys" + type accountJobScanner interface{ Scan(...any) error } func scanAccountJob(row accountJobScanner) (*service.AccountJob, error) { @@ -181,8 +186,11 @@ func (r *accountJobRepository) Create(ctx context.Context, params service.Create return job, false, nil } +// normalizeRepositoryJobMetadata returns a jsonb object for a metadata write; +// values are kept, only text jsonb rejects becomes U+FFFD. func normalizeRepositoryJobMetadata(raw json.RawMessage) json.RawMessage { - if len(raw) == 0 || string(raw) == "null" { + raw = service.StorableAccountJobMetadata(raw) + if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" { return json.RawMessage(`{}`) } return raw @@ -438,11 +446,20 @@ func (r *accountJobRepository) CompleteItems(ctx context.Context, jobID int64, r status = service.AccountJobItemStatusFailed } metadata := normalizeRepositoryJobMetadata(result.Metadata) + // result_keys lists the keys this result added to the seed metadata, so a + // retry is seeded with the original keys only (FailedItemSeeds). updated, updateErr := tx.ExecContext(ctx, `UPDATE admin_account_job_items - SET status=$3, metadata=metadata || $4::jsonb, error_code=NULLIF($5,''), error_message=NULLIF($6,''), + SET status=$3, + metadata=metadata || $4::jsonb || COALESCE(( + SELECT jsonb_build_object('`+accountJobResultKeysField+`', jsonb_agg(result_key ORDER BY result_key)) + FROM jsonb_object_keys($4::jsonb) AS result_key + WHERE NOT (admin_account_job_items.metadata ? result_key) + HAVING COUNT(*) > 0), '{}'::jsonb), + error_code=NULLIF($5,''), error_message=NULLIF($6,''), finished_at=NOW(), updated_at=NOW() WHERE job_id=$1 AND id=$2 AND status='running'`, - jobID, result.ItemID, status, string(metadata), result.ErrorCode, result.ErrorMessage) + jobID, result.ItemID, status, string(metadata), + service.StorableAccountJobText(result.ErrorCode), service.StorableAccountJobText(result.ErrorMessage)) if updateErr != nil { return updateErr } @@ -487,6 +504,7 @@ func refreshAccountJobCounts(ctx context.Context, tx *sql.Tx, jobID int64) error } func (r *accountJobRepository) Finish(ctx context.Context, jobID int64, errorCode, errorMessage string) (*service.AccountJob, error) { + errorCode, errorMessage = service.StorableAccountJobText(errorCode), service.StorableAccountJobText(errorMessage) tx, err := r.db.BeginTx(ctx, nil) if err != nil { return nil, err @@ -591,7 +609,13 @@ func (r *accountJobRepository) FailedItemSeeds(ctx context.Context, jobID, creat if err != nil { return nil, nil, "", time.Time{}, err } - rows, err := r.db.QueryContext(ctx, `SELECT ordinal, action, target_account_id, metadata + // A retry is seeded with the original item keys; keys a previous result + // added (warnings, matched ids, proxy errors, account id, action, ...) are + // results of that attempt, not input of the next one. + rows, err := r.db.QueryContext(ctx, `SELECT ordinal, action, target_account_id, + (metadata - '`+accountJobResultKeysField+`') - ARRAY(SELECT jsonb_array_elements_text( + CASE WHEN jsonb_typeof(metadata->'`+accountJobResultKeysField+`')='array' + THEN metadata->'`+accountJobResultKeysField+`' ELSE '[]'::jsonb END)) FROM admin_account_job_items WHERE job_id=$1 AND status='failed' ORDER BY ordinal`, jobID) if err != nil { return nil, nil, "", time.Time{}, err diff --git a/backend/internal/repository/account_repo.go b/backend/internal/repository/account_repo.go index 410e9f47244a..bd630e757f77 100644 --- a/backend/internal/repository/account_repo.go +++ b/backend/internal/repository/account_repo.go @@ -74,6 +74,8 @@ var schedulerNeutralExtraKeys = map[string]struct{}{ "codex_referral_snapshot": {}, "grok_billing_snapshot": {}, "session_window_utilization": {}, + // Administrator record of the last upstream 401/403/429; scheduling never reads it. + service.OpenAILastUpstreamErrorExtraKey: {}, } const postgresParameterBatchSize = 50000 @@ -2526,6 +2528,13 @@ func (r *accountRepository) SetModelRateLimit(ctx context.Context, id int64, sco payload["reason"] = value } } + // reason[1] is the upstream's own error text, kept for administrators next + // to the reason code, which stays the only value anything matches on. + if len(reason) > 1 { + if value := strings.TrimSpace(reason[1]); value != "" { + payload[service.ModelRateLimitUpstreamMessageKey] = value + } + } raw, err := json.Marshal(payload) if err != nil { return err diff --git a/backend/internal/repository/api_key_repo.go b/backend/internal/repository/api_key_repo.go index 99f9552ae392..bdfb649cb159 100644 --- a/backend/internal/repository/api_key_repo.go +++ b/backend/internal/repository/api_key_repo.go @@ -45,6 +45,15 @@ func (r *apiKeyRepository) userVisibleQuery() *dbent.APIKeyQuery { return r.activeQuery().Where(apikey.PurposeEQ(service.APIKeyPurposeUser)) } +// listedQuery serves the key lists only administrators read: a user's keys in +// the admin console (ListByUserIDForAdmin), group keys and the usage search. +// It includes internal release-acceptance keys, which carry their purpose in +// the response. The owner's own key list and lookups used by mutations keep +// userVisibleQuery, so those keys never reach the user side and stay read-only. +func (r *apiKeyRepository) listedQuery() *dbent.APIKeyQuery { + return r.activeQuery() +} + func (r *apiKeyRepository) Create(ctx context.Context, key *service.APIKey) error { builder := r.client.APIKey.Create(). SetUserID(key.UserID). @@ -443,8 +452,12 @@ func (r *apiKeyRepository) deleteWithTombstone(ctx context.Context, exec *dbent. return nil } -func (r *apiKeyRepository) apiKeyListByUserIDQuery(userID int64, filters service.APIKeyListFilters) *dbent.APIKeyQuery { - q := r.userVisibleQuery().Where(apikey.UserIDEQ(userID)) +func (r *apiKeyRepository) apiKeyListByUserIDQuery(userID int64, filters service.APIKeyListFilters, includeInternal bool) *dbent.APIKeyQuery { + q := r.userVisibleQuery() + if includeInternal { + q = r.listedQuery() + } + q = q.Where(apikey.UserIDEQ(userID)) if filters.Search != "" { q = q.Where(apikey.Or( @@ -467,7 +480,17 @@ func (r *apiKeyRepository) apiKeyListByUserIDQuery(userID int64, filters service } func (r *apiKeyRepository) ListByUserID(ctx context.Context, userID int64, params pagination.PaginationParams, filters service.APIKeyListFilters) ([]service.APIKey, *pagination.PaginationResult, error) { - q := r.apiKeyListByUserIDQuery(userID, filters) + return r.listByUserID(ctx, userID, params, filters, false) +} + +// ListByUserIDForAdmin is the administrator's view of one user's keys: it also +// lists internal release-acceptance keys, marked by their purpose. +func (r *apiKeyRepository) ListByUserIDForAdmin(ctx context.Context, userID int64, params pagination.PaginationParams, filters service.APIKeyListFilters) ([]service.APIKey, *pagination.PaginationResult, error) { + return r.listByUserID(ctx, userID, params, filters, true) +} + +func (r *apiKeyRepository) listByUserID(ctx context.Context, userID int64, params pagination.PaginationParams, filters service.APIKeyListFilters, includeInternal bool) ([]service.APIKey, *pagination.PaginationResult, error) { + q := r.apiKeyListByUserIDQuery(userID, filters, includeInternal) total, err := q.Count(ctx) if err != nil { @@ -499,7 +522,7 @@ func (r *apiKeyRepository) ListByUserID(ctx context.Context, userID int64, param } func (r *apiKeyRepository) ListAllByUserID(ctx context.Context, userID int64, filters service.APIKeyListFilters) ([]service.APIKey, error) { - keys, err := r.apiKeyListByUserIDQuery(userID, filters). + keys, err := r.apiKeyListByUserIDQuery(userID, filters, false). WithGroup(). Order(dbent.Asc(apikey.FieldID)). All(ctx) @@ -636,7 +659,7 @@ func (r *apiKeyRepository) ExistsByKey(ctx context.Context, key string) (bool, e } func (r *apiKeyRepository) ListByGroupID(ctx context.Context, groupID int64, params pagination.PaginationParams) ([]service.APIKey, *pagination.PaginationResult, error) { - q := r.userVisibleQuery().Where(apikey.GroupIDEQ(groupID)) + q := r.listedQuery().Where(apikey.GroupIDEQ(groupID)) total, err := q.Count(ctx) if err != nil { @@ -702,7 +725,7 @@ func apiKeyListOrder(params pagination.PaginationParams) []func(*entsql.Selector // SearchAPIKeys searches API keys by user ID and/or keyword (name) func (r *apiKeyRepository) SearchAPIKeys(ctx context.Context, userID int64, keyword string, limit int) ([]service.APIKey, error) { - q := r.userVisibleQuery() + q := r.listedQuery() if userID > 0 { q = q.Where(apikey.UserIDEQ(userID)) } diff --git a/backend/internal/repository/api_key_repo_integration_test.go b/backend/internal/repository/api_key_repo_integration_test.go index b90e4ae2e329..9b5a505c6457 100644 --- a/backend/internal/repository/api_key_repo_integration_test.go +++ b/backend/internal/repository/api_key_repo_integration_test.go @@ -86,7 +86,7 @@ func (s *APIKeyRepoSuite) TestGetByKey_NotFound() { s.Require().Error(err, "expected error for non-existent key") } -func (s *APIKeyRepoSuite) TestReleaseAcceptanceKeyIsHiddenButAuthenticates() { +func (s *APIKeyRepoSuite) TestReleaseAcceptanceKeyIsAdminListedReadOnlyAndAuthenticates() { user := s.mustCreateUser("release-acceptance-key@test.com") group := s.mustCreateGroup("g-release-acceptance") leaseID := "accept-0123456789abcdef0123456789abcdef0123456789abcdef" @@ -114,9 +114,31 @@ func (s *APIKeyRepoSuite) TestReleaseAcceptanceKeyIsHiddenButAuthenticates() { service.APIKeyListFilters{}, ) s.Require().NoError(err) + // The owner's own key list never shows the internal key. s.Require().Empty(listed) s.Require().Zero(result.Total) + // Administrators see it with its purpose; it stays read-only because + // lookups used by mutations (GetByID above) do not return it. + adminListed, adminResult, err := s.repo.ListByUserIDForAdmin( + s.ctx, + user.ID, + pagination.PaginationParams{Page: 1, PageSize: 20}, + service.APIKeyListFilters{}, + ) + s.Require().NoError(err) + s.Require().Len(adminListed, 1) + s.Require().EqualValues(1, adminResult.Total) + s.Require().Equal(service.APIKeyPurposeReleaseAcceptance, adminListed[0].Purpose) + s.Require().Equal(leaseID, *adminListed[0].LeaseID) + grouped, groupResult, err := s.repo.ListByGroupID(s.ctx, group.ID, pagination.PaginationParams{Page: 1, PageSize: 20}) + s.Require().NoError(err) + s.Require().Len(grouped, 1) + s.Require().EqualValues(1, groupResult.Total) + searched, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "Release acceptance", 10) + s.Require().NoError(err) + s.Require().Len(searched, 1) + authenticated, err := s.repo.GetByKeyForAuth(s.ctx, key.Key) s.Require().NoError(err) s.Require().Equal(key.ID, authenticated.ID) diff --git a/backend/internal/repository/native_plugin_boundary.go b/backend/internal/repository/native_plugin_boundary.go index 8d7ed156ceda..0242b7a341c9 100644 --- a/backend/internal/repository/native_plugin_boundary.go +++ b/backend/internal/repository/native_plugin_boundary.go @@ -20,7 +20,9 @@ type nativePluginBoundary struct { } func ProvidePluginRepository(db *sql.DB, bootstrap *service.NativeFeatureBootstrap) service.PluginRepository { - return newNativePluginBoundary(NewPluginRepository(db), bootstrap) + boundary := newNativePluginBoundary(NewPluginRepository(db), bootstrap) + bootstrap.SetRetiredPluginSource(boundary) + return boundary } func newNativePluginBoundary(base service.PluginRepository, bootstrap *service.NativeFeatureBootstrap) *nativePluginBoundary { @@ -52,6 +54,22 @@ func (r *nativePluginBoundary) List(ctx context.Context) ([]*service.PluginInsta return visible, nil } +// RetiredPluginInstallations returns exactly the rows List hides: the retired +// first-party installations, for the read-only admin view. +func (r *nativePluginBoundary) RetiredPluginInstallations(ctx context.Context) ([]*service.PluginInstallation, error) { + all, err := r.PluginRepository.List(ctx) + if err != nil { + return nil, err + } + retired := make([]*service.PluginInstallation, 0, len(r.keys)) + for _, plugin := range all { + if plugin != nil && (r.keys[plugin.PluginKey] || r.ids[plugin.ID]) { + retired = append(retired, plugin) + } + } + return retired, nil +} + func (r *nativePluginBoundary) GetByID(ctx context.Context, id int64) (*service.PluginInstallation, error) { if r.ids[id] { return nil, sql.ErrNoRows diff --git a/backend/internal/repository/native_plugin_boundary_test.go b/backend/internal/repository/native_plugin_boundary_test.go index 5b72e158d288..0c65a0c76627 100644 --- a/backend/internal/repository/native_plugin_boundary_test.go +++ b/backend/internal/repository/native_plugin_boundary_test.go @@ -61,6 +61,9 @@ func TestNativePluginBoundaryProtectsRetiredRecordsAndAllowsThirdParties(t *test require.NoError(t, err) require.Len(t, list, 1) require.EqualValues(t, 100, list[0].ID) + retired, err := r.RetiredPluginInstallations(ctx) + require.NoError(t, err) + require.Len(t, retired, len(service.FirstPartyNativePluginKeys), "the read-only view lists exactly the rows the managed list hides") for id, p := range base.rows { if id == 100 { continue diff --git a/backend/internal/repository/ops_error_where_test.go b/backend/internal/repository/ops_error_where_test.go index af9af28de825..9179cf794b9f 100644 --- a/backend/internal/repository/ops_error_where_test.go +++ b/backend/internal/repository/ops_error_where_test.go @@ -151,3 +151,13 @@ func TestBuildOpsErrorLogsWhere_UserOwnershipIsDirectOnly(t *testing.T) { t.Fatalf("user ownership must not depend on deleted-key attribution: %s", where) } } + +func TestBuildOpsErrorLogsWhere_ExcludesClientRequestIDPrefix(t *testing.T) { + where, args := buildOpsErrorLogsWhere(&service.OpsErrorLogFilter{ExcludeClientRequestIDPrefix: service.ImageStudioClientRequestIDPrefix}) + if !strings.Contains(where, "COALESCE(e.client_request_id,'') NOT LIKE $1") { + t.Fatalf("where missing the client request ID exclusion\nfull: %s", where) + } + if len(args) != 1 || args[0] != "image-studio-%" { + t.Fatalf("unexpected args: %v", args) + } +} diff --git a/backend/internal/repository/ops_repo.go b/backend/internal/repository/ops_repo.go index 11f427a00028..c167593beaf0 100644 --- a/backend/internal/repository/ops_repo.go +++ b/backend/internal/repository/ops_repo.go @@ -953,6 +953,10 @@ func buildOpsErrorLogsWhere(filter *service.OpsErrorLogFilter) (string, []any) { args = append(args, crid) clauses = append(clauses, "COALESCE(e.client_request_id,'') = $"+itoa(len(args))) } + if prefix := strings.TrimSpace(filter.ExcludeClientRequestIDPrefix); prefix != "" { + args = append(args, escapeLikePattern(prefix)+"%") + clauses = append(clauses, "COALESCE(e.client_request_id,'') NOT LIKE $"+itoa(len(args))) + } if q := strings.TrimSpace(filter.Query); q != "" { like := "%" + q + "%" diff --git a/backend/internal/repository/scheduler_cache.go b/backend/internal/repository/scheduler_cache.go index 977f794b9989..5ae5c4fe9f6c 100644 --- a/backend/internal/repository/scheduler_cache.go +++ b/backend/internal/repository/scheduler_cache.go @@ -1138,6 +1138,12 @@ func filterSchedulerExtra(extra map[string]any) map[string]any { } value = filteredProbe } + if key == service.NativeCodexAccountProjectionKey { + value = filterSchedulerCodexProjection(value) + } + if key == "model_rate_limits" { + value = filterSchedulerModelRateLimits(value) + } filtered[key] = value } } @@ -1147,6 +1153,84 @@ func filterSchedulerExtra(extra map[string]any) map[string]any { return filtered } +// filterSchedulerCodexProjection copies the Codex account projection without +// the administrator-only observation text (message, response_model); the +// scheduler reads only the constraints and the observation state. +func filterSchedulerCodexProjection(value any) any { + plugins, ok := value.(map[string]any) + if !ok { + return value + } + filtered := make(map[string]any, len(plugins)) + for plugin, raw := range plugins { + projection, ok := raw.(map[string]any) + observations, hasObservations := projection["observations"].(map[string]any) + if !ok || !hasObservations { + filtered[plugin] = raw + continue + } + copied := make(map[string]any, len(projection)) + for key, field := range projection { + copied[key] = field + } + kept := make(map[string]any, len(observations)) + for model, rawObservation := range observations { + observation, ok := rawObservation.(map[string]any) + if !ok { + kept[model] = rawObservation + continue + } + trimmed := make(map[string]any, len(observation)) + for key, field := range observation { + if key != "message" && key != "response_model" { + trimmed[key] = field + } + } + kept[model] = trimmed + } + copied["observations"] = kept + filtered[plugin] = copied + } + return filtered +} + +// filterSchedulerModelRateLimits drops the administrator-only upstream text +// from each cooldown entry without touching the account's own map; scheduling +// reads only the reset time and the reason. +func filterSchedulerModelRateLimits(value any) any { + limits, ok := value.(map[string]any) + if !ok { + return value + } + var out map[string]any + for scope, raw := range limits { + entry, ok := raw.(map[string]any) + if !ok { + continue + } + if _, has := entry[service.ModelRateLimitUpstreamMessageKey]; !has { + continue + } + if out == nil { + out = make(map[string]any, len(limits)) + for k, v := range limits { + out[k] = v + } + } + trimmed := make(map[string]any, len(entry)) + for k, v := range entry { + if k != service.ModelRateLimitUpstreamMessageKey { + trimmed[k] = v + } + } + out[scope] = trimmed + } + if out == nil { + return value + } + return out +} + func filterSchedulerUpstreamBillingProbe(value any) map[string]any { source, ok := value.(map[string]any) if !ok { diff --git a/backend/internal/repository/scheduler_cache_reasoning_policy_test.go b/backend/internal/repository/scheduler_cache_reasoning_policy_test.go index 8a0c92b2a3a2..c483317463e7 100644 --- a/backend/internal/repository/scheduler_cache_reasoning_policy_test.go +++ b/backend/internal/repository/scheduler_cache_reasoning_policy_test.go @@ -10,6 +10,16 @@ import ( "github.com/stretchr/testify/require" ) +func TestSchedulerCacheCodexProjectionDropsAdministratorText(t *testing.T) { + observation := map[string]any{"key": "gpt-6-astra", "kind": "codex_routing", "state": "retry", "code": "routing_upstream", "http_status": float64(429), "message": "Rate limit reached", "response_model": "gpt-6-luna"} + projection := map[string]any{"identity": "owner", "scheduling": map[string]any{}, "observations": map[string]any{"gpt-6-astra": observation}} + source := map[string]any{service.NativeCodexAccountProjectionKey: map[string]any{"codexrip.codex-runtime": projection}} + filtered := filterSchedulerExtra(source) + copied := filtered[service.NativeCodexAccountProjectionKey].(map[string]any)["codexrip.codex-runtime"].(map[string]any)["observations"].(map[string]any)["gpt-6-astra"].(map[string]any) + require.Equal(t, map[string]any{"key": "gpt-6-astra", "kind": "codex_routing", "state": "retry", "code": "routing_upstream", "http_status": float64(429)}, copied) + require.Equal(t, "Rate limit reached", observation["message"], "the persistent account extra is not rewritten") +} + func TestSchedulerCacheReasoningPolicyProjectionPreservesMissingAndBooleanSemantics(t *testing.T) { keys := []string{service.OpenAIChatReasoningReplayEnabledExtraKey, service.OpenAIReasoningSignatureRecoveryEnabledExtraKey} for _, source := range []map[string]any{nil, {}, {"unrelated": true}, {"quota_limit": nil}} { diff --git a/backend/internal/repository/scheduler_cache_unit_test.go b/backend/internal/repository/scheduler_cache_unit_test.go index ca0aec8fe73d..53d73836760a 100644 --- a/backend/internal/repository/scheduler_cache_unit_test.go +++ b/backend/internal/repository/scheduler_cache_unit_test.go @@ -681,8 +681,14 @@ func TestBuildSchedulerMetadataAccount_KeepsModelRateLimits(t *testing.T) { "antigravity:gemini": map[string]any{ "rate_limit_reset_at": "2026-05-30T10:10:00Z", }, + "gpt-5.6-luna": map[string]any{ + "rate_limit_reset_at": "2026-05-30T10:10:00Z", + "reason": "upstream_400_model_not_supported", + service.ModelRateLimitUpstreamMessageKey: "Model 'gpt-5.6-luna' is temporarily not supported.", + }, }, - "unused_large_field": "drop-me", + service.OpenAILastUpstreamErrorExtraKey: map[string]any{"status": 429, "message": "slow down"}, + "unused_large_field": "drop-me", }, } @@ -693,6 +699,14 @@ func TestBuildSchedulerMetadataAccount_KeepsModelRateLimits(t *testing.T) { require.Contains(t, limits, "gemini-3-flash") require.Contains(t, limits, "antigravity:gemini") require.Nil(t, got.Extra["unused_large_field"]) + // Administrator-only text stays out of the scheduler snapshot. + require.NotContains(t, got.Extra, service.OpenAILastUpstreamErrorExtraKey) + luna, ok := limits["gpt-5.6-luna"].(map[string]any) + require.True(t, ok) + require.Equal(t, "upstream_400_model_not_supported", luna["reason"]) + require.NotContains(t, luna, service.ModelRateLimitUpstreamMessageKey) + source := account.Extra["model_rate_limits"].(map[string]any)["gpt-5.6-luna"].(map[string]any) + require.Contains(t, source, service.ModelRateLimitUpstreamMessageKey, "the account's own map is not modified") } func TestBuildSchedulerMetadataAccount_KeepsSparkShadowRoutingIdentity(t *testing.T) { diff --git a/backend/internal/server/api_contract_test.go b/backend/internal/server/api_contract_test.go index 3eceaae9b533..9b8415650839 100644 --- a/backend/internal/server/api_contract_test.go +++ b/backend/internal/server/api_contract_test.go @@ -1029,6 +1029,7 @@ func TestAPIContracts(t *testing.T) { "wechat_connect_redirect_url": "", "wechat_connect_frontend_redirect_url": "/auth/wechat/callback", "wechat_connect_scopes": "snsapi_login", + "deprecated_settings": {"openai_apikey_alpha_search_responses_bridge_enabled": false, "openai_apikey_prompt_cache_key_normalization_enabled": false}, "allow_user_view_error_requests": false } }`, @@ -1389,6 +1390,7 @@ func TestAPIContracts(t *testing.T) { "auth_source_default_dingtalk_grant_on_signup": false, "auth_source_default_dingtalk_grant_on_first_bind": false, "force_email_on_third_party_signup": false, + "deprecated_settings": {"openai_apikey_alpha_search_responses_bridge_enabled": false, "openai_apikey_prompt_cache_key_normalization_enabled": false}, "allow_user_view_error_requests": false } }`, diff --git a/backend/internal/server/middleware/audit_log.go b/backend/internal/server/middleware/audit_log.go index aebd7aef0a12..9651982f7816 100644 --- a/backend/internal/server/middleware/audit_log.go +++ b/backend/internal/server/middleware/audit_log.go @@ -3,6 +3,7 @@ package middleware import ( "bytes" "io" + "strconv" "strings" "time" @@ -150,26 +151,27 @@ var auditActionOverrides = map[string]string{ // auditBodyOmittedRoutes 请求体几乎整体由凭证构成的路由（如整块粘贴 auth JSON 的导入接口）。 // 这类 body 的凭证内嵌在普通字符串值里，键级脱敏无法覆盖，整体不入库。 var auditBodyOmittedRoutes = map[string]struct{}{ - "POST /api/v1/auth/passkey/login/finish": {}, - "POST /api/v1/user/passkeys/register/finish": {}, - "POST /api/v1/admin/accounts/import/codex-session": {}, - "PUT /api/v1/admin/accounts/:id/ollama-cloud-usage/session": {}, - "PUT /api/v1/admin/prompt-audit/config": {}, - "POST /api/v1/admin/prompt-audit/endpoints/probe": {}, + "POST /api/v1/auth/passkey/login/finish": {}, + "POST /api/v1/user/passkeys/register/finish": {}, + "POST /api/v1/admin/accounts/import/codex-session": {}, + "PUT /api/v1/admin/accounts/:id/ollama-cloud-usage/session": {}, + "PUT /api/v1/admin/prompt-audit/config": {}, + "POST /api/v1/admin/prompt-audit/endpoints/probe": {}, + "DELETE /api/v1/admin/prompt-audit/events/:id": {}, + "POST /api/v1/admin/prompt-audit/events/batch-delete": {}, + "POST /api/v1/admin/prompt-audit/events/delete-preview": {}, + "POST /api/v1/admin/prompt-audit/events/delete-by-filter": {}, +} + +// auditVerbatimBodyRoutes store the request body exactly as received, up to +// the capture limit, without key-level redaction: administrators review the +// full system prompt library and the Codex proxy and runtime settings they +// submitted, proxy credentials included. +var auditVerbatimBodyRoutes = map[string]struct{}{ + "PUT /api/v1/admin/system-prompts": {}, "POST /api/v1/admin/settings/openai-codex-ticket/proxy-parse": {}, "POST /api/v1/admin/settings/openai-codex-ticket/proxy-test": {}, "PUT /api/v1/admin/settings/codex-runtime": {}, - "DELETE /api/v1/admin/prompt-audit/events/:id": {}, - "POST /api/v1/admin/prompt-audit/events/batch-delete": {}, - "POST /api/v1/admin/prompt-audit/events/delete-preview": {}, - "POST /api/v1/admin/prompt-audit/events/delete-by-filter": {}, -} - -// System prompt bodies are business-sensitive and may be much larger than -// ordinary admin payloads. They are omitted entirely; the handler attaches -// only allowlisted flags. -var auditPromptBodyOmittedRoutes = map[string]struct{}{ - "PUT /api/v1/admin/system-prompts": {}, } // NewAuditLogMiddleware 创建审计中间件。 @@ -203,9 +205,7 @@ func NewAuditLogMiddleware(auditService *service.AuditLogService) AuditLogMiddle // 只读取脱敏解析上限内的字节，超出部分与已读部分拼接回填， // 避免大体积导入请求被完整复制进内存两次。 var bodyRedacted string - if _, omit := auditPromptBodyOmittedRoutes[routeKey]; omit { - bodyRedacted = "" - } else if _, omit := auditBodyOmittedRoutes[routeKey]; omit { + if _, omit := auditBodyOmittedRoutes[routeKey]; omit { bodyRedacted = "" } else if c.Request.Body != nil && c.Request.Method != "GET" { orig := c.Request.Body @@ -215,7 +215,11 @@ func NewAuditLogMiddleware(auditService *service.AuditLogService) AuditLogMiddle Reader: io.MultiReader(bytes.NewReader(raw), orig), closer: orig, } - bodyRedacted = service.RedactAuditBody(raw, c.GetHeader("Content-Type")) + if _, verbatim := auditVerbatimBodyRoutes[routeKey]; verbatim { + bodyRedacted = auditVerbatimBody(raw) + } else { + bodyRedacted = service.RedactAuditBody(raw, c.GetHeader("Content-Type")) + } } } @@ -311,6 +315,21 @@ func NewAuditLogMiddleware(auditService *service.AuditLogService) AuditLogMiddle }) } +// auditVerbatimBody keeps a captured body as received. A body beyond the +// capture limit keeps its first AuditRequestBodyCaptureLimit bytes and says so. +// Invalid UTF-8 and NUL characters, which a text column rejects, become U+FFFD. +func auditVerbatimBody(raw []byte) string { + if len(raw) > service.AuditRequestBodyCaptureLimit { + return auditStorableText(raw[:service.AuditRequestBodyCaptureLimit]) + + "..." + } + return auditStorableText(raw) +} + +func auditStorableText(raw []byte) string { + return strings.ReplaceAll(strings.ToValidUTF8(string(raw), "\uFFFD"), "\x00", "\uFFFD") +} + // restoredBody 把审计中间件按上限读出的前缀与未读完的原始 body 拼接回填， // 保证 handler 读到完整请求体；Close 委托给原始 body。 type restoredBody struct { diff --git a/backend/internal/server/middleware/audit_log_test.go b/backend/internal/server/middleware/audit_log_test.go index d8a633862e14..7b734404ee6a 100644 --- a/backend/internal/server/middleware/audit_log_test.go +++ b/backend/internal/server/middleware/audit_log_test.go @@ -3,7 +3,6 @@ package middleware import ( "bytes" "context" - "encoding/json" "io" "net/http" "net/http/httptest" @@ -149,14 +148,16 @@ func TestPromptAuditMutationAuditRoutesHaveStableActionsAndOmitBodies(t *testing } } -func TestSystemPromptAuditRoutesHaveStableActionsAndOmitPromptBodies(t *testing.T) { +func TestSystemPromptAuditRoutesHaveStableActionsAndKeepPromptBodies(t *testing.T) { require.Equal(t, "admin.system_prompts.read", auditSensitiveReads["GET /api/v1/admin/system-prompts"]) require.Equal(t, "admin.system_prompts.update", auditActionOverrides["PUT /api/v1/admin/system-prompts"]) require.Equal(t, "admin.system_prompts.bindings.update", auditActionOverrides["PUT /api/v1/admin/system-prompts/bindings"]) - require.Contains(t, auditPromptBodyOmittedRoutes, "PUT /api/v1/admin/system-prompts") + require.Contains(t, auditVerbatimBodyRoutes, "PUT /api/v1/admin/system-prompts") } -func TestSystemPromptConfigAuditOmitsPromptBodies(t *testing.T) { +// Administrators review what a system prompt save changed, so the audit row +// keeps the submitted body verbatim. Extra fields remain the allowlisted flags. +func TestSystemPromptConfigAuditStoresPromptBodiesVerbatim(t *testing.T) { gin.SetMode(gin.TestMode) repository := &auditCaptureRepository{} auditService := service.NewAuditLogService(repository, nil) @@ -194,14 +195,11 @@ func TestSystemPromptConfigAuditOmitsPromptBodies(t *testing.T) { repository.mu.Unlock() require.Len(t, logs, 1) require.Equal(t, "admin.system_prompts.update", logs[0].Action) - require.Equal(t, "", logs[0].RequestBody) + require.Equal(t, payload, logs[0].RequestBody) require.Equal(t, true, logs[0].Extra["enabled"]) for _, field := range []string{"body", "contents"} { require.NotContains(t, logs[0].Extra, field) } - encoded, err := json.Marshal(logs[0]) - require.NoError(t, err) - require.NotContains(t, string(encoded), "audit-canary") } func TestPasskeyLoginAuditUsesCanonicalLoginActionAndOmitsCredentialBody(t *testing.T) { @@ -210,7 +208,9 @@ func TestPasskeyLoginAuditUsesCanonicalLoginActionAndOmitsCredentialBody(t *test require.Contains(t, auditBodyOmittedRoutes, route) } -func TestCodexProxyDraftRoutesOmitCredentialBearingAuditBodies(t *testing.T) { +// The Codex proxy draft and runtime saves are audited as submitted, proxy +// credentials included, so administrators can see which proxy was entered. +func TestCodexProxyDraftRoutesStoreAuditBodiesVerbatim(t *testing.T) { gin.SetMode(gin.TestMode) for _, route := range []struct{ method, path string }{ {http.MethodPost, "/api/v1/admin/settings/openai-codex-ticket/proxy-parse"}, @@ -240,10 +240,7 @@ func TestCodexProxyDraftRoutesOmitCredentialBearingAuditBodies(t *testing.T) { logs := append([]*service.AuditLog(nil), repository.logs...) repository.mu.Unlock() require.Len(t, logs, 1) - require.Equal(t, "", logs[0].RequestBody) - encoded, err := json.Marshal(logs[0]) - require.NoError(t, err) - require.NotContains(t, string(encoded), "audit-canary") + require.Equal(t, body, logs[0].RequestBody) }) } } @@ -284,3 +281,9 @@ func TestOllamaCloudUsageSessionRouteOmitsAuditBody(t *testing.T) { require.Equal(t, "", logs[0].RequestBody) require.NotContains(t, logs[0].RequestBody, "audit-canary") } + +// Text columns reject NUL characters and invalid UTF-8; the stored body keeps +// every other byte of the request. +func TestAuditVerbatimBodyReplacesNULAndInvalidUTF8(t *testing.T) { + require.Equal(t, "{\"a\":\"x\uFFFDy\uFFFD\"}", auditVerbatimBody([]byte("{\"a\":\"x\x00y\xff\"}"))) +} diff --git a/backend/internal/server/middleware/security_headers_test.go b/backend/internal/server/middleware/security_headers_test.go index cdd169c15404..cbc539ac2016 100644 --- a/backend/internal/server/middleware/security_headers_test.go +++ b/backend/internal/server/middleware/security_headers_test.go @@ -380,6 +380,12 @@ func TestEnhanceCSPPolicy(t *testing.T) { assert.Equal(t, 1, countDirectiveValue(enhanced, "worker-src", TencentCaptchaWorkerSource)) }) + t.Run("default_policy_allows_the_misans_font_service", func(t *testing.T) { + // The console theme loads MiSans from Xiaomi's official font service (stylesheet + font files). + assert.Equal(t, 1, countDirectiveValue(config.DefaultCSPPolicy, "style-src", "https://font.sec.miui.com")) + assert.Equal(t, 1, countDirectiveValue(config.DefaultCSPPolicy, "font-src", "https://cdn-file.hyperos.mi.com")) + }) + t.Run("default_policy_already_carries_tencent_captcha_domains", func(t *testing.T) { // 默认策略与中间件强制注入表必须同形，否则 config.example.yaml 会误导自建用户 for _, required := range requiredCSPDirectiveValues { diff --git a/backend/internal/server/routes/admin.go b/backend/internal/server/routes/admin.go index a768cffd462b..74770c2ef28b 100644 --- a/backend/internal/server/routes/admin.go +++ b/backend/internal/server/routes/admin.go @@ -830,6 +830,7 @@ func registerPluginRoutes(admin *gin.RouterGroup, h *handler.Handlers, stepUpAut plugins := admin.Group("/plugins") { plugins.GET("", h.Admin.Plugin.List) + plugins.GET("/retired", h.Admin.Plugin.ListRetired) plugins.GET("/:id", h.Admin.Plugin.Get) plugins.POST("/upload", gin.HandlerFunc(stepUpAuth), h.Admin.Plugin.Upload) plugins.POST("/:id/enable", gin.HandlerFunc(stepUpAuth), h.Admin.Plugin.Enable) diff --git a/backend/internal/service/account_business_errors.go b/backend/internal/service/account_business_errors.go index fe8321518614..e1825ed56a0b 100644 --- a/backend/internal/service/account_business_errors.go +++ b/backend/internal/service/account_business_errors.go @@ -58,8 +58,8 @@ var accountBusinessMessageCatalog = map[string]accountBusinessMessage{ "import_failed": {message: "account import failed", failure: true}, } -// AccountBusinessMessage returns a fixed, credential-safe business message. -// Callers must not fall back to raw errors when the code is unknown. +// AccountBusinessMessage returns the catalog sentence for a code. The sentence +// only describes a result that has no underlying error text of its own. func AccountBusinessMessage(code string) (string, bool) { if strings.HasPrefix(code, "ticket_") { r := CodexTicketFailure(code) @@ -71,17 +71,34 @@ func AccountBusinessMessage(code string) (string, bool) { return entry.message, ok } -// NormalizeAccountBusinessFailure admits only catalog entries that are safe -// failure results. Preview success codes are deliberately rejected here. -func NormalizeAccountBusinessFailure(code string) (string, string) { +// AccountJobFailure returns the code and message stored for a failed account +// job item or job. The reported code is kept as is (execution_failed when it is +// empty) and a non-empty message is the verbatim underlying error text, which +// is never replaced. Only a failure without underlying text gets the catalog +// sentence of its code, or the generic item sentence for codes without one. +func AccountJobFailure(code, message string) (string, string) { code = strings.TrimSpace(code) - if strings.HasPrefix(code, "ticket_") { - r := CodexTicketFailure(code) - return r.Code, r.Message + if code == "" { + code = AccountJobCodeExecutionFailed + } + if strings.TrimSpace(message) != "" { + return code, message + } + if strings.HasPrefix(code, "ticket_") || strings.HasPrefix(code, "routing_") { + if r := CodexTicketFailure(code); r.Code == code { + return code, r.Message + } } if entry, ok := accountBusinessMessageCatalog[code]; ok && entry.failure { return code, entry.message } - fallback := accountBusinessMessageCatalog[AccountJobCodeExecutionFailed] - return AccountJobCodeExecutionFailed, fallback.message + return code, accountBusinessMessageCatalog[AccountJobCodeExecutionFailed].message +} + +// NormalizeAccountBusinessFailure returns the code and sentence of a failure +// without underlying error text. Every reported code is kept, including +// routing_* and unknown ticket_* codes; only an empty code becomes +// execution_failed. +func NormalizeAccountBusinessFailure(code string) (string, string) { + return AccountJobFailure(code, "") } diff --git a/backend/internal/service/account_business_errors_test.go b/backend/internal/service/account_business_errors_test.go index 2844bc627cf2..cecf85cf350c 100644 --- a/backend/internal/service/account_business_errors_test.go +++ b/backend/internal/service/account_business_errors_test.go @@ -13,16 +13,36 @@ func TestAccountBusinessMessageCatalogSeparatesPreviewAndFailureCodes(t *testing require.True(t, ok) require.Equal(t, "account will be created", message) - code, message := NormalizeAccountBusinessFailure(AccountImportCodeIdentityConflict) + code, message := AccountJobFailure(AccountImportCodeIdentityConflict, "") require.Equal(t, AccountImportCodeIdentityConflict, code) require.Equal(t, "account identity matches multiple existing accounts", message) - code, message = NormalizeAccountBusinessFailure("delete_failed") + code, message = AccountJobFailure("delete_failed", "") require.Equal(t, "delete_failed", code) require.Equal(t, "account could not be deleted", message) - for _, unsafe := range []string{AccountImportCodeCreate, AccountImportCodeUpdate, "unknown", ""} { - code, message = NormalizeAccountBusinessFailure(unsafe) - require.Equal(t, AccountJobCodeExecutionFailed, code) + // The underlying error text is kept verbatim; the catalog sentence is only + // the fallback for failures without one. Reported codes are never renamed. + code, message = AccountJobFailure("delete_failed", "pq: account 7 is referenced by usage_logs") + require.Equal(t, "delete_failed", code) + require.Equal(t, "pq: account 7 is referenced by usage_logs", message) + // Codex ticket and routing codes use the Codex result sentences; an + // unknown one names itself instead of becoming a generic failure. + for reported, sentence := range map[string]string{ + "routing_capacity": "上游容量不足或流内限流，未完成路由验证", + "ticket_not_in_catalog": "未归类的结果代码：ticket_not_in_catalog", + } { + code, message = NormalizeAccountBusinessFailure(reported) + require.Equal(t, reported, code) + require.Equal(t, sentence, message) + } + + // Preview success codes never lend their success sentence to a failure. + for _, previewCode := range []string{AccountImportCodeCreate, AccountImportCodeUpdate} { + code, message = AccountJobFailure(previewCode, "") + require.Equal(t, previewCode, code) require.Equal(t, "account job item failed", message) } + code, message = AccountJobFailure("", "") + require.Equal(t, AccountJobCodeExecutionFailed, code) + require.Equal(t, "account job item failed", message) } diff --git a/backend/internal/service/account_job.go b/backend/internal/service/account_job.go index e12ad5443a2a..40bad4844acf 100644 --- a/backend/internal/service/account_job.go +++ b/backend/internal/service/account_job.go @@ -1,14 +1,18 @@ package service import ( + "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" + "fmt" + "log/slog" "strings" "sync" "time" + "unicode/utf8" ) const ( @@ -416,10 +420,21 @@ type AccountJobRuntime struct { stopOnce sync.Once wg sync.WaitGroup concurrentSlots chan struct{} + + finishRetryDelay time.Duration + unfinishedMu sync.Mutex + unfinished map[int64]accountJobFinishResult +} + +// accountJobFinishResult is a job result whose Finish write failed; the +// cleanup loop keeps writing it while the runtime runs. +type accountJobFinishResult struct { + code string + message string } func NewAccountJobRuntime(jobs *AccountJobService, executor AccountJobExecutor) *AccountJobRuntime { - return &AccountJobRuntime{jobs: jobs, executor: executor, concurrentSlots: make(chan struct{}, 5)} + return &AccountJobRuntime{jobs: jobs, executor: executor, concurrentSlots: make(chan struct{}, 5), finishRetryDelay: 500 * time.Millisecond} } func (r *AccountJobRuntime) Start(parent context.Context) error { @@ -473,25 +488,106 @@ func (r *AccountJobRuntime) runOne() bool { return false } code, message := r.execute(job) - finishCtx, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 10*time.Second) - defer cancel() - _, _ = r.jobs.repo.Finish(finishCtx, job.ID, code, message) + r.finish(job.ID, code, message) return true } +const ( + accountJobFinishAttempts = 3 + accountJobFallbackMessageRune = 500 +) + +// finish stores the job result. A failed write is logged and retried with a +// shortened storable copy of the message, so the text of a result can never +// keep a job running. A result that still cannot be written is retried by the +// cleanup loop while the runtime runs; after a restart the job is marked +// interrupted. +func (r *AccountJobRuntime) finish(jobID int64, code, message string) { + delay := r.finishRetryDelay + for attempt := 1; ; attempt++ { + if r.tryFinish(jobID, code, message, attempt) { + return + } + message = accountJobFallbackMessage(message) + if attempt >= accountJobFinishAttempts || r.ctx.Err() != nil { + r.unfinishedMu.Lock() + if r.unfinished == nil { + r.unfinished = make(map[int64]accountJobFinishResult) + } + r.unfinished[jobID] = accountJobFinishResult{code: code, message: message} + r.unfinishedMu.Unlock() + return + } + timer := time.NewTimer(delay) + select { + case <-timer.C: + case <-r.ctx.Done(): + timer.Stop() + } + delay *= 2 + } +} + +func (r *AccountJobRuntime) tryFinish(jobID int64, code, message string, attempt int) bool { + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 10*time.Second) + defer cancel() + _, err := r.jobs.repo.Finish(ctx, jobID, code, message) + if err == nil || errors.Is(err, ErrAccountJobNotFound) { + return true + } + slog.Error("account_job.finish_failed", "job_id", jobID, "attempt", attempt, "error_code", code, "error", err.Error()) + return false +} + +// retryUnfinished writes the results whose Finish failed earlier. +func (r *AccountJobRuntime) retryUnfinished() { + r.unfinishedMu.Lock() + pending := make(map[int64]accountJobFinishResult, len(r.unfinished)) + for jobID, result := range r.unfinished { + pending[jobID] = result + } + r.unfinishedMu.Unlock() + for jobID, result := range pending { + if r.tryFinish(jobID, result.code, result.message, 0) { + r.unfinishedMu.Lock() + delete(r.unfinished, jobID) + r.unfinishedMu.Unlock() + } + } +} + +// accountJobFallbackMessage keeps the start of a message in a form that any +// error_message column accepts. +func accountJobFallbackMessage(message string) string { + message = StorableAccountJobText(message) + if utf8.RuneCountInString(message) <= accountJobFallbackMessageRune { + return message + } + runes := []rune(message) + return string(runes[:accountJobFallbackMessageRune]) + "..." +} + func (r *AccountJobRuntime) execute(job *AccountJob) (string, string) { ciphertext, expires, err := r.jobs.repo.Payload(r.ctx, job.ID) - if err != nil || ciphertext == "" || time.Now().UTC().After(expires) { - return normalizeAccountJobFailure("payload_expired") + switch { + case err != nil: + return accountJobRuntimeFailure(AccountJobCodePayloadExpired, err) + case ciphertext == "": + return accountJobRuntimeFailure(AccountJobCodePayloadExpired, nil) + case time.Now().UTC().After(expires): + return accountJobRuntimeFailure(AccountJobCodePayloadExpired, fmt.Errorf("account job payload expired at %s", expires.UTC().Format(time.RFC3339))) } plaintext, err := r.jobs.encryptor.Decrypt(ciphertext) - if err != nil || !json.Valid([]byte(plaintext)) { - return normalizeAccountJobFailure("payload_unavailable") + if err == nil && !json.Valid([]byte(plaintext)) { + err = errors.New("decrypted account job payload is not valid JSON") + } + if err != nil { + return accountJobRuntimeFailure(AccountJobCodePayloadUnavailable, err) } payload := json.RawMessage(plaintext) canceled, cancelErr := r.jobs.repo.CancelRequested(r.ctx, job.ID) if cancelErr != nil { - return normalizeAccountJobFailure("cancel_check_failed") + return accountJobRuntimeFailure(AccountJobCodeCancelCheckFailed, cancelErr) } if canceled { return "", "" @@ -501,7 +597,7 @@ func (r *AccountJobRuntime) execute(job *AccountJob) (string, string) { if preparer, ok := r.executor.(AccountJobPreparingExecutor); ok { preparedCtx, preparedCleanup, prepareErr := preparer.PrepareAccountJob(r.ctx, job, payload) if prepareErr != nil { - return normalizeAccountJobFailure("preparation_failed") + return accountJobRuntimeFailure("preparation_failed", prepareErr) } if preparedCtx != nil { executionCtx = preparedCtx @@ -521,14 +617,14 @@ func (r *AccountJobRuntime) execute(job *AccountJob) (string, string) { } canceled, cancelErr := r.jobs.repo.CancelRequested(r.ctx, job.ID) if cancelErr != nil { - return normalizeAccountJobFailure("cancel_check_failed") + return accountJobRuntimeFailure(AccountJobCodeCancelCheckFailed, cancelErr) } if canceled { return "", "" } items, reserveErr := r.jobs.repo.ReservePendingItems(r.ctx, job.ID, AccountJobBatchSize) if reserveErr != nil { - return normalizeAccountJobFailure("item_reservation_failed") + return accountJobRuntimeFailure(AccountJobCodeReservationFailed, reserveErr) } if len(items) == 0 { return "", "" @@ -539,7 +635,7 @@ func (r *AccountJobRuntime) execute(job *AccountJob) (string, string) { } canceled, cancelErr = r.jobs.repo.CancelRequested(r.ctx, job.ID) if cancelErr != nil { - return normalizeAccountJobFailure("cancel_check_failed") + return accountJobRuntimeFailure(AccountJobCodeCancelCheckFailed, cancelErr) } if canceled { remaining := make([]AccountJobExecutionResult, 0, len(items)-index) @@ -551,7 +647,7 @@ func (r *AccountJobRuntime) execute(job *AccountJob) (string, string) { } result := r.executeItem(executionCtx, job, payload, item) if completeErr := r.jobs.repo.CompleteItems(r.ctx, job.ID, []AccountJobExecutionResult{result}); completeErr != nil { - return normalizeAccountJobFailure("item_completion_failed") + return accountJobRuntimeFailure(AccountJobCodeCompletionFailed, completeErr) } } } @@ -573,6 +669,7 @@ func (r *AccountJobRuntime) cleanupWorker() { } func (r *AccountJobRuntime) cleanup(now time.Time) { + r.retryUnfinished() ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second) defer cancel() _ = r.jobs.repo.ExpirePayloads(ctx, now) @@ -585,10 +682,14 @@ func (r *AccountJobRuntime) cleanup(now time.Time) { func (r *AccountJobRuntime) executeConcurrently(parent context.Context, job *AccountJob, payload json.RawMessage) (string, string) { ctx, cancel := context.WithCancel(parent) defer cancel() - failures := make(chan string, 1) - fail := func(code string) { + type runtimeFailure struct { + code string + err error + } + failures := make(chan runtimeFailure, 1) + fail := func(code string, err error) { select { - case failures <- code: + case failures <- runtimeFailure{code: code, err: err}: default: } cancel() @@ -606,7 +707,7 @@ func (r *AccountJobRuntime) executeConcurrently(parent context.Context, job *Acc requested, err := r.jobs.repo.CancelRequested(ctx, job.ID) if err != nil { if ctx.Err() == nil { - fail("cancel_check_failed") + fail(AccountJobCodeCancelCheckFailed, err) } return } @@ -622,7 +723,7 @@ func (r *AccountJobRuntime) executeConcurrently(parent context.Context, job *Acc items, err := r.jobs.repo.ReservePendingItems(ctx, job.ID, AccountJobBatchSize) if err != nil { if ctx.Err() == nil { - fail("item_reservation_failed") + fail(AccountJobCodeReservationFailed, err) } break } @@ -650,7 +751,7 @@ func (r *AccountJobRuntime) executeConcurrently(parent context.Context, job *Acc case <-ctx.Done(): } if err := r.jobs.repo.CompleteItems(r.ctx, job.ID, []AccountJobExecutionResult{result}); err != nil { - fail("item_completion_failed") + fail(AccountJobCodeCompletionFailed, err) return } } @@ -661,34 +762,121 @@ func (r *AccountJobRuntime) executeConcurrently(parent context.Context, job *Acc cancel() <-monitorDone select { - case code := <-failures: - return normalizeAccountJobFailure(code) + case failure := <-failures: + return accountJobRuntimeFailure(failure.code, failure.err) default: return "", "" } } +// executeItem stores what the executor reported. A failed item keeps its code +// and verbatim error text (the catalog sentence only fills a missing message), +// metadata is stored as returned, and a reported status is never flipped. func (r *AccountJobRuntime) executeItem(ctx context.Context, job *AccountJob, payload json.RawMessage, item AccountJobItem) AccountJobExecutionResult { - result := AccountJobExecutionResult{ItemID: item.ID, Status: AccountJobItemStatusFailed, - ErrorCode: "execution_failed", ErrorMessage: "account job item failed"} - if r.executor != nil { + result := AccountJobExecutionResult{ItemID: item.ID, Status: AccountJobItemStatusFailed, ErrorCode: AccountJobCodeExecutionFailed} + if r.executor == nil { + result.ErrorMessage = "account job executor is unavailable" + } else { results, err := r.executor.ExecuteAccountJob(ctx, job, payload, []AccountJobItem{item}) - if err == nil && len(results) == 1 && results[0].ItemID == item.ID { + switch { + case err != nil: + result.ErrorMessage = err.Error() + case len(results) != 1 || results[0].ItemID != item.ID: + result.ErrorMessage = fmt.Sprintf("account job executor returned %d results for item %d", len(results), item.ID) + default: result = results[0] } } - if err := ValidateAccountJobMetadata(result.Metadata); err != nil { - result = AccountJobExecutionResult{ItemID: item.ID, Status: AccountJobItemStatusFailed, - ErrorCode: "result_redacted", ErrorMessage: "account job result was rejected"} - } - if result.Status == AccountJobItemStatusFailed { - result.ErrorCode, result.ErrorMessage = normalizeAccountJobFailure(result.ErrorCode) - } else { - result.ErrorCode, result.ErrorMessage = "", "" + result.Metadata = StorableAccountJobMetadata(result.Metadata) + switch result.Status { + case AccountJobItemStatusSucceeded, AccountJobItemStatusCanceled: + case AccountJobItemStatusFailed: + result.ErrorCode, result.ErrorMessage = AccountJobFailure(result.ErrorCode, result.ErrorMessage) + default: + if strings.TrimSpace(result.ErrorMessage) == "" { + result.ErrorMessage = fmt.Sprintf("account job executor returned item status %q", result.Status) + } + result.Status = AccountJobItemStatusFailed + result.ErrorCode, result.ErrorMessage = AccountJobFailure(result.ErrorCode, result.ErrorMessage) } return result } +// StorableAccountJobText returns text that a PostgreSQL text column accepts: +// invalid UTF-8 and NUL characters become U+FFFD, everything else is kept. +func StorableAccountJobText(text string) string { + if utf8.ValidString(text) && !strings.ContainsRune(text, 0) { + return text + } + return strings.ReplaceAll(strings.ToValidUTF8(text, "\uFFFD"), "\x00", "\uFFFD") +} + +// StorableAccountJobMetadata keeps executor metadata and returns it in a form +// the jsonb columns accept. Values stay as reported; only invalid UTF-8, NUL +// characters (jsonb rejects \u0000) and unpaired surrogates become U+FFFD. +// Bytes that are not JSON are kept as text under metadata_raw, and a JSON +// value that is not an object under metadata_value. +func StorableAccountJobMetadata(raw json.RawMessage) json.RawMessage { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || string(trimmed) == "null" { + return raw + } + if !json.Valid(trimmed) { + wrapped, _ := json.Marshal(map[string]string{"metadata_raw": StorableAccountJobText(string(raw))}) + return wrapped + } + if trimmed[0] == '{' && utf8.Valid(trimmed) && bytes.IndexByte(trimmed, 0) < 0 && !bytes.Contains(trimmed, []byte(`\u`)) { + return raw + } + decoder := json.NewDecoder(bytes.NewReader(trimmed)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + wrapped, _ := json.Marshal(map[string]string{"metadata_raw": StorableAccountJobText(string(raw))}) + return wrapped + } + value = storableAccountJobValue(value) + if _, ok := value.(map[string]any); !ok { + value = map[string]any{"metadata_value": value} + } + encoded, err := json.Marshal(value) + if err != nil { + wrapped, _ := json.Marshal(map[string]string{"metadata_raw": StorableAccountJobText(string(raw))}) + return wrapped + } + return encoded +} + +func storableAccountJobValue(value any) any { + switch typed := value.(type) { + case string: + return StorableAccountJobText(typed) + case []any: + for index := range typed { + typed[index] = storableAccountJobValue(typed[index]) + } + return typed + case map[string]any: + out := make(map[string]any, len(typed)) + for key, item := range typed { + out[StorableAccountJobText(key)] = storableAccountJobValue(item) + } + return out + default: + return value + } +} + +// accountJobRuntimeFailure keeps the underlying error text of a job-level +// failure; the catalog sentence is used only when there is none. +func accountJobRuntimeFailure(code string, err error) (string, string) { + message := "" + if err != nil { + message = err.Error() + } + return AccountJobFailure(code, message) +} + func validAccountJobKind(kind string) bool { switch kind { case AccountJobKindCodexTicketHarvest, AccountJobKindCodexTicketStop, AccountJobKindExtensionOperation: @@ -748,10 +936,8 @@ func normalizeAccountJobMetadata(raw json.RawMessage) json.RawMessage { return append(json.RawMessage(nil), raw...) } +// NormalizeAccountJobFailure keeps code and supplies the catalog sentence for +// a failure without underlying error text; see AccountJobFailure. func NormalizeAccountJobFailure(code string) (string, string) { return NormalizeAccountBusinessFailure(code) } - -func normalizeAccountJobFailure(code string) (string, string) { - return NormalizeAccountJobFailure(code) -} diff --git a/backend/internal/service/account_job_test.go b/backend/internal/service/account_job_test.go index e15a217785a1..2527904b72b0 100644 --- a/backend/internal/service/account_job_test.go +++ b/backend/internal/service/account_job_test.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "strings" "sync" "testing" "time" @@ -418,3 +419,82 @@ func TestAccountJobMetadataRejectsCredentialFields(t *testing.T) { require.NoError(t, ValidateAccountJobMetadata(json.RawMessage(`{"account_id":7,"action":"updated"}`))) require.ErrorIs(t, ValidateAccountJobMetadata(json.RawMessage(`{"api_key":"secret"}`)), ErrAccountJobInvalidMetadata) } + +type accountJobResultExecutor struct { + results map[int64]AccountJobExecutionResult + errs map[int64]error +} + +func (e accountJobResultExecutor) ExecuteAccountJob(_ context.Context, _ *AccountJob, _ json.RawMessage, items []AccountJobItem) ([]AccountJobExecutionResult, error) { + if err := e.errs[items[0].ID]; err != nil { + return nil, err + } + return []AccountJobExecutionResult{e.results[items[0].ID]}, nil +} + +// Item results keep the executor's code, verbatim error text and metadata; a +// success is never turned into a failure because of its metadata keys. +func TestAccountJobRuntimeStoresItemResultsVerbatim(t *testing.T) { + runtime := NewAccountJobRuntime(nil, accountJobResultExecutor{ + results: map[int64]AccountJobExecutionResult{ + 1: {ItemID: 1, Status: AccountJobItemStatusFailed, ErrorCode: "delete_failed", ErrorMessage: "pq: account 7 is still referenced"}, + 2: {ItemID: 2, Status: AccountJobItemStatusSucceeded, Metadata: json.RawMessage(`{"cookie_names":["__cflb"],"proxy_password":"kept"}`)}, + 3: {ItemID: 3, Status: AccountJobItemStatusFailed, ErrorCode: "routing_capacity"}, + }, + errs: map[int64]error{4: errors.New("executor exploded")}, + }) + job := &AccountJob{ID: 1, Kind: AccountJobKindBatchDelete} + + failed := runtime.executeItem(context.Background(), job, nil, AccountJobItem{ID: 1}) + require.Equal(t, "delete_failed", failed.ErrorCode) + require.Equal(t, "pq: account 7 is still referenced", failed.ErrorMessage) + + succeeded := runtime.executeItem(context.Background(), job, nil, AccountJobItem{ID: 2}) + require.Equal(t, AccountJobItemStatusSucceeded, succeeded.Status) + require.JSONEq(t, `{"cookie_names":["__cflb"],"proxy_password":"kept"}`, string(succeeded.Metadata)) + + routing := runtime.executeItem(context.Background(), job, nil, AccountJobItem{ID: 3}) + require.Equal(t, "routing_capacity", routing.ErrorCode) + require.Equal(t, "上游容量不足或流内限流，未完成路由验证", routing.ErrorMessage) + + errored := runtime.executeItem(context.Background(), job, nil, AccountJobItem{ID: 4}) + require.Equal(t, AccountJobItemStatusFailed, errored.Status) + require.Equal(t, AccountJobCodeExecutionFailed, errored.ErrorCode) + require.Equal(t, "executor exploded", errored.ErrorMessage) +} + +type accountJobFinishFailingRepo struct { + *accountJobTestRepo + failures int + messages []string +} + +func (r *accountJobFinishFailingRepo) Finish(_ context.Context, _ int64, _, message string) (*AccountJob, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.messages = append(r.messages, message) + if len(r.messages) <= r.failures { + return nil, errors.New("finish write failed") + } + return nil, nil +} + +// A failed finish write is retried with a shortened storable message, and the +// cleanup loop keeps retrying it, so a job never stays running because its +// result could not be written. +func TestAccountJobRuntimeRetriesFailedFinishWithStorableMessage(t *testing.T) { + repo := &accountJobFinishFailingRepo{accountJobTestRepo: newAccountJobTestRepo(), failures: accountJobFinishAttempts} + runtime := NewAccountJobRuntime(NewAccountJobService(repo, accountJobTestCipher{}), nil) + runtime.ctx, runtime.finishRetryDelay = context.Background(), time.Millisecond + message := strings.Repeat("x", 600) + "\xff\x00" + runtime.finish(1, AccountJobCodeCancelCheckFailed, message) + require.Len(t, repo.messages, accountJobFinishAttempts) + require.Equal(t, message, repo.messages[0]) + require.Equal(t, strings.Repeat("x", 500)+"...", repo.messages[1]) + require.Contains(t, runtime.unfinished, int64(1)) + + runtime.retryUnfinished() + require.Len(t, repo.messages, accountJobFinishAttempts+1) + require.Equal(t, strings.Repeat("x", 500)+"...", repo.messages[accountJobFinishAttempts]) + require.Empty(t, runtime.unfinished) +} diff --git a/backend/internal/service/account_test_prompt_test.go b/backend/internal/service/account_test_prompt_test.go index 0e5bdd535282..3ba8fb7c75b6 100644 --- a/backend/internal/service/account_test_prompt_test.go +++ b/backend/internal/service/account_test_prompt_test.go @@ -104,6 +104,7 @@ func TestAccountTestPromptOAuthFinalTicketUsesMappedModel(t *testing.T) { cfg := &config.Config{Gateway: config.GatewayConfig{OpenAICodexTicket: config.OpenAICodexTicketConfig{Enabled: true, FailClosed: true}, OpenAICodexRequestZstd: true}} a := ticketTestAccount(41) a.Status = StatusActive + a.Credentials["access_token"] = "account-test-access-token-canary" a.Credentials["model_mapping"] = map[string]any{"alias": "gpt-5.6-sol"} a.Credentials["header_override_enabled"] = true a.Credentials["header_overrides"] = map[string]any{openAICodexTurnStateHeader: fakeCodexTicketState(312)} @@ -160,8 +161,23 @@ func TestAccountTestPromptOAuthFinalTicketUsesMappedModel(t *testing.T) { body := zstdDecodeForTest(t, raw) require.Equal(t, "gpt-5.6-sol", gjson.GetBytes(body, "model").String()) require.Equal(t, prompt, gjson.GetBytes(body, "input.0.content.0.text").String()) - require.NotContains(t, rec.Body.String(), "verified-route") + // The test output reports the routing cookie exactly as it went on the wire. + require.Contains(t, rec.Body.String(), `"Cookie":"__cflb=verified-route"`) require.NotContains(t, rec.Body.String(), fakeCodexTicketState(292)) + // The reported wire headers leave out Authorization: the access token went + // on the wire but never appears in the test output. + require.Equal(t, "Bearer account-test-access-token-canary", req.Header.Get("Authorization")) + require.NotContains(t, rec.Body.String(), "account-test-access-token-canary") + var wireHeaders map[string]string + for _, line := range strings.Split(rec.Body.String(), "\n") { + if payload, ok := strings.CutPrefix(line, "data: "); ok && gjson.Get(payload, "text").String() == "Codex wire headers" { + require.NoError(t, json.Unmarshal([]byte(gjson.Get(payload, "data").Raw), &wireHeaders)) + } + } + require.NotEmpty(t, wireHeaders) + for name := range wireHeaders { + require.False(t, strings.EqualFold(name, "Authorization"), "Authorization is never reported") + } require.Equal(t, 1, strings.Count(rec.Body.String(), "Final Codex ticket"), "the leased wire is reported once") c, rec = newTestContext() qualification = nil // Legacy Extra still has a ticket; only the plugin qualification may admit the model. diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index fd130bd6d49e..256e2807aa52 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -909,7 +909,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account if credentialAccount.IsOpenAIAgentIdentity() { authHeaders, authErr := buildAgentIdentityAuthenticationHeaders(ctx, s.accountRepo, s.agentIdentityWS, &s.agentIdentityTaskMu, credentialAccount) if authErr != nil { - return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication") + return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication: "+authErr.Error()) } for key, values := range authHeaders { for _, value := range values { @@ -927,7 +927,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account req.Header.Set("OpenAI-Beta", "responses=experimental") canonical, identityErr := resolveCodexOutboundIdentityForAccountContext(req.Context(), credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)) if identityErr != nil { - return s.sendErrorAndEnd(c, "Codex identity policy unavailable") + return s.sendErrorAndEnd(c, "Codex identity policy unavailable: "+identityErr.Error()) } req.Header.Set("Originator", canonical.originator) req.Header.Set("User-Agent", canonical.userAgent) @@ -935,7 +935,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account // 与真实转发一致：使用该账号的 Codex TUI 身份，账号级自定义 UA 同样作为管理员 // 显式配置传入，否则测试用的身份与该账号真实出站的身份不是同一个。 if err := enforceCodexIdentityHeadersForAccountContext(req.Context(), req.Header, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)); err != nil { - return s.sendErrorAndEnd(c, "Codex identity policy unavailable") + return s.sendErrorAndEnd(c, "Codex identity policy unavailable: "+err.Error()) } } @@ -944,7 +944,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account if isOAuth && !account.IsCredentialShadow() && s.openAIGatewayService != nil { req = withCodexRoutingModel(req, upstreamTestModelID) if err := s.openAIGatewayService.applyOpenAICodexTicket(ctx, credentialAccount, upstreamTestModelID, req.Header); err != nil { - return s.sendErrorAndEnd(c, "当前账号/模型没有有效的 Cookie 路由验证，请先执行采集与业务出口复验；此次连接测试未发送上游请求") + return s.sendErrorAndEnd(c, "当前账号/模型没有有效的 Cookie 路由验证，请先执行采集与业务出口复验；此次连接测试未发送上游请求。原始错误："+err.Error()) } s.sendEvent(c, TestEvent{Type: "status", Text: "Codex ticket", Data: codexTicketWireSummary(req.Header)}) } @@ -962,6 +962,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account s.sendEvent(c, TestEvent{Type: "status", Text: "Final Codex ticket", Data: codexTicketWireSummary(wire.Header)}) snapshot := base.withWire("http", wire.Header) s.sendEvent(c, TestEvent{Type: "status", Text: snapshot.summary(), Data: snapshot}) + s.sendEvent(c, TestEvent{Type: "status", Text: "Codex wire headers", Data: accountTestWireHeaders(wire.Header)}) })) } @@ -974,6 +975,7 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account base := resolveCodexIdentitySnapshotContext(ctx, account, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)) snapshot := base.withWire(transport, req.Header) s.sendEvent(c, TestEvent{Type: "status", Text: snapshot.summary(), Data: snapshot}) + s.sendEvent(c, TestEvent{Type: "status", Text: "Codex wire headers", Data: accountTestWireHeaders(req.Header)}) } if err != nil { return s.sendErrorAndEnd(c, fmt.Sprintf("Request failed: %s", err.Error())) @@ -1599,7 +1601,7 @@ func (s *AccountTestService) emitGrokVideoResult(c *gin.Context, ctx context.Con s.sendEvent(c, TestEvent{Type: "test_complete", Success: true}) return nil } - return s.sendErrorAndEnd(c, fmt.Sprintf("Grok video content returned %d: %s", resp.StatusCode, truncateString(string(body), 300))) + return s.sendErrorAndEnd(c, fmt.Sprintf("Grok video content returned %d: %s", resp.StatusCode, string(body))) } ct := resp.Header.Get("Content-Type") if ct == "" || strings.HasPrefix(ct, "application/octet-stream") { @@ -1693,9 +1695,6 @@ User query: if text == "" { continue } - if len(text) > 300 { - text = text[:300] + "..." - } s.sendEvent(c, TestEvent{Type: "content", Text: text + "\n"}) return false } @@ -1843,9 +1842,6 @@ func (s *AccountTestService) testGrokSTT(c *gin.Context, ctx context.Context, ac text := strings.TrimSpace(gjson.GetBytes(respBody, "text").String()) if text == "" { text = strings.TrimSpace(string(respBody)) - if len(text) > 200 { - text = text[:200] + "..." - } } s.sendEvent(c, TestEvent{Type: "content", Text: fmt.Sprintf("stt ok: %s\n", text)}) s.sendEvent(c, TestEvent{Type: "test_complete", Success: true}) @@ -1915,11 +1911,7 @@ func (s *AccountTestService) testGrokRealtime(c *gin.Context, ctx context.Contex detail := dialErr.Error() var hs *openAIWSHandshakeError if errors.As(dialErr, &hs) && len(hs.Body) > 0 { - body := strings.TrimSpace(string(hs.Body)) - if len(body) > 300 { - body = body[:300] + "..." - } - detail = fmt.Sprintf("%s body=%s", detail, body) + detail = fmt.Sprintf("%s body=%s", detail, strings.TrimSpace(string(hs.Body))) } if status > 0 { return s.sendErrorAndEnd(c, fmt.Sprintf("Grok Realtime WS handshake failed (HTTP %d): %s", status, detail)) @@ -1940,9 +1932,6 @@ func (s *AccountTestService) testGrokRealtime(c *gin.Context, ctx context.Contex eventType = "unknown" } preview := strings.TrimSpace(string(msg)) - if len(preview) > 240 { - preview = preview[:240] + "..." - } s.sendEvent(c, TestEvent{ Type: "content", Text: fmt.Sprintf("realtime first event: type=%s payload=%s\n", eventType, preview), @@ -2043,9 +2032,6 @@ func formatGrokImageTransportError(err error, hasSourceImage bool, payloadBytes func formatGrokImagesAPIError(status int, body []byte, hasSourceImage bool) string { msg := strings.TrimSpace(string(body)) - if len(msg) > 800 { - msg = msg[:800] + "..." - } prefix := fmt.Sprintf("Grok images API returned %d: %s", status, msg) lower := strings.ToLower(msg) if hasSourceImage && (strings.Contains(lower, "too small") || strings.Contains(lower, "at least 8")) { @@ -2296,7 +2282,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if credentialAccount.IsOpenAIAgentIdentity() { authHeaders, authErr := buildAgentIdentityAuthenticationHeaders(ctx, s.accountRepo, s.agentIdentityWS, &s.agentIdentityTaskMu, credentialAccount) if authErr != nil { - return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication") + return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication: "+authErr.Error()) } for key, values := range authHeaders { for _, value := range values { @@ -2311,7 +2297,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account // 与真实转发一致：使用该账号的 Codex TUI 身份，账号级自定义 UA 经 ForceCodexCLI // 策略过滤后作为管理员显式配置传入（同普通 OAuth 连接测试）。 if err := enforceCodexIdentityHeadersForAccountContext(req.Context(), req.Header, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)); err != nil { - return s.sendErrorAndEnd(c, "Codex identity policy unavailable") + return s.sendErrorAndEnd(c, "Codex identity policy unavailable: "+err.Error()) } } probeSessionID := compactProbeSessionID(account.ID) @@ -2993,7 +2979,7 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co if credentialAccount.IsOpenAIAgentIdentity() { authHeaders, authErr := buildAgentIdentityAuthenticationHeaders(ctx, s.accountRepo, s.agentIdentityWS, &s.agentIdentityTaskMu, credentialAccount) if authErr != nil { - return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication") + return s.sendErrorAndEnd(c, "Failed to build Agent Identity authentication: "+authErr.Error()) } for key, values := range authHeaders { for _, value := range values { @@ -3008,7 +2994,7 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co req.Header.Set("OpenAI-Beta", "responses=experimental") canonical, identityErr := resolveCodexOutboundIdentityForAccountContext(req.Context(), credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)) if identityErr != nil { - return s.sendErrorAndEnd(c, "Codex identity policy unavailable") + return s.sendErrorAndEnd(c, "Codex identity policy unavailable: "+identityErr.Error()) } req.Header.Set("originator", canonical.originator) req.Header.Set("User-Agent", canonical.userAgent) @@ -3016,7 +3002,7 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co // 与真实转发一致（同普通 OAuth 连接测试）：使用该账号的 Codex TUI 身份，账号级自定义 UA // 经 ForceCodexCLI 策略过滤后作为管理员显式配置传入。 if err := enforceCodexIdentityHeadersForAccountContext(req.Context(), req.Header, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount)); err != nil { - return s.sendErrorAndEnd(c, "Codex identity policy unavailable") + return s.sendErrorAndEnd(c, "Codex identity policy unavailable: "+err.Error()) } proxyURL := "" @@ -3040,11 +3026,7 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co if resp.StatusCode >= 400 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) body = redactAgentIdentitySensitiveBodyForAccount(ctx, s.accountRepo, credentialAccount, body) - message := strings.TrimSpace(extractUpstreamErrorMessage(body)) - if message == "" { - message = fmt.Sprintf("Responses API returned %d", resp.StatusCode) - } - return s.sendErrorAndEnd(c, message) + return s.sendErrorAndEnd(c, fmt.Sprintf("Responses API returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))) } body, err := io.ReadAll(resp.Body) @@ -3058,11 +3040,19 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to parse image response: %s", err.Error())) } if len(results) == 0 { + // The upstream error frame is reported as sent, not only its message. + if frame := accountTestImageErrorFrame(body); frame != "" { + return s.sendErrorAndEnd(c, "Responses API stream error: "+frame) + } if upstreamErr := extractOpenAIImagesUpstreamError(body); upstreamErr != nil { return s.sendErrorAndEnd(c, upstreamErr.clientMessage()) } if textErr := openAIImagesTextFallbackError(body); textErr != nil { - return s.sendErrorAndEnd(c, textErr.clientMessage()) + message := textErr.clientMessage() + if modelText := extractOpenAIImagesModelText(body); modelText != "" && modelText != message { + message += "\nModel text: " + modelText + } + return s.sendErrorAndEnd(c, message) } return s.sendErrorAndEnd(c, "No images returned from responses API") } @@ -3164,6 +3154,37 @@ func (s *AccountTestService) testOpenAIImageOAuthDirect(c *gin.Context, ctx cont return nil } +// accountTestImageErrorFrame returns the first upstream error, failed or +// incomplete event of an image stream exactly as it was sent. +func accountTestImageErrorFrame(body []byte) string { + var frame string + forEachOpenAISSEDataPayload(string(body), func(payload []byte) { + if frame != "" || !gjson.ValidBytes(payload) { + return + } + switch gjson.GetBytes(payload, "type").String() { + case "error", "response.failed", "response.incomplete": + frame = strings.TrimSpace(string(payload)) + } + }) + return frame +} + +// accountTestWireHeaders reports the request headers a Codex connection test +// sent upstream, values verbatim (identity headers, ticket and cookies +// included). The Authorization header is the account credential itself and +// stays behind the account credential view. +func accountTestWireHeaders(header http.Header) map[string]string { + out := make(map[string]string, len(header)) + for name, values := range header { + if strings.EqualFold(name, "Authorization") { + continue + } + out[name] = strings.Join(values, ", ") + } + return out +} + // sendErrorAndEnd sends an error event and ends the stream func (s *AccountTestService) sendErrorAndEnd(c *gin.Context, errorMsg string) error { log.Printf("Account test error: %s", errorMsg) diff --git a/backend/internal/service/account_test_service_openai_test.go b/backend/internal/service/account_test_service_openai_test.go index 599a7d3829cf..1016dbf1d94e 100644 --- a/backend/internal/service/account_test_service_openai_test.go +++ b/backend/internal/service/account_test_service_openai_test.go @@ -174,7 +174,7 @@ func TestAccountTestService_Budget429PutsAccountIntoError(t *testing.T) { require.Error(t, err) require.Len(t, upstream.bodies, 1, "a manual connection test must issue only its requested call") require.Equal(t, 1, repo.setErrorCalls) - require.Equal(t, "ExceededBudget: key over budget. Spend=3.05, Budget=3.0", repo.lastErrorMsg, "the account keeps the upstream message") + require.Equal(t, "ExceededBudget: key over budget. Spend=3.05, Budget=3.0 (type=budget_exceeded, code=429)", repo.lastErrorMsg, "the account keeps the upstream message with its error type and code") } func TestAccountTestService_EmptyModelKeepsOpenAIDefault(t *testing.T) { diff --git a/backend/internal/service/account_test_stream.go b/backend/internal/service/account_test_stream.go index dd1ba2c2a295..a7ddf5591986 100644 --- a/backend/internal/service/account_test_stream.go +++ b/backend/internal/service/account_test_stream.go @@ -32,20 +32,23 @@ func accountTestTerminal(format string, args ...any) error { return accountTestTerminalError(fmt.Sprintf(format, args...)) } -// accountTestUpstreamError returns the upstream's own error message, or the raw -// JSON it sent when there is no message field. +// accountTestUpstreamError returns the upstream's error exactly as sent: a +// plain string as is, and an error object as its full JSON (type, code, param +// and every other field), led by its message when it has one. func accountTestUpstreamError(value any) string { - switch typed := value.(type) { - case string: - if strings.TrimSpace(typed) != "" { - return typed - } - case map[string]any: - if message, _ := typed["message"].(string); strings.TrimSpace(message) != "" { - return message - } + if typed, ok := value.(string); ok && strings.TrimSpace(typed) != "" { + return typed } raw, _ := json.Marshal(value) + if typed, ok := value.(map[string]any); ok { + message, _ := typed["message"].(string) + if strings.TrimSpace(message) != "" { + if len(typed) == 1 { + return message + } + return message + " " + string(raw) + } + } return string(raw) } @@ -327,7 +330,7 @@ func parseAccountConnectionStream(protocol string, body io.Reader, allowMedia bo } var data map[string]any if json.Unmarshal([]byte(raw), &data) != nil { - return false, fmt.Errorf("%w: %s", ErrAccountTestProtocol, truncateUTF8(raw, 200)) + return false, fmt.Errorf("%w: %s", ErrAccountTestProtocol, raw) } for _, observer := range observe { observer(data) diff --git a/backend/internal/service/account_test_stream_contract_test.go b/backend/internal/service/account_test_stream_contract_test.go index 52147e6538fc..8bc88ae501fa 100644 --- a/backend/internal/service/account_test_stream_contract_test.go +++ b/backend/internal/service/account_test_stream_contract_test.go @@ -90,7 +90,8 @@ func TestAccountConnectionStreamContract(t *testing.T) { require.Equal(t, err.Error(), errorText, "the event carries the same text as the returned failure") } if tc.name == "responses_failed_verbatim" { - require.Equal(t, "You have no credits remaining.", errorText) + // The whole upstream error object is reported, led by its message. + require.Equal(t, `You have no credits remaining. {"code":"credit_balance_exhausted","message":"You have no credits remaining."}`, errorText) } }) } diff --git a/backend/internal/service/account_tools.go b/backend/internal/service/account_tools.go index bfa176e55fe5..56bb18c1fdba 100644 --- a/backend/internal/service/account_tools.go +++ b/backend/internal/service/account_tools.go @@ -29,7 +29,8 @@ func accountToolsOperationScoped(ctx context.Context, accountID int64, operation defer cancel() result, err := invokeAccountTools(call, extensionv1.Invocation{Capability: extensionv1.CapabilityAdmin, Operation: operation, AccountID: accountID, Payload: raw}) if err != nil { - return infraerrors.New(503, "ACCOUNT_TOOLS_UNAVAILABLE", "account tools are unavailable") + // Administrators see why the operation failed (for example a timeout). + return infraerrors.New(503, "ACCOUNT_TOOLS_UNAVAILABLE", "account tools are unavailable: "+err.Error()).WithCause(err) } if result.Code != "" { status := result.HTTPStatus diff --git a/backend/internal/service/admin_account.go b/backend/internal/service/admin_account.go index 2e67b194ad4b..1b766afbfc7b 100644 --- a/backend/internal/service/admin_account.go +++ b/backend/internal/service/admin_account.go @@ -126,6 +126,7 @@ var duplicateAccountDiscardedExtraKeys = map[string]struct{}{ "quota_daily_reset_at": {}, "quota_weekly_reset_at": {}, // Provider observations, capability probes, and transient scheduling state. + OpenAILastUpstreamErrorExtraKey: {}, "model_rate_limits": {}, "session_window_utilization": {}, "passive_usage_7d_utilization": {}, diff --git a/backend/internal/service/admin_account_taxonomy_test.go b/backend/internal/service/admin_account_taxonomy_test.go index d2425df8bcba..884c49f5a44a 100644 --- a/backend/internal/service/admin_account_taxonomy_test.go +++ b/backend/internal/service/admin_account_taxonomy_test.go @@ -12,6 +12,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" dbent "github.com/Wei-Shaw/sub2api/ent" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" + infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/stretchr/testify/require" ) @@ -65,6 +66,7 @@ func TestTaxonomyAssignmentPreservesPolicyNormalizationAndFailsClosed(t *testing svc := &adminServiceImpl{} account, err := svc.SetAccountTaxonomy(context.Background(), 42, input) require.Error(t, err) + require.Equal(t, "account tools are unavailable: policy failed", infraerrors.Message(err), "administrators see the underlying failure") require.Nil(t, account, "a failed policy must stop before using the nil database") result, err := svc.BulkUpdateAccountTaxonomy(context.Background(), BulkAccountTaxonomyInput{AccountIDs: []int64{42}, FolderAction: "clear"}) require.Error(t, err) diff --git a/backend/internal/service/admin_service_apikey_test.go b/backend/internal/service/admin_service_apikey_test.go index 8abd913b94eb..f5348687d7fb 100644 --- a/backend/internal/service/admin_service_apikey_test.go +++ b/backend/internal/service/admin_service_apikey_test.go @@ -565,3 +565,30 @@ func TestAdminService_AdminUpdateAPIKeyGroupID_Unbind_NoAllowedGroupUpdate(t *te require.False(t, userRepo.addGroupCalled) require.False(t, got.AutoGrantedGroupAccess) } + +type adminKeyListRepoStub struct { + APIKeyRepository + userCalls, adminCalls int +} + +func (s *adminKeyListRepoStub) ListByUserID(context.Context, int64, pagination.PaginationParams, APIKeyListFilters) ([]APIKey, *pagination.PaginationResult, error) { + s.userCalls++ + return nil, &pagination.PaginationResult{}, nil +} + +func (s *adminKeyListRepoStub) ListByUserIDForAdmin(context.Context, int64, pagination.PaginationParams, APIKeyListFilters) ([]APIKey, *pagination.PaginationResult, error) { + s.adminCalls++ + return []APIKey{{ID: 5, Purpose: APIKeyPurposeReleaseAcceptance}}, &pagination.PaginationResult{Total: 1}, nil +} + +// The admin console lists a user's internal release-acceptance keys too; the +// user's own key list (ListByUserID) does not. +func TestAdminService_GetUserAPIKeysUsesAdminListing(t *testing.T) { + repo := &adminKeyListRepoStub{} + keys, total, err := (&adminServiceImpl{apiKeyRepo: repo}).GetUserAPIKeys(context.Background(), 9, 1, 20, "created_at", "desc") + require.NoError(t, err) + require.EqualValues(t, 1, total) + require.Equal(t, APIKeyPurposeReleaseAcceptance, keys[0].Purpose) + require.Equal(t, 1, repo.adminCalls) + require.Zero(t, repo.userCalls) +} diff --git a/backend/internal/service/admin_user.go b/backend/internal/service/admin_user.go index 12f71f96a267..1d12c0f9884c 100644 --- a/backend/internal/service/admin_user.go +++ b/backend/internal/service/admin_user.go @@ -601,9 +601,20 @@ func (s *adminServiceImpl) tryAccrueAffiliateRebateForAdminRecharge(ctx context. } } +// apiKeyAdminLister lists one user's keys for administrators, internal +// release-acceptance keys included (marked by their purpose). The user's own +// key list keeps using ListByUserID, which excludes them. +type apiKeyAdminLister interface { + ListByUserIDForAdmin(ctx context.Context, userID int64, params pagination.PaginationParams, filters APIKeyListFilters) ([]APIKey, *pagination.PaginationResult, error) +} + func (s *adminServiceImpl) GetUserAPIKeys(ctx context.Context, userID int64, page, pageSize int, sortBy, sortOrder string) ([]APIKey, int64, error) { params := pagination.PaginationParams{Page: page, PageSize: pageSize, SortBy: sortBy, SortOrder: sortOrder} - keys, result, err := s.apiKeyRepo.ListByUserID(ctx, userID, params, APIKeyListFilters{}) + list := s.apiKeyRepo.ListByUserID + if lister, ok := s.apiKeyRepo.(apiKeyAdminLister); ok { + list = lister.ListByUserIDForAdmin + } + keys, result, err := list(ctx, userID, params, APIKeyListFilters{}) if err != nil { return nil, 0, err } diff --git a/backend/internal/service/audit_log_service.go b/backend/internal/service/audit_log_service.go index 7ff895713d9a..0f8d0fdd6d42 100644 --- a/backend/internal/service/audit_log_service.go +++ b/backend/internal/service/audit_log_service.go @@ -138,16 +138,7 @@ func (s *AuditLogService) runWriter() { if len(batch) == 0 { return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - inserted, err := s.repo.BatchInsert(ctx, batch) - cancel() - if err != nil { - atomic.AddUint64(&s.writeFailed, uint64(len(batch))) - _, _ = fmt.Fprintf(os.Stderr, "time=%s level=WARN msg=\"audit log flush failed\" err=%v batch=%d\n", - time.Now().Format(time.RFC3339Nano), err, len(batch)) - } else { - atomic.AddUint64(&s.writtenCount, uint64(inserted)) - } + s.writeBatch(batch) batch = batch[:0] } @@ -184,6 +175,35 @@ func (s *AuditLogService) runWriter() { } } +// writeBatch stores one batch. One row the database rejects fails the whole +// batch insert, so the rows are then inserted one by one: the other records +// are kept and only the rejected rows are counted as failed. +func (s *AuditLogService) writeBatch(batch []*AuditLog) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + inserted, err := s.repo.BatchInsert(ctx, batch) + cancel() + if err == nil { + atomic.AddUint64(&s.writtenCount, uint64(inserted)) + return + } + _, _ = fmt.Fprintf(os.Stderr, "time=%s level=WARN msg=\"audit log batch insert failed; inserting rows one by one\" err=%v batch=%d\n", + time.Now().Format(time.RFC3339Nano), err, len(batch)) + rowsCtx, cancelRows := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelRows() + for _, entry := range batch { + if entry == nil { + continue + } + if rowErr := s.repo.Insert(rowsCtx, entry); rowErr != nil { + atomic.AddUint64(&s.writeFailed, 1) + _, _ = fmt.Fprintf(os.Stderr, "time=%s level=WARN msg=\"audit log insert failed\" err=%v action=%s\n", + time.Now().Format(time.RFC3339Nano), rowErr, entry.Action) + continue + } + atomic.AddUint64(&s.writtenCount, 1) + } +} + // runRetentionLoop 按保留期定期删除过期审计日志。 // 删除操作幂等，多实例并发执行无害，因此无需选主。 func (s *AuditLogService) runRetentionLoop() { diff --git a/backend/internal/service/audit_log_test.go b/backend/internal/service/audit_log_test.go index 0045c936f2ea..4ef0c93b16bf 100644 --- a/backend/internal/service/audit_log_test.go +++ b/backend/internal/service/audit_log_test.go @@ -1,7 +1,9 @@ package service import ( + "context" "encoding/json" + "errors" "strings" "testing" ) @@ -198,3 +200,33 @@ func TestParseAuditLogRetentionDays(t *testing.T) { } } } + +type auditRowFallbackRepository struct { + AuditLogRepository + inserted []string +} + +func (r *auditRowFallbackRepository) BatchInsert(context.Context, []*AuditLog) (int64, error) { + return 0, errors.New("invalid byte sequence for encoding \"UTF8\": 0x00") +} + +func (r *auditRowFallbackRepository) Insert(_ context.Context, entry *AuditLog) error { + if entry.Action == "rejected" { + return errors.New("invalid byte sequence for encoding \"UTF8\": 0x00") + } + r.inserted = append(r.inserted, entry.Action) + return nil +} + +// One row the database rejects must not drop the rest of its batch. +func TestAuditLogWriteBatchFallsBackToRowInserts(t *testing.T) { + repo := &auditRowFallbackRepository{} + s := &AuditLogService{repo: repo} + s.writeBatch([]*AuditLog{{Action: "first"}, {Action: "rejected"}, {Action: "last"}}) + if strings.Join(repo.inserted, ",") != "first,last" { + t.Fatalf("inserted rows = %v, want first,last", repo.inserted) + } + if s.writtenCount != 2 || s.writeFailed != 1 { + t.Fatalf("written=%d failed=%d, want 2 and 1", s.writtenCount, s.writeFailed) + } +} diff --git a/backend/internal/service/codex_error_diagnostics.go b/backend/internal/service/codex_error_diagnostics.go index 721fc27c5f25..7823853f6045 100644 --- a/backend/internal/service/codex_error_diagnostics.go +++ b/backend/internal/service/codex_error_diagnostics.go @@ -1,40 +1,58 @@ package service import ( - "context" "encoding/json" ) -// CodexErrorDiagnostic contains only host-sanitized structural facts. Request -// bodies, headers, error messages, URLs and account names never enter the UI. +// CodexErrorDiagnostic projects one persisted upstream attempt that carries a +// continuation diagnostic, together with that attempt's own context. The +// administrator sees every stored record, including the bounded original values +// kept inside the diagnostic. type CodexErrorDiagnostic struct { - AccountID int64 `json:"account_id"` - Attempt int `json:"attempt"` - Diagnostic *OpenAIContinuationDiagnostic `json:"diagnostic"` + AccountID int64 `json:"account_id"` + AccountName string `json:"account_name,omitempty"` + Platform string `json:"platform,omitempty"` + Attempt int `json:"attempt"` + Kind string `json:"kind,omitempty"` + UpstreamStatusCode int `json:"upstream_status_code,omitempty"` + UpstreamRequestID string `json:"upstream_request_id,omitempty"` + Message string `json:"message,omitempty"` + AtUnixMs int64 `json:"at_unix_ms,omitempty"` + Diagnostic *OpenAIContinuationDiagnostic `json:"diagnostic"` } -func ProjectCodexErrorDiagnostics(ctx context.Context, detail *OpsErrorLogDetail, readAccount func(context.Context, int64) (*Account, error)) []CodexErrorDiagnostic { +// ProjectCodexErrorDiagnostics returns every stored attempt diagnostic of one +// Ops error. The stored upstream_errors document is already bounded by the Ops +// queue limits, so no further count or account-state filter applies. +func ProjectCodexErrorDiagnostics(detail *OpsErrorLogDetail) []CodexErrorDiagnostic { result := make([]CodexErrorDiagnostic, 0) - if detail == nil || readAccount == nil || len(detail.UpstreamErrors) > 4<<20 { + if detail == nil { return result } var events []OpsUpstreamErrorEvent - if json.Unmarshal([]byte(detail.UpstreamErrors), &events) != nil || len(events) > 128 { + if json.Unmarshal([]byte(detail.UpstreamErrors), &events) != nil { return result } for index, event := range events { - if event.AccountID <= 0 || event.ContinuationDiagnostic == nil { + if event.ContinuationDiagnostic == nil { continue } - // Scope comes from the persisted event and current account, never the UI - // payload. Deleted accounts remain visible in ordinary host history only. - account, err := readAccount(ctx, event.AccountID) - if err != nil || account == nil || account.ID != event.AccountID || account.Platform != PlatformOpenAI { + diagnostic := sanitizeOpenAIContinuationDiagnostic(event.ContinuationDiagnostic) + if diagnostic == nil { continue } - if safe := sanitizeOpenAIContinuationDiagnostic(event.ContinuationDiagnostic); safe != nil { - result = append(result, CodexErrorDiagnostic{event.AccountID, index + 1, safe}) - } + result = append(result, CodexErrorDiagnostic{ + AccountID: event.AccountID, + AccountName: event.AccountName, + Platform: event.Platform, + Attempt: index + 1, + Kind: event.Kind, + UpstreamStatusCode: event.UpstreamStatusCode, + UpstreamRequestID: event.UpstreamRequestID, + Message: event.Message, + AtUnixMs: event.AtUnixMs, + Diagnostic: diagnostic, + }) } return result } diff --git a/backend/internal/service/codex_error_diagnostics_test.go b/backend/internal/service/codex_error_diagnostics_test.go index d6b887b2baad..9f24b55c3da2 100644 --- a/backend/internal/service/codex_error_diagnostics_test.go +++ b/backend/internal/service/codex_error_diagnostics_test.go @@ -1,38 +1,35 @@ package service import ( - "context" - "encoding/json" "testing" "github.com/stretchr/testify/require" ) -func TestNativeCodexErrorDiagnosticsUseStoredAccountsAndRemovePrivateFields(t *testing.T) { - read := func(_ context.Context, id int64) (*Account, error) { - if id == 1 { - return &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth}, nil - } - if id == 2 { - return &Account{ID: 2, Platform: PlatformAnthropic, Type: AccountTypeAPIKey}, nil - } - return nil, nil - } - ctx := context.Background() - +// Every stored attempt diagnostic is projected with its own attempt context, +// including the bounded original values; account state no longer filters it. +func TestNativeCodexErrorDiagnosticsProjectEveryStoredAttempt(t *testing.T) { detail := &OpsErrorLogDetail{UpstreamErrors: `[ - {"account_id":1,"account_name":"private-name","upstream_response_body":"private-body","continuation_diagnostic":{"classification":"thinking_signature_invalid","upstream_error":{"error_code":{"kind":"string","value":"thinking_signature_invalid"},"message":{"kind":"string","value":"private-message"},"hints":["private-hint","call_id"]},"incoming":{"instructions":{"kind":"string","value":"private-instructions"}}}}, - {"account_id":2,"platform":"openai","continuation_diagnostic":{"classification":"invalid_encrypted_content"}}, - {"account_id":0,"continuation_diagnostic":{"classification":"invalid_encrypted_content"}} + {"account_id":1,"account_name":"primary","platform":"openai","kind":"request_rejected","upstream_status_code":400,"upstream_request_id":"req_1","message":"Missing required parameter","continuation_diagnostic":{"classification":"thinking_signature_invalid","upstream_error":{"error_code":{"kind":"string","value":"thinking_signature_invalid"},"message":{"kind":"string","value":"signature invalid for input[3]"},"hints":["unlisted-hint","call_id"]},"incoming":{"instructions":{"kind":"string","value":"You are Codex"}}}}, + {"account_id":2,"platform":"anthropic","continuation_diagnostic":{"classification":"invalid_encrypted_content"}}, + {"account_id":3,"message":"no diagnostic"}, + {"account_id":0,"continuation_diagnostic":{"classification":"previous_response_not_found"}} ]`} - projected := ProjectCodexErrorDiagnostics(ctx, detail, read) - require.Len(t, projected, 1) - require.EqualValues(t, 1, projected[0].AccountID) - require.Equal(t, "thinking_signature_invalid", projected[0].Diagnostic.UpstreamError.ErrorCode.Value) - require.Equal(t, []string{"call_id"}, projected[0].Diagnostic.UpstreamError.Hints) - raw, err := json.Marshal(projected) - require.NoError(t, err) - require.NotContains(t, string(raw), "private-") - require.Empty(t, ProjectCodexErrorDiagnostics(ctx, detail, nil), "without an account reader no facts can be projected") - require.Empty(t, ProjectCodexErrorDiagnostics(ctx, detail, func(context.Context, int64) (*Account, error) { return nil, nil }), "deleted accounts are not resurrected from event metadata") + projected := ProjectCodexErrorDiagnostics(detail) + require.Len(t, projected, 3) + first := projected[0] + require.EqualValues(t, 1, first.AccountID) + require.Equal(t, "primary", first.AccountName) + require.Equal(t, 1, first.Attempt) + require.Equal(t, "request_rejected", first.Kind) + require.Equal(t, 400, first.UpstreamStatusCode) + require.Equal(t, "req_1", first.UpstreamRequestID) + require.Equal(t, "Missing required parameter", first.Message) + require.Equal(t, "thinking_signature_invalid", first.Diagnostic.UpstreamError.ErrorCode.Value) + require.Equal(t, "signature invalid for input[3]", first.Diagnostic.UpstreamError.Message.Value) + require.Equal(t, "You are Codex", first.Diagnostic.Incoming.Instructions.Value) + require.Equal(t, []string{"call_id"}, first.Diagnostic.UpstreamError.Hints, "hints stay the fixed derived labels") + require.Equal(t, 2, projected[1].Attempt) + require.Equal(t, 4, projected[2].Attempt) + require.Empty(t, ProjectCodexErrorDiagnostics(nil)) } diff --git a/backend/internal/service/codex_native_host.go b/backend/internal/service/codex_native_host.go index 97ad0d2f4a6f..4be0f70bba55 100644 --- a/backend/internal/service/codex_native_host.go +++ b/backend/internal/service/codex_native_host.go @@ -141,8 +141,13 @@ func (h *nativeCodexHost) Call(ctx context.Context, in extensionv1.HostInvocatio if json.Unmarshal(in.Payload, &req) != nil || validatePluginKVNamespace(req.Namespace) != nil || req.Limit < 1 || req.Limit > 100 { return extensionv1.Result{}, status.Error(codes.InvalidArgument, "invalid due-state query") } + // Bundles, cookie clocks, first-seen and validation records are host-only: + // they hold live cookie values that bundle expiry and quality-run close + // clear. What administrators see (raw STATE, upstream response headers, + // errors) reaches the runtime in probe results and is kept in its tickets + // records, never read back from this namespace. if req.Namespace == codexRoutingPrivateNamespace { - return extensionv1.Result{}, status.Error(codes.PermissionDenied, "host-owned routing material is private") + return extensionv1.Result{}, status.Error(codes.PermissionDenied, "codex routing bundles and cookie clocks are host-only state") } value, err = h.state.DueExtensionStates(ctx, h.key, req) case extensionv1.HostAccountRead, extensionv1.HostAccountList, extensionv1.HostResolveIdentity, extensionv1.HostFinishObservation: @@ -205,8 +210,13 @@ func (h *nativeCodexHost) Call(ctx context.Context, in extensionv1.HostInvocatio if json.Unmarshal(in.Payload, &req) != nil || validatePluginKVNamespace(req.Namespace) != nil || validatePluginKVKey(req.Key) != nil || req.ExpectedRevision < 0 { return extensionv1.Result{}, status.Error(codes.InvalidArgument, "invalid state request") } + // Bundles, cookie clocks, first-seen and validation records are host-only: + // they hold live cookie values that bundle expiry and quality-run close + // clear. What administrators see (raw STATE, upstream response headers, + // errors) reaches the runtime in probe results and is kept in its tickets + // records, never read back from this namespace. if req.Namespace == codexRoutingPrivateNamespace { - return extensionv1.Result{}, status.Error(codes.PermissionDenied, "host-owned routing material is private") + return extensionv1.Result{}, status.Error(codes.PermissionDenied, "codex routing bundles and cookie clocks are host-only state") } if in.Operation == extensionv1.HostStateRead { value, err = h.state.ReadExtensionState(ctx, h.key, req) @@ -231,7 +241,8 @@ func (h *nativeCodexHost) Call(ctx context.Context, in extensionv1.HostInvocatio return extensionv1.Result{}, status.Error(codes.InvalidArgument, "too many observations") } for _, observation := range projection.Observations { - if observation.Key == "" || len(observation.Key) > 256 || len(observation.Kind) > 64 || len(observation.State) > 64 || len(observation.Code) > 80 { + if observation.Key == "" || len(observation.Key) > 256 || len(observation.Kind) > 64 || len(observation.State) > 64 || len(observation.Code) > 80 || + len(observation.Message) > extensionv1.AccountObservationMessageLimit || len(observation.ResponseModel) > extensionv1.AccountObservationModelLimit { return extensionv1.Result{}, status.Error(codes.InvalidArgument, "invalid account observation") } } diff --git a/backend/internal/service/codex_native_policy.go b/backend/internal/service/codex_native_policy.go index 302e92c8a78a..1b410b67866b 100644 --- a/backend/internal/service/codex_native_policy.go +++ b/backend/internal/service/codex_native_policy.go @@ -3,7 +3,7 @@ package service import ( "context" "encoding/json" - "errors" + "fmt" "io" "net/http" "slices" @@ -15,6 +15,25 @@ import ( "golang.org/x/net/http/httpguts" ) +// NativeCodexResultError carries a result code returned by the native Codex +// runtime (routing_stale, ticket_missing, ...) and the runtime's own reason. +// It still matches ErrNativeCodexRuntimeUnavailable with errors.Is. +type NativeCodexResultError struct { + Operation string + Code string + Message string +} + +func (e *NativeCodexResultError) Error() string { + text := fmt.Sprintf("native Codex %s returned %s (%s)", e.Operation, e.Code, CodexTicketFailure(e.Code).Message) + if e.Message != "" { + text += ": " + e.Message + } + return text +} + +func (e *NativeCodexResultError) Unwrap() error { return ErrNativeCodexRuntimeUnavailable } + func (r *NativeCodexRuntime) ApplyRequestHeaders(ctx context.Context, account *Account, model string, headers http.Header) error { if account == nil || headers == nil || !account.IsOpenAIOAuthLike() { return nil @@ -28,11 +47,16 @@ func (r *NativeCodexRuntime) ApplyRequestHeaders(ctx context.Context, account *A return err } var changes extensionv1.CodexRoutingInjection - if result.Code != "" || json.Unmarshal(result.Payload, &changes) != nil { - return ErrNativeCodexRuntimeUnavailable + if result.Code != "" { + return &NativeCodexResultError{Operation: "inject", Code: result.Code, Message: result.Message} } - if changes.Qualification != nil && !changes.Qualification.Valid(time.Now(), account.ID, CodexTicketAccountIdentity(account), model) { - return ErrNativeCodexRuntimeUnavailable + if err := json.Unmarshal(result.Payload, &changes); err != nil { + return fmt.Errorf("%w: decode the inject result: %v", ErrNativeCodexRuntimeUnavailable, err) + } + if changes.Qualification != nil { + if problem := changes.Qualification.Problem(time.Now(), account.ID, CodexTicketAccountIdentity(account), model); problem != "" { + return fmt.Errorf("%w: inject returned an unusable route qualification: %s", ErrNativeCodexRuntimeUnavailable, problem) + } } projected, seen := headers.Clone(), map[string]bool{} for name, value := range changes.Headers { @@ -43,7 +67,7 @@ func (r *NativeCodexRuntime) ApplyRequestHeaders(ctx context.Context, account *A blocked = true } if blocked || seen[lower] || !httpguts.ValidHeaderFieldName(name) || !httpguts.ValidHeaderFieldValue(value) { - return errors.New("invalid native Codex request header") + return fmt.Errorf("invalid native Codex request header %q (blocked=%t, duplicate=%t)", name, blocked, seen[lower]) } seen[lower] = true projected.Set(name, value) diff --git a/backend/internal/service/codex_quality_health.go b/backend/internal/service/codex_quality_health.go index 21c1f4a06246..a8d05caef030 100644 --- a/backend/internal/service/codex_quality_health.go +++ b/backend/internal/service/codex_quality_health.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "time" ) @@ -12,47 +13,61 @@ type codexQualityBreakerReader interface { } func (s *OpenAIGatewayService) codexQualityHealthBlocked(ctx context.Context, account *Account) bool { + return s.codexQualityHealthBlock(ctx, account) != "" +} + +// codexQualityHealthBlock names the ordinary health state that blocks a +// diagnostic send, or returns "" when none does. It only reads that state. +func (s *OpenAIGatewayService) codexQualityHealthBlock(ctx context.Context, account *Account) string { now := time.Now() if value, exists := s.openaiAccountRuntimeBlockUntil.Load(account.ID); exists { until, valid := value.(time.Time) - if !valid || until.IsZero() || now.Before(until) { - return true + switch { + case !valid: + return fmt.Sprintf("account runtime block holds an unreadable value %v", value) + case until.IsZero(): + return "account runtime block has no end time" + case now.Before(until): + return "account runtime block until " + until.UTC().Format(time.RFC3339) } } model := openAIAccountModelTransientModel(codexQualityModel) state := s.getOpenAIAccountModelTransientState() state.mu.Lock() entry := state.entries[openAIAccountModelKey{AccountID: account.ID, Model: model}] - modelBlocked := now.Before(entry.blockUntil) state.mu.Unlock() - if modelBlocked { - return true + if now.Before(entry.blockUntil) { + return fmt.Sprintf("model %s transient block until %s (failure streak %d)", model, entry.blockUntil.UTC().Format(time.RFC3339), entry.failureStreak) } if proxyID, exists := openAIProxyStreamCircuitProxyID(account); exists { circuit := s.getOpenAIProxyStreamCircuit() circuit.mu.Lock() - blocked := !circuit.settings.disabled && now.Before(circuit.entries[proxyID].blockedUntil) + disabled := circuit.settings.disabled + blockedUntil := circuit.entries[proxyID].blockedUntil circuit.mu.Unlock() - if blocked { - return true + if !disabled && now.Before(blockedUntil) { + return fmt.Sprintf("proxy %d stream circuit is open until %s", proxyID, blockedUntil.UTC().Format(time.RFC3339)) } } if s.cache != nil { reader, ok := s.cache.(codexQualityBreakerReader) if !ok { - return true + return "the runtime breaker state cannot be read without claiming a probe" } blocked, err := reader.CodexQualityRuntimeBlocked(ctx, account.ID, []string{"", model}) - if err != nil || blocked { - return true + if err != nil { + return "read the runtime breaker state: " + err.Error() + } + if blocked { + return fmt.Sprintf("the runtime breaker of account %d is open or half-open for %s", account.ID, model) } } if s.rateLimitService != nil && s.rateLimitService.settingService != nil { thresholds := s.rateLimitService.settingService.GetAccountSchedulingThresholds(ctx) decision := EvaluateAccountSchedulingThreshold(account, thresholds, now) if decision.ShouldPause && decision.Until != nil && now.Before(*decision.Until) { - return true + return fmt.Sprintf("scheduling threshold pause until %s (%s %s window %s: %.1f%% used, threshold %d%%)", decision.Until.UTC().Format(time.RFC3339), decision.Platform, decision.Scope, decision.Window, decision.UsedPercent, decision.ThresholdPercent) } } - return false + return "" } diff --git a/backend/internal/service/codex_quality_manual_stop_test.go b/backend/internal/service/codex_quality_manual_stop_test.go index 27c8def2d53a..9e3d4ae8e80a 100644 --- a/backend/internal/service/codex_quality_manual_stop_test.go +++ b/backend/internal/service/codex_quality_manual_stop_test.go @@ -57,6 +57,14 @@ func TestCodexQualityManualStopRequiredBeforeCreateRenewAndSend(t *testing.T) { require.False(t, account.Schedulable, "the eligibility copy must not enable the persisted account") run, revision, err := readCodexQualityRun(ctx, store, request.RunID) require.NoError(t, err) + // Any administrator reads the run; only its creator closes or renews it. + view, err := s.ReadCodexQualityRun(ctx, key.UserID+1, account.ID, request.RunID) + require.NoError(t, err) + require.Equal(t, key.UserID, view.ActorID) + _, err = s.CloseCodexQualityRun(ctx, key.UserID+1, account.ID, request.RunID) + require.ErrorContains(t, err, "only its creator can close it") + _, err = s.RenewCodexQualityRoute(ctx, key.UserID+1, account.ID, request.RunID, uuid.NewString(), lookup) + require.ErrorContains(t, err, "only its creator can renew") // Simulate an administrator turning ordinary scheduling back on while the // existing diagnostic run and its downstream grant remain unexpired. diff --git a/backend/internal/service/codex_quality_observation.go b/backend/internal/service/codex_quality_observation.go index 514bac6ddc5b..939a0ca5eee2 100644 --- a/backend/internal/service/codex_quality_observation.go +++ b/backend/internal/service/codex_quality_observation.go @@ -3,12 +3,14 @@ package service import ( "bytes" "context" + "fmt" "io" "net/http" "slices" "strings" "sync" "time" + "unicode/utf8" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" "github.com/klauspost/compress/zstd" @@ -17,7 +19,7 @@ import ( func codexQualityWireFields(req *http.Request) (string, string, error) { if req == nil { - return "", "", ErrCodexQualityUnavailable + return "", "", codexQualityUnavailable("no outbound request") } // Reasoning recovery deliberately clears GetBody to prevent transparent POST // replay. Inspect the final encoded body without restoring that capability. @@ -30,11 +32,11 @@ func codexQualityWireFields(req *http.Request) (string, string, error) { if body != nil { _ = body.Close() } - return "", "", ErrCodexQualityUnavailable + return "", "", codexQualityUnavailable("copy the outbound body: %v", err) } } if body == nil { - return "", "", ErrCodexQualityUnavailable + return "", "", codexQualityUnavailable("the outbound request has no body") } wire, readErr := io.ReadAll(io.LimitReader(body, (2<<20)+1)) closeErr := body.Close() @@ -44,7 +46,13 @@ func codexQualityWireFields(req *http.Request) (string, string, error) { // unread suffix may become a later request after a failed snapshot. req.Body = io.NopCloser(codexQualityRejectedWireBody{}) } - return "", "", ErrCodexQualityUnavailable + switch { + case readErr != nil: + return "", "", codexQualityUnavailable("read the outbound body: %v", readErr) + case closeErr != nil: + return "", "", codexQualityUnavailable("close the outbound body: %v", closeErr) + } + return "", "", codexQualityUnavailable("the outbound body exceeds 2 MiB") } if !independent { req.Body = io.NopCloser(bytes.NewReader(wire)) @@ -53,14 +61,19 @@ func codexQualityWireFields(req *http.Request) (string, string, error) { if strings.EqualFold(req.Header.Get("Content-Encoding"), "zstd") { decoder, err := zstd.NewReader(reader, zstd.WithDecoderMaxMemory(8<<20), zstd.WithDecoderConcurrency(1)) if err != nil { - return "", "", ErrCodexQualityUnavailable + return "", "", codexQualityUnavailable("zstd decoder: %v", err) } defer decoder.Close() reader = decoder } raw, err := io.ReadAll(io.LimitReader(reader, (1<<20)+1)) - if err != nil || len(raw) > 1<<20 || !gjson.ValidBytes(raw) { - return "", "", ErrCodexQualityUnavailable + switch { + case err != nil: + return "", "", codexQualityUnavailable("decode the outbound body: %v", err) + case len(raw) > 1<<20: + return "", "", codexQualityUnavailable("the decoded outbound body exceeds 1 MiB") + case !gjson.ValidBytes(raw): + return "", "", codexQualityUnavailable("the outbound body is not valid JSON") } return gjson.GetBytes(raw, "model").String(), gjson.GetBytes(raw, "reasoning.effort").String(), nil } @@ -77,7 +90,7 @@ func reserveCodexQualityAcquisition(req *http.Request, accountID int64) error { return nil } if e.accountID != accountID || e.stage != "acquire" { - return ErrCodexQualityUnavailable + return codexQualityUnavailable("acquisition for account %d does not match the %s stage of account %d", accountID, e.stage, e.accountID) } run, err := e.current(req.Context()) if err != nil { @@ -98,7 +111,7 @@ func reserveCodexQualitySend(req *http.Request, account *Account, q *extensionv1 return nil } if account == nil || account.ID != e.accountID { - return ErrCodexQualityUnavailable + return codexQualityUnavailable("the outbound account is not account %d of the quality run", e.accountID) } run, err := e.current(req.Context()) if err != nil { @@ -108,13 +121,21 @@ func reserveCodexQualitySend(req *http.Request, account *Account, q *extensionv1 return err } model, effort, err := codexQualityWireFields(req) - if err != nil || model != codexQualityModel || effort != codexQualityEffort { - return ErrCodexQualityUnavailable + if err != nil { + return err + } + if model != codexQualityModel || effort != codexQualityEffort { + return codexQualityUnavailable("the outbound body asks for model %q effort %q; a quality diagnosis sends %s with %s", model, effort, codexQualityModel, codexQualityEffort) } if e.stage == "business" { current, err := e.runtime.qualification(req.Context(), run) - if err != nil || q == nil || current.Bundle.Key != q.Bundle.Key || current.Bundle.Revision != q.Bundle.Revision { - return ErrCodexQualityUnavailable + switch { + case err != nil: + return err + case q == nil: + return codexQualityUnavailable("the business request carries no route qualification") + case current.Bundle.Key != q.Bundle.Key || current.Bundle.Revision != q.Bundle.Revision: + return codexQualityUnavailable("the route changed to bundle %s revision %d while the request used %s revision %d", current.Bundle.Key, current.Bundle.Revision, q.Bundle.Key, q.Bundle.Revision) } } e.requestModel, e.effort, e.qualification = model, effort, q @@ -190,10 +211,16 @@ func observeCodexQualityResponse(ctx context.Context, response *http.Response, e a := e.attempt() if err != nil || response == nil { a.State, a.ErrorCode = "unknown", "transport" + if err != nil { + a.ErrorMessage = err.Error() + } else { + a.ErrorMessage = "the upstream returned no response" + } finishCodexQualityAttempt(e.runtime.ctx(ctx), e.runtime.store, e.runID, a) return } a.HTTPStatus = response.StatusCode + a.RequestID, a.CFRay = response.Header.Get("x-request-id"), response.Header.Get("cf-ray") observedAt := time.Now().UTC() deleted := false if e.stage == "business" && q != nil { @@ -235,29 +262,34 @@ func observeCodexQualityResponse(ctx context.Context, response *http.Response, e response.Body = observer } -func qualitySafeModel(model string) string { +// codexQualityDistinctModels bounds how many distinct declared models one +// attempt keeps; codexQualityModelNameLimit bounds one declared name. +const ( + codexQualityDistinctModels = 32 + codexQualityModelNameLimit = 256 +) + +// qualityRecordedModel keeps the upstream model declaration verbatim (only +// surrounding whitespace is trimmed); detecting a replaced model needs the name. +// A name over the limit is cut on a rune boundary and marked with its length. +func qualityRecordedModel(model string) string { model = strings.TrimSpace(model) - if model == "" { - return "" + if len(model) <= codexQualityModelNameLimit { + return model } - if len(model) > 128 || !strings.HasPrefix(model, "gpt-") { - return "other" + cut := codexQualityModelNameLimit + for cut > 0 && !utf8.RuneStart(model[cut]) { + cut-- } - for _, r := range model { - if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("._-", r) { - continue - } - return "other" - } - return model + return fmt.Sprintf("%s…(truncated, %d bytes)", model[:cut], len(model)) } func recordCodexQualityHeaderModel(attempt *CodexQualityAttempt, value string) { - model := qualitySafeModel(value) + model := qualityRecordedModel(value) if model == "" { return } - if !slices.Contains(attempt.HeaderModels, model) && len(attempt.HeaderModels) < 8 { + if !slices.Contains(attempt.HeaderModels, model) && len(attempt.HeaderModels) < codexQualityDistinctModels { attempt.HeaderModels = append(attempt.HeaderModels, model) } if len(attempt.HeaderModels) == 1 { @@ -285,6 +317,7 @@ func (s *OpenAIGatewayService) newCodexQualityObservedBody(response *http.Respon type codexQualityObservedBody struct { io.ReadCloser + detail string sse, sniff, failed, oversized bool pending []byte scanFrom, maxEventBytes int @@ -317,6 +350,7 @@ func (b *codexQualityObservedBody) feed(raw []byte) { b.readBytes += int64(len(raw)) if b.readBytes > b.maxReadBytes { b.oversized, b.failed, b.pending = true, true, nil + b.detail = fmt.Sprintf("the response exceeded the %d-byte observation limit", b.maxReadBytes) return } } @@ -345,6 +379,7 @@ func (b *codexQualityObservedBody) feed(raw []byte) { } if int64(i) > limit { b.oversized, b.failed, b.pending = true, true, nil + b.detail = fmt.Sprintf("an SSE event of %d bytes exceeded the %d-byte limit", i, limit) return } var data [][]byte @@ -360,10 +395,22 @@ func (b *codexQualityObservedBody) feed(raw []byte) { b.scanFrom = max(0, len(b.pending)-2) } if int64(len(b.pending)) > limit { + b.detail = fmt.Sprintf("an unterminated event of %d bytes exceeded the %d-byte limit", len(b.pending), limit) b.oversized, b.failed, b.pending = true, true, nil } } +// recordUpstream keeps the first upstream failure: its error object and the +// failing event or body (bounded). +func (b *codexQualityObservedBody) recordUpstream(found codexUpstreamError, raw []byte) { + a := &b.attempt + if a.UpstreamBody != "" || a.UpstreamErrorType != "" || a.UpstreamErrorCode != "" || a.UpstreamErrorMessage != "" { + return + } + a.UpstreamErrorType, a.UpstreamErrorCode, a.UpstreamErrorMessage, a.UpstreamErrorParam = found.Type, found.Code, found.Message, found.Param + a.UpstreamBody = string(boundedCodexUpstreamBody(raw)) +} + func (b *codexQualityObservedBody) event(raw []byte) { if !gjson.ValidBytes(raw) { return @@ -391,8 +438,8 @@ func (b *codexQualityObservedBody) event(raw []byte) { if !r.Exists() { r = root } - model := qualitySafeModel(r.Get("model").String()) - if model != "" && !slices.Contains(b.attempt.ResponseModels, model) && len(b.attempt.ResponseModels) < 8 { + model := qualityRecordedModel(r.Get("model").String()) + if model != "" && !slices.Contains(b.attempt.ResponseModels, model) && len(b.attempt.ResponseModels) < codexQualityDistinctModels { b.attempt.ResponseModels = append(b.attempt.ResponseModels, model) } if kind == "response.created" && model != "" { @@ -401,6 +448,17 @@ func (b *codexQualityObservedBody) event(raw []byte) { if kind == "error" || kind == "response.failed" || kind == "response.incomplete" || (root.Get("error").Exists() && root.Get("error").Type != gjson.Null) { b.failed = true b.attempt.ErrorCode = "upstream_error" + found := codexUpstreamErrorObject(root.Get("error")) + if found.empty() { + found = codexUpstreamErrorObject(r.Get("error")) + } + if found.empty() && kind == "error" { + found = codexUpstreamError{Code: codexJSONText(root.Get("code")), Message: codexJSONText(root.Get("message")), Param: codexJSONText(root.Get("param"))} + } + if found.empty() && kind == "response.incomplete" { + found = codexUpstreamError{Type: "incomplete", Code: codexJSONText(r.Get("incomplete_details.reason"))} + } + b.recordUpstream(found, raw) } if kind == "response.completed" || (root.Get("object").String() == "response" && r.Get("status").String() == "completed") { if model != "" { @@ -425,6 +483,10 @@ func (b *codexQualityObservedBody) complete(err error) { b.mu.Lock() if !b.sse && !b.oversized { b.event(b.pending) + if b.attempt.HTTPStatus >= 400 && len(b.pending) > 0 { + // Non-streaming error bodies may carry {"detail":...} or plain text. + b.recordUpstream(codexUpstreamErrorFromBody(b.pending), b.pending) + } } b.attempt.Completed = b.attempt.Completed && !b.failed && b.attempt.HTTPStatus == http.StatusOK b.attempt.State = "unknown" @@ -432,10 +494,15 @@ func (b *codexQualityObservedBody) complete(err error) { b.attempt.State = "complete" } else if b.failed || b.attempt.HTTPStatus >= 400 { b.attempt.State, b.attempt.ErrorCode = "error", "upstream_error" + if b.attempt.ErrorMessage == "" { + b.attempt.ErrorMessage = b.detail + } } else if err != nil && err != io.EOF { b.attempt.ErrorCode = "stream_error" + b.attempt.ErrorMessage = err.Error() } else { b.attempt.ErrorCode = "incomplete" + b.attempt.ErrorMessage = "the response ended without a completed terminal event" } b.done, b.pending = true, nil attempt := b.attempt diff --git a/backend/internal/service/codex_quality_observation_test.go b/backend/internal/service/codex_quality_observation_test.go index 259915dd6f8e..962da466baaf 100644 --- a/backend/internal/service/codex_quality_observation_test.go +++ b/backend/internal/service/codex_quality_observation_test.go @@ -34,7 +34,13 @@ func TestCodexQualityObservationClassifiesStreamFailures(t *testing.T) { require.Equal(t, test.code, observation.Code) encoded, err := json.Marshal(observation) require.NoError(t, err) - require.NotContains(t, string(encoded), "private upstream text") + // Classification uses explicit codes only, but the upstream's own text + // and the failing event are kept for administrators. + if strings.Contains(test.payload, "private upstream text") { + require.Contains(t, observation.UpstreamErrorMessage, "private upstream text") + require.JSONEq(t, test.payload, observation.UpstreamBody, "the failure summary keeps the event type and error object") + require.Contains(t, string(encoded), "private upstream text") + } }) } var cancelled extensionv1.CodexRoutingObservation diff --git a/backend/internal/service/codex_quality_route.go b/backend/internal/service/codex_quality_route.go index 31e37813ba73..4ecd93d57a06 100644 --- a/backend/internal/service/codex_quality_route.go +++ b/backend/internal/service/codex_quality_route.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" @@ -12,13 +13,17 @@ import ( func (s *OpenAIGatewayService) RenewCodexQualityRoute(ctx context.Context, actor, accountID int64, id, operation string, lookup CodexQualityKeyLookup) (*CodexQualityRunView, error) { canonical, valid := canonicalCodexQualityID(operation) - if !valid || lookup == nil { - return nil, ErrCodexQualityUnavailable + if !valid { + return nil, codexQualityUnavailable("operation_id %q is not a canonical UUID", operation) + } + if lookup == nil { + return nil, codexQualityUnavailable("no API key lookup for the route renewal") } operation = canonical + requested := id id, valid = canonicalCodexQualityID(id) if !valid { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("run id %q is not a canonical UUID", requested) } rt, err := s.codexQualityRuntime() if err != nil { @@ -28,12 +33,24 @@ func (s *OpenAIGatewayService) RenewCodexQualityRoute(ctx context.Context, actor ctx, cancel := context.WithTimeout(ctx, 65*time.Second) defer cancel() run, _, err := readCodexQualityRun(ctx, rt.store, id) - if err != nil || run.ActorID != actor || run.AccountID != accountID || !codexQualityActive(run, time.Now()) { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, err + case run.RunID == "": + return nil, codexQualityUnavailable("quality run %s not found", id) + case run.ActorID != actor: + return nil, codexQualityUnavailable("quality run %s was created by administrator %d; only its creator can renew its route with that administrator's API key", run.RunID, run.ActorID) + case run.AccountID != accountID: + return nil, codexQualityUnavailable("quality run %s belongs to account %d, not %d", run.RunID, run.AccountID, accountID) + case !codexQualityActive(run, time.Now()): + return nil, codexQualityUnavailable("%s", codexQualityInactiveReason(run, time.Now())) } key, err := lookup(ctx, run.APIKeyID) - if err != nil || !codexQualityKeyUsable(key, actor, run.APIKeyID, run.GroupID) { - return nil, ErrCodexQualityUnavailable + if err != nil { + return nil, codexQualityUnavailable("look up API key %d: %v", run.APIKeyID, err) + } + if !codexQualityKeyUsable(key, actor, run.APIKeyID, run.GroupID) { + return nil, codexQualityUnavailable("API key %d is no longer usable for quality run %s", run.APIKeyID, run.RunID) } for _, a := range run.Attempts { if a.OperationID == operation { @@ -45,17 +62,23 @@ func (s *OpenAIGatewayService) RenewCodexQualityRoute(ctx context.Context, actor return nil, err } if s.concurrencyService == nil { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("no concurrency service") } concurrency, err := s.concurrencyService.AcquireAccountSlot(ctx, accountID, account.Concurrency) - if err != nil || concurrency == nil || !concurrency.Acquired { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, codexQualityUnavailable("acquire a concurrency slot of account %d: %v", accountID, err) + case concurrency == nil || !concurrency.Acquired: + return nil, codexQualityUnavailable("account %d has no free concurrency slot (limit %d)", accountID, account.Concurrency) } defer concurrency.ReleaseFunc() lease := extensionv1.LeaseRequest{Namespace: "routing-account", Key: fmt.Sprint(accountID), Owner: uuid.NewString(), TTLSeconds: 75} locked, err := rt.store.AcquireExtensionLease(ctx, codexRuntimePluginKey, lease) - if err != nil || !locked.Acquired { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, codexQualityUnavailable("acquire the routing lease of account %d: %v", accountID, err) + case !locked.Acquired: + return nil, codexQualityUnavailable("another routing operation holds account %d", accountID) } lease.Generation = locked.Generation defer func() { @@ -70,24 +93,35 @@ func (s *OpenAIGatewayService) RenewCodexQualityRoute(ctx context.Context, actor raw, _ := json.Marshal(query) result, callErr := rt.host.Call(stageCtx, extensionv1.HostInvocation{Operation: extensionv1.HostCodexRoutingProbe, Payload: raw}) var out extensionv1.CodexRoutingProbeResult - if callErr != nil || result.Code != "" || json.Unmarshal(result.Payload, &out) != nil { - return out, ErrCodexQualityUnavailable + switch { + case callErr != nil: + return out, codexQualityUnavailable("%s probe: %v", query.Stage, callErr) + case result.Code != "": + return out, codexQualityUnavailable("%s probe returned %s: %s", query.Stage, result.Code, result.Message) + } + if err := json.Unmarshal(result.Payload, &out); err != nil { + return out, codexQualityUnavailable("decode %s probe result: %v", query.Stage, err) } return out, nil } acquired, err := call() if err != nil || !acquired.Valid || acquired.Bundle == nil { - return s.ReadCodexQualityRun(ctx, actor, accountID, id) + return s.codexQualityRenewalFailure(ctx, actor, accountID, id, codexQualityRenewalReason("acquire", err, acquired, run.Scope), acquired.Observation) } query.Stage, query.Bundle = "verify", acquired.Bundle verified, err := call() if err != nil || !verified.Valid || verified.Bundle == nil || !verified.Observation.Completed || !verified.Observation.ModelMatched || verified.Observation.ResponseModel != codexQualityModel || verified.Scope.ConnectionLeaseID == "" || !run.Scope.SameOwner(verified.Scope) { - return s.ReadCodexQualityRun(ctx, actor, accountID, id) + return s.codexQualityRenewalFailure(ctx, actor, accountID, id, codexQualityRenewalReason("verify", err, verified, run.Scope), acquired.Observation, verified.Observation) } q := &extensionv1.CodexRoutingQualification{Scope: verified.Scope, Bundle: *verified.Bundle, Model: codexQualityModel, VerifiedAt: time.Now().UTC(), ExpiresAt: verified.Bundle.ExpiresAt} saved, err := mutateCodexQualityRun(ctx, rt.store, id, func(current *codexQualityRun) error { - if !codexQualityActive(*current, time.Now()) || current.ActorID != actor || !current.Scope.SameOwner(q.Scope) { - return ErrCodexQualityUnavailable + switch { + case !codexQualityActive(*current, time.Now()): + return codexQualityUnavailable("%s", codexQualityInactiveReason(*current, time.Now())) + case current.ActorID != actor: + return codexQualityUnavailable("quality run %s was created by administrator %d", current.RunID, current.ActorID) + case !current.Scope.SameOwner(q.Scope): + return codexQualityUnavailable("verified route %s", codexRoutingScopeChange(current.Scope, q.Scope)) } current.Qualification, current.RouteRuntimeGeneration = q, rt.installation.RuntimeGeneration current.RouteGeneration++ @@ -104,6 +138,55 @@ func (s *OpenAIGatewayService) RenewCodexQualityRoute(ctx context.Context, actor return codexQualityView(saved, rt.installation.RuntimeGeneration), nil } +// codexQualityRenewalFailure returns the unchanged run together with the reason +// the renewal did not install a route and the observations of its stages. +func (s *OpenAIGatewayService) codexQualityRenewalFailure(ctx context.Context, actor, accountID int64, id, reason string, observations ...extensionv1.CodexRoutingObservation) (*CodexQualityRunView, error) { + view, err := s.ReadCodexQualityRun(ctx, actor, accountID, id) + if err != nil { + return nil, fmt.Errorf("%w (route renewal failed: %s)", err, reason) + } + view.RenewalError = reason + for _, observation := range observations { + if observation.Code != "" || !observation.ObservedAt.IsZero() { + view.RenewalObservations = append(view.RenewalObservations, observation) + } + } + return view, nil +} + +func codexQualityRenewalReason(stage string, err error, result extensionv1.CodexRoutingProbeResult, owner extensionv1.CodexRoutingScope) string { + if err != nil { + return err.Error() + } + observation := result.Observation + var reasons []string + switch { + case !result.Valid: + reasons = append(reasons, "the probe result is not valid") + case result.Bundle == nil: + reasons = append(reasons, "the probe stored no route bundle") + } + if stage == "verify" { + if !observation.Completed { + reasons = append(reasons, "the response did not complete") + } + if !observation.ModelMatched || observation.ResponseModel != codexQualityModel { + reasons = append(reasons, fmt.Sprintf("the upstream declared model %q instead of %s", observation.ResponseModel, codexQualityModel)) + } + if result.Scope.ConnectionLeaseID == "" { + reasons = append(reasons, "no connection lease was bound") + } + if !owner.SameOwner(result.Scope) { + reasons = append(reasons, codexRoutingScopeChange(owner, result.Scope)) + } + } + reason := fmt.Sprintf("%s stage ended with %s: %s", stage, observation.Code, strings.Join(reasons, "; ")) + if summary := observation.Summary(); summary != "" { + reason += " (" + summary + ")" + } + return reason +} + func codexQualityRenewQuery(run codexQualityRun, operation string) extensionv1.CodexRoutingQuery { return extensionv1.CodexRoutingQuery{ AccountID: run.AccountID, Model: codexQualityModel, ReasoningEffort: codexQualityEffort, diff --git a/backend/internal/service/codex_quality_runtime.go b/backend/internal/service/codex_quality_runtime.go index acd5534a0afb..e790e2b32dab 100644 --- a/backend/internal/service/codex_quality_runtime.go +++ b/backend/internal/service/codex_quality_runtime.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "slices" @@ -89,12 +90,12 @@ func StageCodexQualityHeaders(c *gin.Context) { func (s *OpenAIGatewayService) codexQualityRuntime() (*codexQualityRuntime, error) { if s == nil || s.nativeCodexRuntime == nil || s.accountRepo == nil { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("native Codex runtime is not configured") } runtime := s.nativeCodexRuntime metadata, err := runtime.repo.LoadNativeCodexMetadata(context.Background()) if err != nil { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("load Codex runtime metadata: %v", err) } snapshot := runtime.current() var host *nativeCodexHost @@ -110,13 +111,22 @@ func (rt *codexQualityRuntime) ctx(ctx context.Context) context.Context { func (rt *codexQualityRuntime) scope(ctx context.Context, id int64) (extensionv1.CodexRoutingScope, error) { if rt.host == nil { - return extensionv1.CodexRoutingScope{}, ErrCodexQualityUnavailable + return extensionv1.CodexRoutingScope{}, codexQualityUnavailable("Codex runtime host is not loaded") } raw, _ := json.Marshal(extensionv1.CodexRoutingQuery{AccountID: id, Transport: "http"}) result, err := rt.host.Call(rt.ctx(ctx), extensionv1.HostInvocation{Operation: extensionv1.HostCodexRoutingScope, Payload: raw}) var scope extensionv1.CodexRoutingScope - if err != nil || result.Code != "" || json.Unmarshal(result.Payload, &scope) != nil || scope.AccountID != id { - return scope, ErrCodexQualityUnavailable + switch { + case err != nil: + return scope, codexQualityUnavailable("routing scope of account %d: %v", id, err) + case result.Code != "": + return scope, codexQualityUnavailable("routing scope of account %d returned %s: %s", id, result.Code, result.Message) + } + if err := json.Unmarshal(result.Payload, &scope); err != nil { + return scope, codexQualityUnavailable("decode routing scope of account %d: %v", id, err) + } + if scope.AccountID != id { + return scope, codexQualityUnavailable("routing scope names account %d instead of %d", scope.AccountID, id) } return scope, nil } @@ -132,40 +142,70 @@ func (rt *codexQualityRuntime) account(ctx context.Context, run codexQualityRun) a, err := rt.s.accountRepo.GetByID(ctx, run.AccountID) // A diagnostic grant never admits an account enabled for ordinary traffic. // Recheck the authoritative row here, including immediately before each send. - if err != nil || a == nil || a.Schedulable || a.ID != run.AccountID || a.Platform != PlatformOpenAI || a.Type != AccountTypeOAuth || a.IsShadow() || !slices.Contains(a.GroupIDs, run.GroupID) || qualityProxyID(a) != run.ProxyID || CodexTicketAccountIdentity(a) != run.Scope.Identity { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, codexQualityUnavailable("read account %d: %v", run.AccountID, err) + case a == nil || a.ID != run.AccountID: + return nil, codexQualityUnavailable("account %d not found", run.AccountID) + case a.Schedulable: + return nil, codexQualityUnavailable("account %d is schedulable; stop its scheduling manually before a quality diagnosis", a.ID) + case a.Platform != PlatformOpenAI || a.Type != AccountTypeOAuth || a.IsShadow(): + return nil, codexQualityUnavailable("account %d (%s/%s, shadow=%t) is not an OpenAI OAuth account", a.ID, a.Platform, a.Type, a.IsShadow()) + case !slices.Contains(a.GroupIDs, run.GroupID): + return nil, codexQualityUnavailable("account %d is not in group %d of the API key (account groups %v)", a.ID, run.GroupID, a.GroupIDs) + case qualityProxyID(a) != run.ProxyID: + return nil, codexQualityUnavailable("account %d proxy is %d, the run was created with proxy %d", a.ID, qualityProxyID(a), run.ProxyID) + case CodexTicketAccountIdentity(a) != run.Scope.Identity: + return nil, codexQualityUnavailable("account %d credential owner changed since the run was created", a.ID) } plan := strings.ToLower(strings.TrimSpace(a.GetCredential("plan_type"))) if plan != "pro" && plan != "chatgpt_pro" { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("account %d plan_type is %q; quality diagnosis requires pro", a.ID, plan) } - if rt.s.checkChannelPricingRestriction(ctx, &run.GroupID, codexQualityModel) || (rt.s.needsUpstreamChannelRestrictionCheck(ctx, &run.GroupID) && rt.s.isUpstreamModelRestrictedByChannel(ctx, run.GroupID, a, codexQualityModel, false)) { - return nil, ErrCodexQualityUnavailable + if rt.s.checkChannelPricingRestriction(ctx, &run.GroupID, codexQualityModel) { + return nil, codexQualityUnavailable("channel pricing of group %d restricts %s", run.GroupID, codexQualityModel) + } + if rt.s.needsUpstreamChannelRestrictionCheck(ctx, &run.GroupID) && rt.s.isUpstreamModelRestrictedByChannel(ctx, run.GroupID, a, codexQualityModel, false) { + return nil, codexQualityUnavailable("the upstream channel of group %d restricts %s for account %d", run.GroupID, codexQualityModel, a.ID) } // This private stack copy changes only the manual switch for eligibility. // It is never returned, cached, projected, or persisted. check := *a check.Schedulable = true - if eligible, _ := openAICompatibleAccountEligibilityBeforeProfit(ctx, &check, PlatformOpenAI, codexQualityModel, false, OpenAIEndpointCapabilityResponses); !eligible { - return nil, ErrCodexQualityUnavailable + if eligible, reason := openAICompatibleAccountEligibilityBeforeProfit(ctx, &check, PlatformOpenAI, codexQualityModel, false, OpenAIEndpointCapabilityResponses); !eligible { + return nil, codexQualityUnavailable("account %d is not eligible for %s: %s", a.ID, codexQualityModel, reason) } - if rt.s.codexQualityHealthBlocked(ctx, &check) { - return nil, ErrCodexQualityUnavailable + if reason := rt.s.codexQualityHealthBlock(ctx, &check); reason != "" { + return nil, codexQualityUnavailable("account %d: %s", a.ID, reason) } if rt.s.openAIGroupRequiresPrivacySet(ctx, &run.GroupID) && !a.IsPrivacySet() { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("group %d requires the privacy setting, which account %d has not set", run.GroupID, a.ID) } scope, err := rt.scope(ctx, a.ID) - if err != nil || !run.Scope.SameOwner(scope) { - return nil, ErrCodexQualityUnavailable + if err != nil { + return nil, err + } + if !run.Scope.SameOwner(scope) { + return nil, codexQualityUnavailable("account %d %s", a.ID, codexRoutingScopeChange(run.Scope, scope)) } return a, nil } func (s *OpenAIGatewayService) CreateCodexQualityRun(ctx context.Context, actor, accountID int64, key *APIKey, request CodexQualityCreateRequest) (*CodexQualityRunView, error) { id, validID := canonicalCodexQualityID(request.RunID) - if !validID || !validCodexQualityHash(request.PromptSHA256) || actor <= 0 || key == nil || key.GroupID == nil || !codexQualityKeyUsable(key, actor, request.APIKeyID, *key.GroupID) { - return nil, ErrCodexQualityUnavailable + switch { + case !validID: + return nil, codexQualityUnavailable("run_id %q is not a canonical UUID", request.RunID) + case !validCodexQualityHash(request.PromptSHA256): + return nil, codexQualityUnavailable("prompt_sha256 must be 64 lowercase hex characters") + case actor <= 0: + return nil, codexQualityUnavailable("no authenticated administrator") + case key == nil: + return nil, codexQualityUnavailable("API key %d is not one of this administrator's keys", request.APIKeyID) + case key.GroupID == nil: + return nil, codexQualityUnavailable("API key %d has no group", key.ID) + case !codexQualityKeyUsable(key, actor, request.APIKeyID, *key.GroupID): + return nil, codexQualityUnavailable("API key %d is not usable (status %q, expired %t, quota exhausted %t)", key.ID, key.Status, key.IsExpired(), key.IsQuotaExhausted()) } request.RunID = id if request.MaxSends == 0 { @@ -174,8 +214,11 @@ func (s *OpenAIGatewayService) CreateCodexQualityRun(ctx context.Context, actor, if request.TTLSeconds == 0 { request.TTLSeconds = 7200 } - if request.MaxSends < 1 || request.MaxSends > codexQualityMaxSends || request.TTLSeconds < 60 || request.TTLSeconds > 7200 { - return nil, ErrCodexQualityUnavailable + if request.MaxSends < 1 || request.MaxSends > codexQualityMaxSends { + return nil, codexQualityUnavailable("max_sends must be 1 to %d, got %d", codexQualityMaxSends, request.MaxSends) + } + if request.TTLSeconds < 60 || request.TTLSeconds > 7200 { + return nil, codexQualityUnavailable("ttl_seconds must be 60 to 7200, got %d", request.TTLSeconds) } rt, err := s.codexQualityRuntime() if err != nil { @@ -183,8 +226,11 @@ func (s *OpenAIGatewayService) CreateCodexQualityRun(ctx context.Context, actor, } ctx = rt.ctx(ctx) a, err := s.accountRepo.GetByID(ctx, accountID) - if err != nil || a == nil { - return nil, ErrCodexQualityUnavailable + if err != nil { + return nil, codexQualityUnavailable("read account %d: %v", accountID, err) + } + if a == nil { + return nil, codexQualityUnavailable("account %d not found", accountID) } scope, err := rt.scope(ctx, accountID) if err != nil { @@ -208,21 +254,28 @@ func (s *OpenAIGatewayService) CreateCodexQualityRun(ctx context.Context, actor, return view, nil } +// ReadCodexQualityRun is readable by every administrator, not only the one +// who created the run (the view names the creator in actor_id). func (s *OpenAIGatewayService) ReadCodexQualityRun(ctx context.Context, actor, accountID int64, id string) (*CodexQualityRunView, error) { rt, err := s.codexQualityRuntime() if err != nil { return nil, err } run, _, err := readCodexQualityRun(rt.ctx(ctx), rt.store, id) - if err != nil || run.RunID == "" || run.ActorID != actor || run.AccountID != accountID { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, err + case run.RunID == "": + return nil, codexQualityUnavailable("quality run %s not found", id) + case run.AccountID != accountID: + return nil, codexQualityUnavailable("quality run %s belongs to account %d, not %d", run.RunID, run.AccountID, accountID) } view := codexQualityView(run, rt.installation.RuntimeGeneration) if view.RouteReady { if _, err = rt.account(ctx, run); err != nil { - view.RouteReady = false + view.RouteReady, view.RouteError = false, err.Error() } else if _, err = rt.qualification(ctx, run); err != nil { - view.RouteReady = false + view.RouteReady, view.RouteError = false, err.Error() } } return view, nil @@ -234,8 +287,13 @@ func (s *OpenAIGatewayService) CloseCodexQualityRun(ctx context.Context, actor, return nil, err } run, err := mutateCodexQualityRun(rt.ctx(ctx), rt.store, id, func(run *codexQualityRun) error { - if run.RunID == "" || run.ActorID != actor || run.AccountID != accountID { - return ErrCodexQualityUnavailable + switch { + case run.RunID == "": + return codexQualityUnavailable("quality run %s not found", id) + case run.ActorID != actor: + return codexQualityUnavailable("quality run %s was created by administrator %d; only its creator can close it", run.RunID, run.ActorID) + case run.AccountID != accountID: + return codexQualityUnavailable("quality run %s belongs to account %d, not %d", run.RunID, run.AccountID, accountID) } run.Status, run.GrantDigest = "closed", "" return nil @@ -256,27 +314,37 @@ func (s *OpenAIGatewayService) AdmitCodexQualityRequest(c *gin.Context, key *API return nil } if len(lookups) != 1 || lookups[0] == nil { - return ErrCodexQualityUnavailable + return codexQualityUnavailable("no API key lookup for the quality grant") } id, secret, ok := strings.Cut(h.grant, ".") id, validID := canonicalCodexQualityID(id) trial, validTrial := canonicalCodexQualityID(h.trial) - if !ok || len(secret) != 43 || !validID || !validTrial || c.Request.Method != http.MethodPost || c.Request.URL.Path != "/v1/responses" || key == nil || key.GroupID == nil { - return ErrCodexQualityUnavailable + switch { + case !ok || len(secret) != 43 || !validID: + return codexQualityUnavailable("the %s header is not a valid grant (exactly one header is required)", CodexQualityGrantHeader) + case !validTrial: + return codexQualityUnavailable("the %s header must be exactly one canonical UUID", CodexQualityTrialHeader) + case c.Request.Method != http.MethodPost || c.Request.URL.Path != "/v1/responses": + return codexQualityUnavailable("quality grants apply only to POST /v1/responses, not %s %s", c.Request.Method, c.Request.URL.Path) + case key == nil || key.GroupID == nil: + return codexQualityUnavailable("the API key has no group") } h.trial = trial var fields map[string]json.RawMessage - if !validCodexQualityJSON(body) || json.Unmarshal(body, &fields) != nil { - return ErrCodexQualityUnavailable + if !validCodexQualityJSON(body) { + return codexQualityUnavailable("the request body must be at most 1 MiB of JSON without duplicate keys") + } + if err := json.Unmarshal(body, &fields); err != nil { + return codexQualityUnavailable("decode request body: %v", err) } for name := range fields { if !slices.Contains([]string{"model", "input", "reasoning", "stream", "store", "prompt_cache_key", "client_metadata"}, name) { - return ErrCodexQualityUnavailable + return codexQualityUnavailable("request field %q is not allowed in a quality diagnosis", name) } } input := gjson.GetBytes(body, "input") if input.Type != gjson.String || gjson.GetBytes(body, "model").String() != codexQualityModel || gjson.GetBytes(body, "reasoning.effort").String() != codexQualityEffort || !gjson.GetBytes(body, "stream").Bool() || gjson.GetBytes(body, "store").Bool() { - return ErrCodexQualityUnavailable + return codexQualityUnavailable("a quality diagnosis needs a string input, model %s, reasoning.effort %s, stream true and store false", codexQualityModel, codexQualityEffort) } rt, err := s.codexQualityRuntime() if err != nil { @@ -284,12 +352,21 @@ func (s *OpenAIGatewayService) AdmitCodexQualityRequest(c *gin.Context, key *API } run, _, err := readCodexQualityRun(rt.ctx(c.Request.Context()), rt.store, id) digest := codexQualityHash(h.grant) - if err != nil || !codexQualityActive(run, time.Now()) || run.APIKeyID != key.ID || run.ActorID != key.UserID || run.GroupID != *key.GroupID || run.PromptSHA256 != codexQualityHash(input.String()) || !codexQualityGrantMatches(run, digest) { - return ErrCodexQualityUnavailable + switch { + case err != nil: + return err + case !codexQualityActive(run, time.Now()): + return codexQualityUnavailable("%s", codexQualityInactiveReason(run, time.Now())) + case run.APIKeyID != key.ID || run.ActorID != key.UserID || run.GroupID != *key.GroupID: + return codexQualityUnavailable("quality run %s belongs to API key %d of user %d in group %d", run.RunID, run.APIKeyID, run.ActorID, run.GroupID) + case run.PromptSHA256 != codexQualityHash(input.String()): + return codexQualityUnavailable("the input does not match the prompt_sha256 of quality run %s", run.RunID) + case !codexQualityGrantMatches(run, digest): + return codexQualityUnavailable("the grant does not match quality run %s (it was reissued)", run.RunID) } for _, old := range run.Attempts { if old.Stage == "business" && old.TrialID == h.trial { - return ErrCodexQualitySpent + return fmt.Errorf("%w: trial %s was already sent", ErrCodexQualitySpent, h.trial) } } ctx := ensureOpenAIRuntimeBreakerProbeOwner(c.Request.Context()) @@ -360,30 +437,46 @@ func validCodexQualityJSON(body []byte) bool { func (rt *codexQualityRuntime) qualification(ctx context.Context, run codexQualityRun) (*extensionv1.CodexRoutingQualification, error) { q := run.Qualification - if !codexQualityActive(run, time.Now()) || q == nil || run.RouteRuntimeGeneration != rt.installation.RuntimeGeneration || !q.Valid(time.Now(), run.AccountID, run.Scope.Identity, codexQualityModel) || !q.Scope.SameOwner(run.Scope) || q.Scope.Transport != "http" || q.Scope.ConnectionLeaseID == "" { - return nil, ErrCodexQualityUnavailable + if reason := codexQualityRouteReason(run, rt.installation.RuntimeGeneration, time.Now()); reason != "" { + return nil, codexQualityUnavailable("%s", reason) } bundle, err := readCodexRoutingBundle(rt.ctx(ctx), rt.store, codexRuntimePluginKey, q.Bundle, q.Scope, true) - if err != nil || bundle.Model != codexQualityModel || bundle.Scope.ConnectionLeaseID != q.Scope.ConnectionLeaseID || bundle.Scope.Transport != q.Scope.Transport || q.ExpiresAt.After(bundle.ExpiresAt) { - return nil, ErrCodexQualityUnavailable - } - if rt.s.CheckCodexRoutingLease(ctx, q.Scope, q.ExpiresAt) != nil { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, codexQualityUnavailable("route bundle: %v", err) + case bundle.Model != codexQualityModel: + return nil, codexQualityUnavailable("route bundle model is %q, not %s", bundle.Model, codexQualityModel) + case bundle.Scope.ConnectionLeaseID != q.Scope.ConnectionLeaseID || bundle.Scope.Transport != q.Scope.Transport: + return nil, codexQualityUnavailable("route bundle is bound to %s connection %q, the run to %s connection %q", bundle.Scope.Transport, bundle.Scope.ConnectionLeaseID, q.Scope.Transport, q.Scope.ConnectionLeaseID) + case q.ExpiresAt.After(bundle.ExpiresAt): + return nil, codexQualityUnavailable("route expires at %s, after its bundle (%s)", q.ExpiresAt.UTC().Format(time.RFC3339), bundle.ExpiresAt.UTC().Format(time.RFC3339)) + } + if err := rt.s.CheckCodexRoutingLease(ctx, q.Scope, q.ExpiresAt); err != nil { + return nil, codexQualityUnavailable("connection lease %q: %v", q.Scope.ConnectionLeaseID, err) } return q, nil } func (e *codexQualityExecution) current(ctx context.Context) (codexQualityRun, error) { run, _, err := readCodexQualityRun(e.runtime.ctx(ctx), e.runtime.store, e.runID) - if err != nil || !codexQualityActive(run, time.Now()) || run.AccountID != e.accountID || (e.grantDigest != "" && !codexQualityGrantMatches(run, e.grantDigest)) { - return run, ErrCodexQualityUnavailable - } - if e.keyLookup == nil { - return run, ErrCodexQualityUnavailable + switch { + case err != nil: + return run, err + case !codexQualityActive(run, time.Now()): + return run, codexQualityUnavailable("%s", codexQualityInactiveReason(run, time.Now())) + case run.AccountID != e.accountID: + return run, codexQualityUnavailable("quality run %s belongs to account %d, not %d", run.RunID, run.AccountID, e.accountID) + case e.grantDigest != "" && !codexQualityGrantMatches(run, e.grantDigest): + return run, codexQualityUnavailable("the grant does not match quality run %s (it was reissued)", run.RunID) + case e.keyLookup == nil: + return run, codexQualityUnavailable("no API key lookup for quality run %s", run.RunID) } key, err := e.keyLookup(ctx, run.APIKeyID) - if err != nil || !codexQualityKeyUsable(key, run.ActorID, run.APIKeyID, run.GroupID) { - return run, ErrCodexQualityUnavailable + if err != nil { + return run, codexQualityUnavailable("look up API key %d: %v", run.APIKeyID, err) + } + if !codexQualityKeyUsable(key, run.ActorID, run.APIKeyID, run.GroupID) { + return run, codexQualityUnavailable("API key %d is no longer usable for quality run %s", run.APIKeyID, run.RunID) } return run, nil } @@ -391,7 +484,7 @@ func (e *codexQualityExecution) current(ctx context.Context) (codexQualityRun, e func (s *OpenAIGatewayService) SelectCodexQualityAccount(ctx context.Context) (*AccountSelectionResult, error) { e := codexQualityExecutionFromContext(ctx) if e == nil { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("not a quality diagnosis request") } run, err := e.current(ctx) if err != nil { @@ -405,19 +498,27 @@ func (s *OpenAIGatewayService) SelectCodexQualityAccount(ctx context.Context) (* return nil, err } if s.concurrencyService == nil { - return nil, ErrCodexQualityUnavailable + return nil, codexQualityUnavailable("no concurrency service") } acquired, err := s.concurrencyService.AcquireAccountSlot(ctx, a.ID, a.Concurrency) - if err != nil || acquired == nil || !acquired.Acquired { - return nil, ErrCodexQualityUnavailable + switch { + case err != nil: + return nil, codexQualityUnavailable("acquire a concurrency slot of account %d: %v", a.ID, err) + case acquired == nil || !acquired.Acquired: + return nil, codexQualityUnavailable("account %d has no free concurrency slot (limit %d)", a.ID, a.Concurrency) } return attachSelectionProfitGate(ctx, attachSelectionRuntimeBreakerProbe(ctx, &AccountSelectionResult{Account: a, Acquired: true, ReleaseFunc: acquired.ReleaseFunc})), nil } func codexQualityRequestQualification(ctx context.Context, account *Account, model string) (*extensionv1.CodexRoutingQualification, *NativeCodexMetadata, error) { e := codexQualityExecutionFromContext(ctx) - if e == nil || account == nil || account.ID != e.accountID || model != codexQualityModel { - return nil, nil, ErrCodexQualityUnavailable + switch { + case e == nil: + return nil, nil, codexQualityUnavailable("not a quality diagnosis request") + case account == nil || account.ID != e.accountID: + return nil, nil, codexQualityUnavailable("the selected account is not the account of the quality run") + case model != codexQualityModel: + return nil, nil, codexQualityUnavailable("model %q is not %s", model, codexQualityModel) } run, err := e.current(ctx) if err != nil { diff --git a/backend/internal/service/codex_quality_state.go b/backend/internal/service/codex_quality_state.go index d3631173d6a1..504659c20fa0 100644 --- a/backend/internal/service/codex_quality_state.go +++ b/backend/internal/service/codex_quality_state.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "strings" "time" @@ -24,10 +25,15 @@ const ( codexQualityMaxSends = 60 ) -// A diagnostic failure is deliberately small and never contains upstream text. +// Failures wrap these sentinels with the concrete precondition that failed; +// attempts record upstream error text separately (see CodexQualityAttempt). var ErrCodexQualityUnavailable = errors.New("codex_quality_unavailable") var ErrCodexQualitySpent = errors.New("codex_quality_attempt_spent") +func codexQualityUnavailable(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrCodexQualityUnavailable, fmt.Sprintf(format, args...)) +} + type CodexQualityCreateRequest struct { RunID string `json:"run_id"` APIKeyID int64 `json:"api_key_id"` @@ -54,6 +60,14 @@ type CodexQualityAttempt struct { Completed bool `json:"completed"` HTTPStatus int `json:"http_status,omitempty"` ErrorCode string `json:"error_code,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + UpstreamErrorType string `json:"upstream_error_type,omitempty"` + UpstreamErrorCode string `json:"upstream_error_code,omitempty"` + UpstreamErrorMessage string `json:"upstream_error_message,omitempty"` + UpstreamErrorParam string `json:"upstream_error_param,omitempty"` + UpstreamBody string `json:"upstream_body,omitempty"` + RequestID string `json:"request_id,omitempty"` + CFRay string `json:"cf_ray,omitempty"` ReasoningTokens *int64 `json:"reasoning_tokens"` InputTokens *int64 `json:"input_tokens"` OutputTokens *int64 `json:"output_tokens"` @@ -64,9 +78,11 @@ type CodexQualityAttempt struct { type CodexQualityRunView struct { RunID string `json:"run_id"` Grant string `json:"grant,omitempty"` + ActorID int64 `json:"actor_id"` AccountID int64 `json:"account_id"` APIKeyID int64 `json:"api_key_id"` GroupID int64 `json:"group_id"` + ProxyID int64 `json:"proxy_id"` Model string `json:"model"` ReasoningEffort string `json:"reasoning_effort"` PromptSHA256 string `json:"prompt_sha256"` @@ -78,11 +94,17 @@ type CodexQualityRunView struct { RouteExpiresAt *time.Time `json:"route_expires_at"` RouteGeneration int64 `json:"route_generation"` ConnectionFingerprint string `json:"connection_fingerprint,omitempty"` + RouteError string `json:"route_error,omitempty"` Attempts []CodexQualityAttempt `json:"attempts"` + // Set only on a renew-route reply whose renewal did not install a route. + RenewalError string `json:"renewal_error,omitempty"` + RenewalObservations []extensionv1.CodexRoutingObservation `json:"renewal_observations,omitempty"` } -// Stored only in the existing host-private namespace. No grant, key, prompt, -// response text, Cookie or STATE value is stored in this ledger. +// Stored only in the existing host-private namespace. No grant, API key, +// prompt, Cookie or STATE value is stored in this ledger. Attempts keep the +// upstream error object, transport/stream error text and the bounded failing +// event or body (which can quote response text) for administrators. type codexQualityRun struct { RunID string `json:"run_id"` ActorID int64 `json:"actor_id"` @@ -123,19 +145,25 @@ func codexQualityHash(value string) string { func readCodexQualityRun(ctx context.Context, store NativeCodexStateStore, id string) (codexQualityRun, int64, error) { var run codexQualityRun canonical, valid := canonicalCodexQualityID(id) - if !valid || store == nil { - return run, 0, ErrCodexQualityUnavailable + if !valid { + return run, 0, codexQualityUnavailable("run id %q is not a canonical UUID", id) + } + if store == nil { + return run, 0, codexQualityUnavailable("Codex runtime state store is unavailable") } id = canonical record, err := store.ReadExtensionState(ctx, codexRuntimePluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: codexQualityKey(id)}) if err != nil { - return run, 0, ErrCodexQualityUnavailable + return run, 0, codexQualityUnavailable("read quality run %s: %v", id, err) } if !record.Found { return run, 0, nil } - if json.Unmarshal(record.Value, &run) != nil || run.RunID != id || run.MaxSends < 1 || run.MaxSends > codexQualityMaxSends || run.UsedSends != len(run.Attempts) || run.UsedSends > run.MaxSends { - return codexQualityRun{}, 0, ErrCodexQualityUnavailable + if err := json.Unmarshal(record.Value, &run); err != nil { + return codexQualityRun{}, 0, codexQualityUnavailable("decode quality run %s: %v", id, err) + } + if run.RunID != id || run.MaxSends < 1 || run.MaxSends > codexQualityMaxSends || run.UsedSends != len(run.Attempts) || run.UsedSends > run.MaxSends { + return codexQualityRun{}, 0, codexQualityUnavailable("quality run record %s is inconsistent (run_id %q, max_sends %d, used_sends %d, attempts %d)", id, run.RunID, run.MaxSends, run.UsedSends, len(run.Attempts)) } return run, record.Revision, nil } @@ -143,7 +171,7 @@ func readCodexQualityRun(ctx context.Context, store NativeCodexStateStore, id st func mutateCodexQualityRun(ctx context.Context, store NativeCodexStateStore, id string, change func(*codexQualityRun) error) (codexQualityRun, error) { canonical, valid := canonicalCodexQualityID(id) if !valid { - return codexQualityRun{}, ErrCodexQualityUnavailable + return codexQualityRun{}, codexQualityUnavailable("run id %q is not a canonical UUID", id) } id = canonical for attempt := 0; attempt < 12; attempt++ { @@ -156,23 +184,23 @@ func mutateCodexQualityRun(ctx context.Context, store NativeCodexStateStore, id } raw, err := json.Marshal(run) if err != nil { - return run, ErrCodexQualityUnavailable + return run, codexQualityUnavailable("encode quality run %s: %v", id, err) } result, err := store.CompareSwapExtensionState(ctx, codexRuntimePluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: codexQualityKey(id), ExpectedRevision: revision, Value: raw}) if err != nil { - return run, ErrCodexQualityUnavailable + return run, codexQualityUnavailable("save quality run %s: %v", id, err) } if result.Applied { return run, nil } } - return codexQualityRun{}, ErrCodexQualityUnavailable + return codexQualityRun{}, codexQualityUnavailable("quality run %s kept changing concurrently", id) } func newCodexQualityGrant(id string) (string, string, error) { var secret [32]byte if _, err := rand.Read(secret[:]); err != nil { - return "", "", ErrCodexQualityUnavailable + return "", "", codexQualityUnavailable("generate the grant: %v", err) } grant := id + "." + base64.RawURLEncoding.EncodeToString(secret[:]) return grant, codexQualityHash(grant), nil @@ -186,17 +214,32 @@ func codexQualityActive(run codexQualityRun, now time.Time) bool { return run.RunID != "" && run.Status == "open" && now.Before(run.ExpiresAt) } +// codexQualityInactiveReason explains why codexQualityActive is false. +func codexQualityInactiveReason(run codexQualityRun, now time.Time) string { + switch { + case run.RunID == "": + return "quality run not found" + case run.Status != "open": + return fmt.Sprintf("quality run %s is %s", run.RunID, run.Status) + case !now.Before(run.ExpiresAt): + return fmt.Sprintf("quality run %s expired at %s", run.RunID, run.ExpiresAt.UTC().Format(time.RFC3339)) + } + return "" +} + func issueCodexQualityGrant(ctx context.Context, store NativeCodexStateStore, wanted codexQualityRun, digest string, expiry time.Time) (codexQualityRun, error) { canonical, valid := canonicalCodexQualityID(wanted.RunID) if !valid { - return codexQualityRun{}, ErrCodexQualityUnavailable + return codexQualityRun{}, codexQualityUnavailable("run id %q is not a canonical UUID", wanted.RunID) } wanted.RunID = canonical return mutateCodexQualityRun(ctx, store, wanted.RunID, func(run *codexQualityRun) error { if run.RunID == "" { *run = wanted - } else if run.Status == "closed" || run.ActorID != wanted.ActorID || run.APIKeyID != wanted.APIKeyID || run.AccountID != wanted.AccountID || run.GroupID != wanted.GroupID || run.ProxyID != wanted.ProxyID || run.PromptSHA256 != wanted.PromptSHA256 || run.MaxSends != wanted.MaxSends || !run.Scope.SameOwner(wanted.Scope) { - return ErrCodexQualityUnavailable + } else if run.Status == "closed" { + return codexQualityUnavailable("quality run %s is closed; use a new run id", run.RunID) + } else if run.ActorID != wanted.ActorID || run.APIKeyID != wanted.APIKeyID || run.AccountID != wanted.AccountID || run.GroupID != wanted.GroupID || run.ProxyID != wanted.ProxyID || run.PromptSHA256 != wanted.PromptSHA256 || run.MaxSends != wanted.MaxSends || !run.Scope.SameOwner(wanted.Scope) { + return codexQualityUnavailable("quality run %s exists with other parameters (actor %d/%d, api key %d/%d, account %d/%d, group %d/%d, proxy %d/%d, max_sends %d/%d, prompt matches %t, routing owner matches %t; stored/requested)", run.RunID, run.ActorID, wanted.ActorID, run.APIKeyID, wanted.APIKeyID, run.AccountID, wanted.AccountID, run.GroupID, wanted.GroupID, run.ProxyID, wanted.ProxyID, run.MaxSends, wanted.MaxSends, run.PromptSHA256 == wanted.PromptSHA256, run.Scope.SameOwner(wanted.Scope)) } run.GrantDigest, run.ExpiresAt = digest, expiry return nil @@ -209,16 +252,22 @@ func sameCodexQualityAttempt(a, b CodexQualityAttempt) bool { func reserveCodexQualityAttempt(ctx context.Context, store NativeCodexStateStore, id, grantDigest string, attempt CodexQualityAttempt) error { _, err := mutateCodexQualityRun(ctx, store, id, func(run *codexQualityRun) error { - if !codexQualityActive(*run, time.Now()) || (grantDigest != "" && !codexQualityGrantMatches(*run, grantDigest)) || run.AccountID != attempt.AccountID { - return ErrCodexQualityUnavailable + if reason := codexQualityInactiveReason(*run, time.Now()); reason != "" { + return codexQualityUnavailable("%s", reason) + } + if grantDigest != "" && !codexQualityGrantMatches(*run, grantDigest) { + return codexQualityUnavailable("the grant does not match quality run %s (it was reissued)", run.RunID) + } + if run.AccountID != attempt.AccountID { + return codexQualityUnavailable("attempt account %d differs from run account %d", attempt.AccountID, run.AccountID) } for _, previous := range run.Attempts { if sameCodexQualityAttempt(previous, attempt) { - return ErrCodexQualitySpent + return fmt.Errorf("%w: %s attempt (trial %q, operation %q) was already sent", ErrCodexQualitySpent, attempt.Stage, attempt.TrialID, attempt.OperationID) } } if run.UsedSends >= run.MaxSends { - return ErrCodexQualitySpent + return fmt.Errorf("%w: %d of %d sends used", ErrCodexQualitySpent, run.UsedSends, run.MaxSends) } attempt.State, attempt.ReservedAt = "unknown", time.Now().UTC() attempt.ResponseModels = []string{} @@ -243,12 +292,12 @@ func finishCodexQualityAttempt(ctx context.Context, store NativeCodexStateStore, return nil } } - return ErrCodexQualityUnavailable + return codexQualityUnavailable("%s attempt (trial %q, operation %q) was never reserved", finished.Stage, finished.TrialID, finished.OperationID) }) } func codexQualityView(run codexQualityRun, generation int64) *CodexQualityRunView { - v := &CodexQualityRunView{RunID: run.RunID, AccountID: run.AccountID, APIKeyID: run.APIKeyID, GroupID: run.GroupID, Model: codexQualityModel, ReasoningEffort: codexQualityEffort, PromptSHA256: run.PromptSHA256, Status: run.Status, MaxSends: run.MaxSends, UsedSends: run.UsedSends, ExpiresAt: run.ExpiresAt, RouteGeneration: run.RouteGeneration, Attempts: run.Attempts} + v := &CodexQualityRunView{RunID: run.RunID, ActorID: run.ActorID, AccountID: run.AccountID, APIKeyID: run.APIKeyID, GroupID: run.GroupID, ProxyID: run.ProxyID, Model: codexQualityModel, ReasoningEffort: codexQualityEffort, PromptSHA256: run.PromptSHA256, Status: run.Status, MaxSends: run.MaxSends, UsedSends: run.UsedSends, ExpiresAt: run.ExpiresAt, RouteGeneration: run.RouteGeneration, Attempts: run.Attempts} if v.Attempts == nil { v.Attempts = []CodexQualityAttempt{} } @@ -256,10 +305,34 @@ func codexQualityView(run codexQualityRun, generation int64) *CodexQualityRunVie v.RouteExpiresAt = &q.ExpiresAt v.RouteReady = codexQualityActive(run, time.Now()) && generation == run.RouteRuntimeGeneration && q.Valid(time.Now(), run.AccountID, run.Scope.Identity, codexQualityModel) v.ConnectionFingerprint = codexQualityHash(q.Scope.ConnectionLeaseID)[:16] + if !v.RouteReady { + v.RouteError = codexQualityRouteReason(run, generation, time.Now()) + } } return v } +// codexQualityRouteReason explains why the installed route of a run cannot be +// used before the bundle and connection checks, or returns "" when it can. +func codexQualityRouteReason(run codexQualityRun, generation int64, now time.Time) string { + q := run.Qualification + switch { + case !codexQualityActive(run, now): + return codexQualityInactiveReason(run, now) + case q == nil: + return "no route is installed; renew the route first" + case run.RouteRuntimeGeneration != generation: + return fmt.Sprintf("the route was installed by runtime generation %d, the current generation is %d", run.RouteRuntimeGeneration, generation) + case !q.Valid(now, run.AccountID, run.Scope.Identity, codexQualityModel): + return fmt.Sprintf("the route qualification is not valid now (model %q, verified %s, expires %s)", q.Model, q.VerifiedAt.UTC().Format(time.RFC3339), q.ExpiresAt.UTC().Format(time.RFC3339)) + case !q.Scope.SameOwner(run.Scope): + return "the route " + codexRoutingScopeChange(run.Scope, q.Scope) + case q.Scope.Transport != "http" || q.Scope.ConnectionLeaseID == "": + return fmt.Sprintf("the route is bound to transport %q connection %q; an HTTP connection lease is required", q.Scope.Transport, q.Scope.ConnectionLeaseID) + } + return "" +} + func validCodexQualityHash(value string) bool { decoded, err := hex.DecodeString(value) return err == nil && len(decoded) == 32 && strings.ToLower(value) == value diff --git a/backend/internal/service/codex_quality_test.go b/backend/internal/service/codex_quality_test.go index fff14495803e..497c678c1dad 100644 --- a/backend/internal/service/codex_quality_test.go +++ b/backend/internal/service/codex_quality_test.go @@ -116,6 +116,17 @@ func TestCodexQualityRawObserverDoesNotRewriteOrInventUsage(t *testing.T) { require.Nil(t, final.InputTokens) require.NotNil(t, final.OutputTokens) require.Zero(t, *final.OutputTokens) + require.Equal(t, strings.Repeat("m", 256)+"…(truncated, 300 bytes)", qualityRecordedModel(strings.Repeat("m", 300)), "one declared name is bounded and marked") + failedEvent := `{"type":"response.failed","response":{"model":"o-internal/Model X","error":{"code":"server_error","message":"upstream said no"}}}` + var failed CodexQualityAttempt + odd := &codexQualityObservedBody{ReadCloser: io.NopCloser(strings.NewReader("data: " + failedEvent + "\n\n")), sse: true, attempt: CodexQualityAttempt{HTTPStatus: 200}, finish: func(a CodexQualityAttempt) { failed = a }} + _, err = io.ReadAll(odd) + require.NoError(t, err) + require.Equal(t, []string{"o-internal/Model X"}, failed.ResponseModels, "declared models are recorded verbatim") + require.Equal(t, "upstream_error", failed.ErrorCode) + require.Equal(t, "server_error", failed.UpstreamErrorCode) + require.Equal(t, "upstream said no", failed.UpstreamErrorMessage) + require.Equal(t, failedEvent, failed.UpstreamBody) } func TestCodexQualityPrivateKeysAndHeaderScope(t *testing.T) { diff --git a/backend/internal/service/codex_routing_completion.go b/backend/internal/service/codex_routing_completion.go index fde4346d0b2a..b52724f6b08a 100644 --- a/backend/internal/service/codex_routing_completion.go +++ b/backend/internal/service/codex_routing_completion.go @@ -5,19 +5,23 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "strings" + "unicode/utf8" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" + "github.com/tidwall/gjson" ) const codexRoutingProbeReadLimit = 128 << 10 var ErrCodexRoutingModelMismatch = errors.New("upstream Codex response model does not match the requested model") -// Only bounded metadata is retained. Probe totals remain limited to 128 KiB; -// business readers explicitly select their existing stream/JSON read limits. +// Probe totals remain limited to 128 KiB; business readers explicitly select +// their existing stream/JSON read limits. The upstream error object, the failing +// event (bounded) and the local reason of a failure are kept for administrators. type codexRoutingCompletion struct { Model string RequestedModel string @@ -25,11 +29,72 @@ type codexRoutingCompletion struct { Failed bool Mismatch bool FailureCode string + FailureDetail string + UpstreamError codexUpstreamError + UpstreamEvent []byte + Body []byte maxEventBytes int pending []byte scanFrom int } +type codexUpstreamError struct { + Type, Code, Message, Param string +} + +func (e codexUpstreamError) empty() bool { + return e.Type == "" && e.Code == "" && e.Message == "" && e.Param == "" +} + +func codexJSONText(value gjson.Result) string { + switch { + case !value.Exists() || value.Type == gjson.Null: + return "" + case value.Type == gjson.String: + return value.String() + } + return value.Raw +} + +func codexUpstreamErrorObject(value gjson.Result) codexUpstreamError { + if value.Type == gjson.String { + return codexUpstreamError{Message: value.String()} + } + if !value.IsObject() { + return codexUpstreamError{} + } + return codexUpstreamError{Type: codexJSONText(value.Get("type")), Code: codexJSONText(value.Get("code")), Message: codexJSONText(value.Get("message")), Param: codexJSONText(value.Get("param"))} +} + +// codexUpstreamErrorFromBody reads the upstream's own error object from a JSON +// error body ({"error":...}, {"response":{"error":...}} or {"detail":...}). +func codexUpstreamErrorFromBody(body []byte) codexUpstreamError { + if !gjson.ValidBytes(body) { + return codexUpstreamError{} + } + root := gjson.ParseBytes(body) + for _, path := range []string{"error", "response.error", "detail"} { + if found := codexUpstreamErrorObject(root.Get(path)); !found.empty() { + return found + } + } + return codexUpstreamError{} +} + +// boundedCodexUpstreamBody keeps at most CodexRoutingUpstreamBodyLimit bytes +// without splitting a UTF-8 sequence. +func boundedCodexUpstreamBody(raw []byte) []byte { + limit := extensionv1.CodexRoutingUpstreamBodyLimit + if len(raw) > limit { + cut := limit + for cut > 0 && !utf8.RuneStart(raw[cut]) { + cut-- + } + raw = raw[:cut] + } + return append([]byte(nil), raw...) +} + func (o *codexRoutingCompletion) fail(code string) { o.Failed = true if o.FailureCode == "" { @@ -37,13 +102,95 @@ func (o *codexRoutingCompletion) fail(code string) { } } +func (o *codexRoutingCompletion) failWithDetail(code, detail string) { + if o.FailureDetail == "" && !o.Failed { + o.FailureDetail = detail + } + o.fail(code) +} + +// recordUpstream keeps the first upstream failure: its error object and a +// summary of the failing event (see codexRoutingEventSummary). +func (o *codexRoutingCompletion) recordUpstream(found codexUpstreamError, event []byte) { + if o.UpstreamEvent != nil || !o.UpstreamError.empty() { + return + } + o.UpstreamError = found + o.UpstreamEvent = codexRoutingEventSummary(event) +} + +// codexRoutingEventSummary keeps what identifies an upstream failure: the +// event type, its error fields (top level, error, detail) and the response id, +// status, model, incomplete_details and error. The response instructions, +// tools, input and output of a business request are never copied. +func codexRoutingEventSummary(raw []byte) []byte { + if !gjson.ValidBytes(raw) { + return boundedCodexUpstreamBody(raw) + } + root := gjson.ParseBytes(raw) + summary := map[string]any{} + for _, key := range []string{"type", "code", "message", "param", "error", "detail"} { + if value := root.Get(key); value.Exists() && value.Type != gjson.Null { + summary[key] = json.RawMessage(value.Raw) + } + } + if response := root.Get("response"); response.IsObject() { + kept := map[string]json.RawMessage{} + for _, key := range []string{"id", "status", "model", "incomplete_details", "error"} { + if value := response.Get(key); value.Exists() && value.Type != gjson.Null { + kept[key] = json.RawMessage(value.Raw) + } + } + summary["response"] = kept + } + encoded, err := json.Marshal(summary) + if err != nil { + return nil + } + return boundedCodexUpstreamBody(encoded) +} + +// apply copies the recorded failure facts into an observation. The upstream +// body is the failing event, or the JSON body of a non-success response. +func (o codexRoutingCompletion) apply(observation *extensionv1.CodexRoutingObservation, status int) { + if observation.UpstreamErrorType == "" && observation.UpstreamErrorCode == "" && observation.UpstreamErrorMessage == "" { + observation.UpstreamErrorType, observation.UpstreamErrorCode = o.UpstreamError.Type, o.UpstreamError.Code + observation.UpstreamErrorMessage, observation.UpstreamErrorParam = o.UpstreamError.Message, o.UpstreamError.Param + } + if observation.UpstreamBody == "" { + if len(o.UpstreamEvent) > 0 { + observation.UpstreamBody = string(o.UpstreamEvent) + } else if status != http.StatusOK && status != http.StatusSwitchingProtocols && len(o.Body) > 0 { + observation.UpstreamBody = string(o.Body) + } + } + if observation.Error == "" { + observation.Error = o.incompleteReason(status) + } +} + +// incompleteReason explains a local (non-upstream) failure classification. +func (o codexRoutingCompletion) incompleteReason(status int) string { + switch { + case o.FailureDetail != "": + return o.FailureDetail + case o.Failed || (status != http.StatusOK && status != http.StatusSwitchingProtocols) || (o.Completed && o.Model != ""): + return "" + case !o.Completed && o.Model == "": + return "response ended without a completed terminal event or a model declaration" + case !o.Completed: + return "response ended without a completed terminal event" + } + return "terminal response did not declare a model" +} + func (o *codexRoutingCompletion) observeModel(model string) { model = strings.TrimSpace(model) if model == "" { return } if len(model) > 256 { - o.fail("routing_incomplete") + o.failWithDetail("routing_incomplete", fmt.Sprintf("response model declaration is %d bytes (limit 256): %s", len(model), model)) return } if (o.Model != "" && o.Model != model) || (o.RequestedModel != "" && o.RequestedModel != model) { @@ -82,7 +229,17 @@ func codexRoutingErrorCode(raw json.RawMessage) string { return "routing_upstream" } -func (o *codexRoutingCompletion) json(raw []byte) { o.jsonEvent(raw, "") } +func (o *codexRoutingCompletion) json(raw []byte) { + o.Body = boundedCodexUpstreamBody(raw) + o.jsonEvent(raw, "") + if !o.Completed && o.UpstreamError.empty() && o.UpstreamEvent == nil { + // Non-streaming error bodies may carry {"detail":...} instead of an + // error object; keep that upstream text too. + if found := codexUpstreamErrorFromBody(raw); !found.empty() { + o.recordUpstream(found, raw) + } + } +} func (o *codexRoutingCompletion) jsonEvent(raw []byte, eventType string) { var event struct { @@ -124,28 +281,36 @@ func (o *codexRoutingCompletion) jsonEvent(raw []byte, eventType string) { } badError := func(raw json.RawMessage) bool { return len(raw) > 0 && string(raw) != "null" } if badError(event.Error) { + o.recordUpstream(codexUpstreamErrorObject(gjson.ParseBytes(event.Error)), raw) o.fail(codexRoutingErrorCode(event.Error)) return } if event.Response != nil && badError(event.Response.Error) { + o.recordUpstream(codexUpstreamErrorObject(gjson.ParseBytes(event.Response.Error)), raw) o.fail(codexRoutingErrorCode(event.Response.Error)) return } switch event.Type { case "response.cancelled", "response.canceled": + o.recordUpstream(codexUpstreamError{Type: event.Type}, raw) o.fail("routing_cancelled") return case "response.incomplete": + o.recordUpstream(codexUpstreamError{Type: "incomplete", Code: codexJSONText(gjson.GetBytes(raw, "response.incomplete_details.reason"))}, raw) o.fail("routing_incomplete") return case "error", "response.failed": + // A streamed error event carries code/message/param at its top level. + root := gjson.ParseBytes(raw) + o.recordUpstream(codexUpstreamError{Code: codexJSONText(root.Get("code")), Message: codexJSONText(root.Get("message")), Param: codexJSONText(root.Get("param"))}, raw) o.fail("routing_upstream") return } terminal := event.Type == "response.completed" && event.Response != nil if terminal { if event.Response.Status != "completed" { - o.fail("routing_incomplete") + o.recordUpstream(codexUpstreamError{Type: "status", Code: event.Response.Status}, raw) + o.failWithDetail("routing_incomplete", fmt.Sprintf("response.completed carried status %q", event.Response.Status)) return } } else if event.Object == "response" && event.Status == "completed" { @@ -156,7 +321,7 @@ func (o *codexRoutingCompletion) jsonEvent(raw []byte, eventType string) { if terminal && o.Model != "" { o.Completed = true } else { - o.fail("routing_incomplete") + o.failWithDetail("routing_incomplete", "terminal response did not declare a model") } } @@ -181,7 +346,7 @@ func (o *codexRoutingCompletion) feed(data []byte) { continue } if i > limit { - o.fail("routing_incomplete") + o.failWithDetail("routing_incomplete", fmt.Sprintf("SSE event of %d bytes exceeds the %d-byte limit", i, limit)) break } var lines [][]byte @@ -203,7 +368,7 @@ func (o *codexRoutingCompletion) feed(data []byte) { } o.scanFrom = max(0, len(o.pending)-2) if len(o.pending) > limit { - o.fail("routing_incomplete") + o.failWithDetail("routing_incomplete", fmt.Sprintf("unterminated SSE event of %d bytes exceeds the %d-byte limit", len(o.pending), limit)) } if o.Failed { o.pending = nil @@ -233,6 +398,12 @@ func readCodexRoutingCompletion(body io.Reader, contentType, requested string, o if errors.Is(err, context.Canceled) { observation.Code = "routing_cancelled" } + if err != nil { + observation.Error = "read response body: " + err.Error() + } else { + observation.Error = fmt.Sprintf("response body exceeds the %d-byte probe limit", codexRoutingProbeReadLimit) + } + observation.UpstreamBody = string(boundedCodexUpstreamBody(data)) return errCodexRoutingUnavailable } completed := codexRoutingCompletion{RequestedModel: requested} @@ -250,6 +421,10 @@ func readCodexRoutingCompletion(body io.Reader, contentType, requested string, o observation.ModelMatched = observation.Completed && !completed.Mismatch && completed.Model == requested observation.Code = completed.observationCode(requested, http.StatusOK) if observation.Code != "routing_verified" { + completed.apply(observation, http.StatusOK) + if observation.Code == "routing_model_mismatch" && observation.Error == "" { + observation.Error = fmt.Sprintf("requested model %q, upstream declared %q", requested, completed.Model) + } return errCodexRoutingUnavailable } return nil diff --git a/backend/internal/service/codex_routing_cookies.go b/backend/internal/service/codex_routing_cookies.go index d4f636dbdfbc..babb1efa2a5d 100644 --- a/backend/internal/service/codex_routing_cookies.go +++ b/backend/internal/service/codex_routing_cookies.go @@ -189,7 +189,7 @@ func preserveCodexCookieFirstSeen(ctx context.Context, store NativeCodexStateSto } if record.Found { if json.Unmarshal(record.Value, &seen) != nil || seen.First.IsZero() { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("first-seen record of cookie %s is invalid: %s", change.Cookie.Name, string(record.Value)) } } else { seen.First = change.Cookie.FirstSeen @@ -202,7 +202,7 @@ func preserveCodexCookieFirstSeen(ctx context.Context, store NativeCodexStateSto return err } if !saved.Applied && (json.Unmarshal(saved.Value, &seen) != nil || seen.First.IsZero()) { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("concurrent first-seen record of cookie %s is invalid: %s", change.Cookie.Name, string(saved.Value)) } } change.Cookie.FirstSeen = seen.First diff --git a/backend/internal/service/codex_routing_fingerprint.go b/backend/internal/service/codex_routing_fingerprint.go index 508a08f0792b..da98f0fbd97b 100644 --- a/backend/internal/service/codex_routing_fingerprint.go +++ b/backend/internal/service/codex_routing_fingerprint.go @@ -5,6 +5,7 @@ import ( "crypto/tls" _ "embed" "encoding/json" + "fmt" "net/http" "strconv" "strings" @@ -70,6 +71,30 @@ func codexRoutingTurnID(c *gin.Context) string { return turn } +// CodexWireCookie is one cookie of the final outbound request, exactly as sent. +type CodexWireCookie struct { + Name string `json:"name"` + Value string `json:"value"` + Routing bool `json:"routing"` +} + +// codexWireCookies splits the Cookie headers as sent, without dropping pairs +// that the stricter net/http parser would reject. +func codexWireCookies(headers http.Header) []CodexWireCookie { + cookies := []CodexWireCookie{} + for _, line := range headers.Values("Cookie") { + for _, part := range strings.Split(line, ";") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + name, value, _ := strings.Cut(part, "=") + cookies = append(cookies, CodexWireCookie{Name: name, Value: value, Routing: isCodexRoutingCookie(name)}) + } + } + return cookies +} + type CodexWireFingerprint struct { IdentityFields map[string]CodexIdentityFieldObservation `json:"identity_fields"` Ingress string `json:"ingress"` @@ -90,6 +115,9 @@ type CodexWireFingerprint struct { RouteHash string `json:"route_hash"` RouteEvidence string `json:"route_evidence"` ConnectionEvidence string `json:"connection_evidence"` + ConnectionLeaseID string `json:"connection_lease_id,omitempty"` + AccountProxyID int64 `json:"account_proxy_id,omitempty"` + Cookies []CodexWireCookie `json:"cookies"` CookieNames []string `json:"cookie_names"` CookieVersions map[string]string `json:"cookie_versions"` SessionPresent bool `json:"session_present"` @@ -97,19 +125,46 @@ type CodexWireFingerprint struct { WindowPresent bool `json:"window_present"` StatePresent bool `json:"state_present"` StateLength int `json:"state_length"` + State string `json:"state,omitempty"` WebSocketExtensions string `json:"websocket_extensions,omitempty"` } +// CodexRoutingModelStatus shows the stored routing record of one model as is, +// including a record written for a previous credential owner. type CodexRoutingModelStatus struct { - Model string `json:"model"` - Phase string `json:"phase"` - Enrolled bool `json:"enrolled"` - Failures int `json:"failed_cycles"` - Qualified bool `json:"qualified"` - VerifiedAt *time.Time `json:"verified_at,omitempty"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` - NextAt *time.Time `json:"next_attempt_at,omitempty"` - LastObservation *extensionv1.CodexRoutingObservation `json:"last_observation,omitempty"` + Model string `json:"model"` + Phase string `json:"phase"` + Enrolled bool `json:"enrolled"` + Failures int `json:"failed_cycles"` + Qualified bool `json:"qualified"` + VerifiedAt *time.Time `json:"verified_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + NextAt *time.Time `json:"next_attempt_at,omitempty"` + LastObservation *extensionv1.CodexRoutingObservation `json:"last_observation,omitempty"` + Schema int `json:"schema,omitempty"` + StateRevision int64 `json:"state_revision,omitempty"` + StateError string `json:"state_error,omitempty"` + RecordedIdentity string `json:"recorded_identity,omitempty"` + IdentityMatches bool `json:"identity_matches"` + LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + OperationID string `json:"operation_id,omitempty"` + ManualWasEnrolled bool `json:"manual_was_enrolled"` + LastCode string `json:"last_code,omitempty"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` + RevocationReason string `json:"revocation_reason,omitempty"` + Qualification *extensionv1.CodexRoutingQualification `json:"qualification,omitempty"` + RouteMatchesCurrent bool `json:"route_matches_current"` +} + +// CodexFingerprintProxy identifies the account proxy that enters RouteHash. +type CodexFingerprintProxy struct { + ID int64 `json:"id"` + Name string `json:"name,omitempty"` + Protocol string `json:"protocol,omitempty"` + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + Username string `json:"username,omitempty"` + Status string `json:"status,omitempty"` } type CodexFingerprintView struct { @@ -118,11 +173,18 @@ type CodexFingerprintView struct { AccountID int64 `json:"account_id"` Configured codexIdentitySnapshot `json:"configured"` Observed *CodexWireFingerprint `json:"observed,omitempty"` + ObservedError string `json:"observed_error,omitempty"` ReferenceCommit string `json:"reference_commit"` ReferenceSource string `json:"reference_source"` Alignment string `json:"alignment"` CookieMaxAgeSeconds int `json:"cookie_max_age_seconds"` RefreshLeadSeconds int `json:"refresh_lead_seconds"` + RoutingEnabled bool `json:"routing_enabled"` + FailClosed bool `json:"fail_closed"` + CurrentScope *extensionv1.CodexRoutingScope `json:"current_scope,omitempty"` + ScopeError string `json:"scope_error,omitempty"` + AccountProxy *CodexFingerprintProxy `json:"account_proxy,omitempty"` + HarvestProxyURL string `json:"harvest_proxy_url,omitempty"` Models []CodexRoutingModelStatus `json:"models"` } @@ -139,6 +201,9 @@ func (s *OpenAIGatewayService) observeCodexWire(ctx context.Context, account *Ac value := CodexWireFingerprint{ObservedAt: time.Now().UTC(), Source: "final_outbound", Transport: "http", UserAgent: request.Header.Get("User-Agent"), Originator: request.Header.Get("originator"), Version: request.Header.Get("version"), HTTPProtocol: response.Proto, RequestEncoding: request.Header.Get("Content-Encoding"), TLSImplementation: "go-crypto-tls", JA3: "unknown", CookieNames: []string{}, SessionPresent: extractClientSessionID(request.Header) != "", ThreadPresent: request.Header.Get("thread-id") != "", WindowPresent: request.Header.Get("x-codex-window-id") != "", StatePresent: request.Header.Get(openAICodexTurnStateHeader) != "", StateLength: len(request.Header.Get(openAICodexTurnStateHeader))} value.Ingress = "http" value.CookieVersions = map[string]string{} + value.Cookies = codexWireCookies(request.Header) + value.State = request.Header.Get(openAICodexTurnStateHeader) + value.AccountProxyID = qualityProxyID(account) body, carried := request.Context().Value(codexIdentityBodyKey{}).(codexIdentityBodyObservation) if !carried { // Only a rewritten (zstd) wire copy carries a pre-encoding observation; @@ -157,6 +222,7 @@ func (s *OpenAIGatewayService) observeCodexWire(ctx context.Context, account *Ac if qualification != nil { value.ProfileHash, value.RouteHash, value.RouteEvidence = qualification.Scope.ProfileHash, qualification.Scope.RouteHash, qualification.Scope.RouteEvidence value.ConnectionEvidence = codexRoutingDigest(qualification.Scope.ConnectionLeaseID)[:16] + value.ConnectionLeaseID = qualification.Scope.ConnectionLeaseID value.Transport = qualification.Scope.Transport } if value.Transport == "ws" { @@ -183,52 +249,88 @@ func (s *OpenAIGatewayService) observeCodexWire(ctx context.Context, account *Ac // CodexFingerprint is read-only: opening an account never refreshes OAuth, // acquires cookies, pings a proxy, or sends a model request. func (s *OpenAIGatewayService) CodexFingerprint(ctx context.Context, id int64) (*CodexFingerprintView, error) { + if s.nativeCodexRuntime == nil { + return nil, codexRoutingUnavailable("native Codex runtime is not configured") + } a, err := s.accountRepo.GetByID(ctx, id) - if err != nil || !isOpenAICodexTicketAccount(a) || s.nativeCodexRuntime == nil { - return nil, errCodexRoutingUnavailable + switch { + case err != nil: + return nil, codexRoutingUnavailable("read account %d: %v", id, err) + case !isOpenAICodexTicketAccount(a): + return nil, codexRoutingUnavailable("account %d is not an OpenAI OAuth/setup-token Codex account", id) } installation := s.nativeCodexRuntime.metadata() store := s.nativeCodexRuntime.repo ok := store != nil if installation == nil || !ok { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("native Codex runtime is not loaded") } - view := &CodexFingerprintView{AccountID: id, Configured: resolveCodexIdentitySnapshotContext(ctx, a, a, codexAccountIdentityOverrideUA(a)), ReferenceCommit: "4b664e0ef0397f82e68c60088a90fcd035deb796", ReferenceSource: "official_source", Alignment: "application_fields_checked_tls_capture_unknown", CookieMaxAgeSeconds: 120, RefreshLeadSeconds: 20, Models: []CodexRoutingModelStatus{}} + cfg := s.openAICodexTicketConfig() + view := &CodexFingerprintView{AccountID: id, Configured: resolveCodexIdentitySnapshotContext(ctx, a, a, codexAccountIdentityOverrideUA(a)), ReferenceCommit: "4b664e0ef0397f82e68c60088a90fcd035deb796", ReferenceSource: "official_source", Alignment: "application_fields_checked_tls_capture_unknown", CookieMaxAgeSeconds: 120, RefreshLeadSeconds: 20, RoutingEnabled: cfg.Enabled, FailClosed: cfg.FailClosed, HarvestProxyURL: cfg.HarvestProxyURL, Models: []CodexRoutingModelStatus{}} currentScope, scopeErr := s.PrepareCodexRoutingScope(ctx, id, "http") + if scopeErr != nil { + view.ScopeError = scopeErr.Error() + } else { + view.CurrentScope = ¤tScope + } + if a.ProxyID != nil { + proxy := &CodexFingerprintProxy{ID: *a.ProxyID} + if a.Proxy != nil { + proxy.Name, proxy.Protocol, proxy.Host, proxy.Port, proxy.Username, proxy.Status = a.Proxy.Name, a.Proxy.Protocol, a.Proxy.Host, a.Proxy.Port, a.Proxy.Username, a.Proxy.Status + } + view.AccountProxy = proxy + } wire, err := store.ReadExtensionState(ctx, NativeCodexPluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: "wire." + strconv.FormatInt(id, 10)}) if err != nil { - return nil, err + return nil, fmt.Errorf("read the last outbound observation of account %d: %w", id, err) } if wire.Found { var observation CodexWireFingerprint - if json.Unmarshal(wire.Value, &observation) == nil { + if err := json.Unmarshal(wire.Value, &observation); err != nil { + view.ObservedError = "decode the last outbound observation: " + err.Error() + } else { view.Observed = &observation } } - for _, model := range s.openAICodexTicketConfig().Models { + for _, model := range cfg.Models { status := CodexRoutingModelStatus{Model: model, Phase: "idle"} record, readErr := store.ReadExtensionState(ctx, NativeCodexPluginKey, extensionv1.StateRequest{Namespace: "tickets", Key: strconv.FormatInt(id, 10) + "." + codexRoutingDigest(model)}) if readErr != nil { - return nil, readErr + return nil, fmt.Errorf("read the routing record of %s: %w", model, readErr) } if record.Found { var state struct { - Schema int `json:"schema"` - Identity string `json:"identity"` - Phase string `json:"phase"` - Enrolled bool `json:"enrolled"` - Failures int `json:"failures"` - ExpiresAt *time.Time `json:"expires_at"` - NextAt *time.Time `json:"next_attempt_at"` - Qualification *extensionv1.CodexRoutingQualification `json:"qualification"` - Observation *extensionv1.CodexRoutingObservation `json:"observation"` + Schema int `json:"schema"` + Identity string `json:"identity"` + Phase string `json:"phase"` + Enrolled bool `json:"enrolled"` + Failures int `json:"failures"` + ExpiresAt *time.Time `json:"expires_at"` + NextAt *time.Time `json:"next_attempt_at"` + LastAttemptAt *time.Time `json:"last_attempt_at"` + OperationID string `json:"operation_id"` + ManualWasEnrolled bool `json:"manual_was_enrolled"` + LastCode string `json:"last_code"` + RevokedAt *time.Time `json:"revoked_at"` + RevocationReason string `json:"revocation_reason"` + Qualification *extensionv1.CodexRoutingQualification `json:"qualification"` + Observation *extensionv1.CodexRoutingObservation `json:"observation"` } - if json.Unmarshal(record.Value, &state) == nil && state.Identity == CodexTicketAccountIdentity(a) { + status.StateRevision = record.Revision + if err := json.Unmarshal(record.Value, &state); err != nil { + status.StateError = err.Error() + } else { + status.RecordedIdentity, status.IdentityMatches, status.Schema = state.Identity, state.Identity == CodexTicketAccountIdentity(a), state.Schema status.Phase, status.Enrolled, status.Failures, status.ExpiresAt, status.NextAt, status.LastObservation = state.Phase, state.Enrolled, state.Failures, state.ExpiresAt, state.NextAt, state.Observation - status.Qualified = scopeErr == nil && state.Schema == extensionv1.CodexRoutingSchema && state.Qualification.Valid(time.Now(), id, state.Identity, model) && state.Qualification.Scope.SameOwner(currentScope) - if state.Qualification != nil && state.Qualification.Model == model && !state.Qualification.VerifiedAt.IsZero() { - verified := state.Qualification.VerifiedAt - status.VerifiedAt = &verified + status.LastAttemptAt, status.OperationID, status.ManualWasEnrolled, status.LastCode, status.Qualification = state.LastAttemptAt, state.OperationID, state.ManualWasEnrolled, state.LastCode, state.Qualification + status.RevokedAt, status.RevocationReason = state.RevokedAt, state.RevocationReason + status.Qualified = status.IdentityMatches && scopeErr == nil && state.Schema == extensionv1.CodexRoutingSchema && state.Qualification.Valid(time.Now(), id, state.Identity, model) && state.Qualification.Scope.SameOwner(currentScope) + if state.Qualification != nil { + status.RouteMatchesCurrent = scopeErr == nil && state.Qualification.Scope.RouteHash == currentScope.RouteHash + if state.Qualification.Model == model && !state.Qualification.VerifiedAt.IsZero() { + verified := state.Qualification.VerifiedAt + status.VerifiedAt = &verified + } } if state.Schema < extensionv1.CodexRoutingSchema && (state.Phase == "ready" || state.Phase == "retry") { status.Phase = "needs_cookie_verification" diff --git a/backend/internal/service/codex_routing_fingerprint_test.go b/backend/internal/service/codex_routing_fingerprint_test.go index 65c1b8734b6f..cf06909143af 100644 --- a/backend/internal/service/codex_routing_fingerprint_test.go +++ b/backend/internal/service/codex_routing_fingerprint_test.go @@ -30,6 +30,7 @@ func TestCodexFingerprintShowsCookieDigestsAndOriginalVerificationTime(t *testin request, err := http.NewRequest(http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", nil) require.NoError(t, err) request.Header.Set("Cookie", "__cflb=first-secret-cookie; auth_token=private-session") + request.Header.Set(openAICodexTurnStateHeader, "raw-request-state") service.observeCodexWire(context.Background(), account, request, &http.Response{StatusCode: http.StatusOK}, qualification) view, err := service.CodexFingerprint(context.Background(), 7) require.NoError(t, err) @@ -42,8 +43,15 @@ func TestCodexFingerprintShowsCookieDigestsAndOriginalVerificationTime(t *testin require.Len(t, first, 16) raw, err = json.Marshal(view) require.NoError(t, err) - require.NotContains(t, string(raw), "first-secret-cookie") - require.NotContains(t, string(raw), "private-session") + // Administrators see every cookie of the last outbound request as sent, and + // the stored qualification of each model. + require.Contains(t, string(raw), "first-secret-cookie") + require.Equal(t, []CodexWireCookie{{Name: "__cflb", Value: "first-secret-cookie", Routing: true}, {Name: "auth_token", Value: "private-session"}}, view.Observed.Cookies) + require.NotNil(t, view.Models[0].Qualification) + require.Equal(t, "bundle.fingerprint", view.Models[0].Qualification.Bundle.Key) + require.True(t, view.Models[0].IdentityMatches) + require.True(t, view.Models[0].RouteMatchesCurrent) + require.Equal(t, "raw-request-state", view.Observed.State) before := len(store.values) _, err = service.CodexFingerprint(context.Background(), 7) require.NoError(t, err) diff --git a/backend/internal/service/codex_routing_host.go b/backend/internal/service/codex_routing_host.go index 7a4dd6a5eeab..2545f83f647a 100644 --- a/backend/internal/service/codex_routing_host.go +++ b/backend/internal/service/codex_routing_host.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "slices" @@ -25,6 +26,14 @@ type codexRoutingLeaseDirectory interface { CheckCodexRoutingLease(context.Context, extensionv1.CodexRoutingScope, time.Time) error } +// errCodexRoutingToken marks a probe that could not obtain the account access +// token; the host reports it as ticket_token instead of a transport failure. +var errCodexRoutingToken = errors.New("account access token unavailable") + +func codexRoutingUnavailable(format string, args ...any) error { + return fmt.Errorf("%w: %s", errCodexRoutingUnavailable, fmt.Sprintf(format, args...)) +} + func isCodexRoutingHostOperation(operation extensionv1.HostOperation) bool { return operation == extensionv1.HostCodexRoutingScope || operation == extensionv1.HostCodexRoutingProbe || operation == extensionv1.HostCodexRoutingCheck || operation == extensionv1.HostCodexRoutingCleanup } @@ -34,26 +43,42 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H return h.redactExpiredCodexRoutingMaterial(ctx) } var query extensionv1.CodexRoutingQuery - if h.key != codexRuntimePluginKey || len(in.Payload) > 16384 || json.Unmarshal(in.Payload, &query) != nil || query.AccountID <= 0 { - return extensionv1.Result{}, errCodexRoutingUnavailable + switch { + case h.key != codexRuntimePluginKey: + return extensionv1.Result{}, codexRoutingUnavailable("routing operations are reserved for the Codex runtime (caller %s)", h.key) + case len(in.Payload) > 16384: + return extensionv1.Result{}, codexRoutingUnavailable("routing query of %d bytes exceeds 16384 bytes", len(in.Payload)) + } + if err := json.Unmarshal(in.Payload, &query); err != nil { + return extensionv1.Result{}, codexRoutingUnavailable("decode routing query: %v", err) + } + if query.AccountID <= 0 { + return extensionv1.Result{}, codexRoutingUnavailable("routing query has no account id") } directory, ok := h.directory.(codexRoutingHostDirectory) if !ok { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("account directory cannot execute routing probes") } account, err := h.directory.ReadExtensionAccount(ctx, query.AccountID) - if err != nil || account == nil || account.Shadow || account.Platform != PlatformOpenAI || (account.Type != AccountTypeOAuth && account.Type != AccountTypeSetupToken) || !h.permitsAccount(account.Platform, account.Type, account.ID, true) { - return extensionv1.Result{}, errCodexRoutingUnavailable + switch { + case err != nil: + return extensionv1.Result{}, codexRoutingUnavailable("read account %d: %v", query.AccountID, err) + case account == nil: + return extensionv1.Result{}, codexRoutingUnavailable("account %d not found", query.AccountID) + case account.Shadow || account.Platform != PlatformOpenAI || (account.Type != AccountTypeOAuth && account.Type != AccountTypeSetupToken): + return extensionv1.Result{}, codexRoutingUnavailable("account %d (%s/%s, shadow=%t) is not an OpenAI OAuth/setup-token account", account.ID, account.Platform, account.Type, account.Shadow) + case !h.permitsAccount(account.Platform, account.Type, account.ID, true): + return extensionv1.Result{}, codexRoutingUnavailable("account %d is not permitted for the Codex runtime", account.ID) } if query.Transport == "" { query.Transport = "http" } if query.Transport != "http" && query.Transport != "ws" { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("unsupported routing transport %q", query.Transport) } scope, err := directory.PrepareCodexRoutingScope(ctx, query.AccountID, query.Transport) if err != nil { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("prepare routing scope: %v", err) } marshal := func(value any) (extensionv1.Result, error) { raw, err := json.Marshal(value) @@ -63,7 +88,7 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H return marshal(scope) } if query.Model == "" || len(query.Model) > 256 { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("routing query model must be 1 to 256 bytes (got %d)", len(query.Model)) } query.ReasoningEffort, err = codexRoutingProbeEffort(query.ReasoningEffort) if err != nil { @@ -78,23 +103,22 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H bundle, err = readCodexRoutingBundle(ctx, h.state, h.key, *query.Bundle, scope, false) if err != nil { result.Observation.Code = "routing_stale" + result.Observation.Error = err.Error() return marshal(result) } bundleRef = *query.Bundle cookieHeader, expires = codexRoutingCookieHeader(bundle.Cookies, time.Now()) if cookieHeader == "" { result.Observation.Code = "routing_cookie_expired" + result.Observation.Error = fmt.Sprintf("bundle %s has no live routing cookie left (%d stored)", query.Bundle.Key, len(bundle.Cookies)) return marshal(result) } } if in.Operation == extensionv1.HostCodexRoutingCheck { result.Scope, result.Bundle = bundle.Scope, query.Bundle - checker, ok := h.directory.(codexRoutingLeaseDirectory) - if !ok || query.Bundle == nil || bundle.Status != "qualified" || bundle.Model != query.Model || - bundle.Scope.Transport != query.Transport || query.Scope == nil || !query.Scope.SameOwner(bundle.Scope) || - query.Scope.Transport != bundle.Scope.Transport || query.Scope.ConnectionLeaseID != bundle.Scope.ConnectionLeaseID || - checker.CheckCodexRoutingLease(ctx, bundle.Scope, bundle.ExpiresAt) != nil { + if reason := h.codexRoutingCheckFailure(ctx, query, bundle); reason != "" { result.Observation.Code = "routing_connection_expired" + result.Observation.Error = reason return marshal(result) } result.Valid = true @@ -102,10 +126,11 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H return marshal(result) } if query.OperationID == "" || len(query.OperationID) > 256 || (query.Stage != "acquire" && query.Stage != "verify") || (query.Stage == "verify" && query.Bundle == nil) { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("routing probe needs an operation id of 1 to 256 bytes and stage acquire or verify (verify with a bundle); got operation %q stage %q bundle=%t", query.OperationID, query.Stage, query.Bundle != nil) } if query.Scope != nil && !query.Scope.SameOwner(scope) { result.Observation.Code = "routing_stale" + result.Observation.Error = codexRoutingScopeChange(*query.Scope, scope) return marshal(result) } // Persist a spent stage before IO. A lost RPC reply is not permission to @@ -114,20 +139,31 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H spent, err := h.state.CompareSwapExtensionState(ctx, h.key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: spentKey, Value: json.RawMessage(`{"spent":true}`)}) if err != nil || !spent.Applied { result.Observation.Code = "ticket_interrupted" + if err != nil { + result.Observation.Error = "record spent stage: " + err.Error() + } else { + result.Observation.Error = fmt.Sprintf("stage %s of operation %s was already sent once; it is never replayed", query.Stage, query.OperationID) + } return marshal(result) } if err := consumeCodexValidationBudget(ctx, h.state, h.key, query); err != nil { result.Observation.Code = "routing_budget_spent" + result.Observation.Error = err.Error() return marshal(result) } clockKey := codexQualityClockKey(ctx, "clock."+codexRoutingScopeKey(scope)) clockRecord, err := h.state.ReadExtensionState(ctx, h.key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: clockKey}) if err != nil { - return extensionv1.Result{}, errCodexRoutingUnavailable + return extensionv1.Result{}, codexRoutingUnavailable("read cookie clock: %v", err) } clock := codexRoutingCookieClock{Scope: scope} - if clockRecord.Found && (json.Unmarshal(clockRecord.Value, &clock) != nil || !clock.Scope.SameOwner(scope)) { - return extensionv1.Result{}, errCodexRoutingUnavailable + if clockRecord.Found { + if err := json.Unmarshal(clockRecord.Value, &clock); err != nil { + return extensionv1.Result{}, codexRoutingUnavailable("decode cookie clock: %v", err) + } + if !clock.Scope.SameOwner(scope) { + return extensionv1.Result{}, codexRoutingUnavailable("cookie clock belongs to another routing owner: %s", codexRoutingScopeChange(clock.Scope, scope)) + } } // Acquisition is cold; existing business-route material never crosses to // an unverified rotating acquisition connection. @@ -139,6 +175,7 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H observationID, activityErr := h.activity.BeginUnbilled(ctx, query.AccountID, h.key) if activityErr != nil { result.Observation.Code = "routing_observation_unavailable" + result.Observation.Error = activityErr.Error() return marshal(result) } defer func() { @@ -155,18 +192,31 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H result.Observation.DurationMS = time.Since(start).Milliseconds() if probeErr != nil || response == nil { result.Observation.Code = "routing_transport" - if errors.Is(probeErr, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + switch { + case errors.Is(probeErr, context.Canceled) || errors.Is(ctx.Err(), context.Canceled): result.Observation.Code = "routing_cancelled" + case errors.Is(probeErr, errCodexRoutingToken): + result.Observation.Code = "ticket_token" + } + if probeErr != nil { + result.Observation.Error = probeErr.Error() + } else { + result.Observation.Error = "probe returned no response" } return marshal(result) } defer func() { _ = response.Body.Close() }() result.Observation.HTTPStatus = response.StatusCode - result.Observation.StateLength = len(strings.TrimSpace(response.Header.Get(openAICodexTurnStateHeader))) + result.Observation.RequestID = response.Header.Get("x-request-id") + result.Observation.CFRay = response.Header.Get("cf-ray") + result.Observation.State = strings.TrimSpace(response.Header.Get(openAICodexTurnStateHeader)) + result.Observation.StateLength = len(result.Observation.State) + result.Observation.ResponseHeaders, result.Observation.ResponseHeadersOmitted = codexRoutingResponseHeaders(response.Header) target, _ := url.Parse("https://chatgpt.com/backend-api/codex/responses") changes := parseCodexRoutingCookies(response.Header, target, start) if err := preserveCodexCookieFirstSeen(ctx, h.state, h.key, scope, &clock, changes, start); err != nil { result.Observation.Code = "routing_persist" + result.Observation.Error = "record cookie first-seen time: " + err.Error() return marshal(result) } // Deletions are authoritative even on an unsuccessful response. A stale @@ -176,10 +226,23 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H savedClock, err := h.state.CompareSwapExtensionState(ctx, h.key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: clockKey, ExpectedRevision: clockRecord.Revision, Value: clockJSON, NextAt: codexCookieClockDeadline(clock)}) if err != nil || !savedClock.Applied { result.Observation.Code = "routing_stale" + if err != nil { + result.Observation.Error = "save cookie clock: " + err.Error() + } else { + result.Observation.Error = fmt.Sprintf("cookie clock changed concurrently (expected revision %d, current %d)", clockRecord.Revision, savedClock.Revision) + } return marshal(result) } if response.StatusCode != http.StatusOK && (query.Transport != "ws" || response.StatusCode != http.StatusSwitchingProtocols) { result.Observation.Code = "routing_upstream" + body, readErr := io.ReadAll(io.LimitReader(response.Body, codexRoutingProbeReadLimit)) + if readErr != nil { + result.Observation.Error = "read upstream error body: " + readErr.Error() + } + found := codexUpstreamErrorFromBody(body) + result.Observation.UpstreamErrorType, result.Observation.UpstreamErrorCode = found.Type, found.Code + result.Observation.UpstreamErrorMessage, result.Observation.UpstreamErrorParam = found.Message, found.Param + result.Observation.UpstreamBody = string(boundedCodexUpstreamBody(body)) return marshal(result) } if err := readCodexRoutingCompletion(response.Body, response.Header.Get("Content-Type"), query.Model, &result.Observation, response.Header); err != nil { @@ -194,6 +257,14 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H current, err := directory.PrepareCodexRoutingScope(ctx, query.AccountID, query.Transport) if err != nil || !scope.SameOwner(current) || ctx.Err() != nil { result.Observation.Code = "routing_stale" + switch { + case err != nil: + result.Observation.Error = "re-read routing scope: " + err.Error() + case ctx.Err() != nil: + result.Observation.Error = "probe context: " + ctx.Err().Error() + default: + result.Observation.Error = codexRoutingScopeChange(scope, current) + } return marshal(result) } if query.Stage == "acquire" { @@ -207,25 +278,37 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H bundleRef.Key = fixedCodexRoutingBundleKey(ctx, query, "live") if wireScope.ConnectionLeaseID == "" { result.Observation.Code = "routing_connection_unknown" + result.Observation.Error = "the verified response did not report the connection lease it used" return marshal(result) } } _, deadline := codexRoutingCookieHeader(bundle.Cookies, time.Now()) if deadline.IsZero() { result.Observation.Code = "routing_cookie_missing" + var names []string + for _, cookie := range (&http.Response{Header: response.Header}).Cookies() { + names = append(names, cookie.Name) + } + result.Observation.Error = fmt.Sprintf("no live __cflb/__oailb routing cookie after this response (Set-Cookie names: %s)", strings.Join(names, ", ")) return marshal(result) } bundle.ExpiresAt = earlierCodexTime(deadline, expires) - bundle.Observation = result.Observation + bundle.Observation = codexRoutingPrivateObservation(result.Observation) raw, _ := json.Marshal(bundle) previousBundle, readErr := h.state.ReadExtensionState(ctx, h.key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: bundleRef.Key}) if readErr != nil { result.Observation.Code = "routing_persist" + result.Observation.Error = "read bundle " + bundleRef.Key + ": " + readErr.Error() return marshal(result) } saved, err := h.state.CompareSwapExtensionState(ctx, h.key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: bundleRef.Key, ExpectedRevision: previousBundle.Revision, Value: raw, NextAt: &bundle.ExpiresAt}) if err != nil || !saved.Applied { result.Observation.Code = "routing_stale" + if err != nil { + result.Observation.Error = "save bundle " + bundleRef.Key + ": " + err.Error() + } else { + result.Observation.Error = fmt.Sprintf("bundle %s changed concurrently (expected revision %d, current %d)", bundleRef.Key, previousBundle.Revision, saved.Revision) + } return marshal(result) } result.Bundle = &extensionv1.CodexRoutingBundleRef{Key: bundleRef.Key, Revision: saved.Revision, ExpiresAt: bundle.ExpiresAt, ConnectionLeaseID: bundle.Scope.ConnectionLeaseID} @@ -241,19 +324,141 @@ func (h *nativeCodexHost) callCodexRouting(ctx context.Context, in extensionv1.H return marshal(result) } +// codexRoutingPrivateObservation is the copy kept inside host-private bundle +// and validation records. The raw STATE and response headers (Set-Cookie +// values included) stay only in the probe result and the runtime's tickets +// records, so bundle expiry and quality-run close keep clearing cookie values. +func codexRoutingPrivateObservation(observation extensionv1.CodexRoutingObservation) extensionv1.CodexRoutingObservation { + observation.State, observation.ResponseHeaders, observation.ResponseHeadersOmitted = "", nil, nil + return observation +} + +// codexRoutingResponseHeaders keeps upstream response header values whole, in +// name order, while they fit CodexRoutingHeaderLimit (name plus value bytes); +// the names of values that did not fit are returned separately. +func codexRoutingResponseHeaders(headers http.Header) ([]extensionv1.CodexRoutingHeader, []string) { + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + slices.Sort(names) + var kept []extensionv1.CodexRoutingHeader + var omitted []string + used := 0 + for _, name := range names { + for _, value := range headers[name] { + if used+len(name)+len(value) > extensionv1.CodexRoutingHeaderLimit { + if len(omitted) == 0 || omitted[len(omitted)-1] != name { + omitted = append(omitted, name) + } + continue + } + used += len(name) + len(value) + kept = append(kept, extensionv1.CodexRoutingHeader{Name: name, Value: value}) + } + } + return kept, omitted +} + +// codexRoutingScopeChange names the parts of a routing owner that differ. +func codexRoutingScopeChange(before, after extensionv1.CodexRoutingScope) string { + var changed []string + for _, field := range []struct{ name, before, after string }{ + {"account", fmt.Sprint(before.AccountID), fmt.Sprint(after.AccountID)}, + {"identity", before.Identity, after.Identity}, + {"profile_hash", before.ProfileHash, after.ProfileHash}, + {"route_hash", before.RouteHash, after.RouteHash}, + } { + if field.before != field.after { + changed = append(changed, fmt.Sprintf("%s %q -> %q", field.name, field.before, field.after)) + } + } + if len(changed) == 0 { + return "routing owner is incomplete (account, identity, profile and route hash are all required)" + } + return "routing owner changed: " + strings.Join(changed, "; ") +} + +// codexRoutingCheckFailure returns why a qualified bundle cannot serve this +// request, or "" when it can. +func (h *nativeCodexHost) codexRoutingCheckFailure(ctx context.Context, query extensionv1.CodexRoutingQuery, bundle codexRoutingPrivateBundle) string { + checker, ok := h.directory.(codexRoutingLeaseDirectory) + switch { + case !ok: + return "account directory cannot check connection leases" + case query.Bundle == nil: + return "check requires a bundle reference" + case bundle.Status != "qualified": + return fmt.Sprintf("bundle status is %q, not qualified", bundle.Status) + case bundle.Model != query.Model: + return fmt.Sprintf("bundle model %q differs from requested model %q", bundle.Model, query.Model) + case bundle.Scope.Transport != query.Transport: + return fmt.Sprintf("bundle transport %q differs from requested transport %q", bundle.Scope.Transport, query.Transport) + case query.Scope == nil: + return "check requires the qualified routing scope" + case !query.Scope.SameOwner(bundle.Scope): + return codexRoutingScopeChange(bundle.Scope, *query.Scope) + case query.Scope.Transport != bundle.Scope.Transport: + return fmt.Sprintf("qualified transport %q differs from bundle transport %q", query.Scope.Transport, bundle.Scope.Transport) + case query.Scope.ConnectionLeaseID != bundle.Scope.ConnectionLeaseID: + return fmt.Sprintf("qualified connection lease %q differs from bundle lease %q", query.Scope.ConnectionLeaseID, bundle.Scope.ConnectionLeaseID) + } + if err := checker.CheckCodexRoutingLease(ctx, bundle.Scope, bundle.ExpiresAt); err != nil { + return fmt.Sprintf("connection lease %q: %v", bundle.Scope.ConnectionLeaseID, err) + } + return "" +} + func readCodexRoutingBundle(ctx context.Context, store NativeCodexStateStore, key string, ref extensionv1.CodexRoutingBundleRef, scope extensionv1.CodexRoutingScope, qualified bool) (codexRoutingPrivateBundle, error) { var bundle codexRoutingPrivateBundle - if store == nil || !strings.HasPrefix(ref.Key, "bundle.") || len(ref.Key) > 80 || ref.Revision <= 0 || !time.Now().Before(ref.ExpiresAt) { - return bundle, errCodexRoutingUnavailable + now := time.Now() + switch { + case store == nil: + return bundle, codexRoutingUnavailable("routing state store unavailable") + case !strings.HasPrefix(ref.Key, "bundle.") || len(ref.Key) > 80 || ref.Revision <= 0: + return bundle, codexRoutingUnavailable("invalid bundle reference %q revision %d", ref.Key, ref.Revision) + case !now.Before(ref.ExpiresAt): + return bundle, codexRoutingUnavailable("bundle %s expired at %s", ref.Key, ref.ExpiresAt.UTC().Format(time.RFC3339)) } record, err := store.ReadExtensionState(ctx, key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: ref.Key}) - if err != nil || !record.Found || record.Revision != ref.Revision || json.Unmarshal(record.Value, &bundle) != nil || bundle.Schema != extensionv1.CodexRoutingSchema || !scope.SameOwner(bundle.Scope) || !time.Now().Before(bundle.ExpiresAt) || !ref.ExpiresAt.Equal(bundle.ExpiresAt) || ref.ConnectionLeaseID != bundle.Scope.ConnectionLeaseID || (qualified && bundle.Status != "qualified") { - return bundle, errCodexRoutingUnavailable + switch { + case err != nil: + return bundle, codexRoutingUnavailable("read bundle %s: %v", ref.Key, err) + case !record.Found: + return bundle, codexRoutingUnavailable("bundle %s not found", ref.Key) + case record.Revision != ref.Revision: + return bundle, codexRoutingUnavailable("bundle %s is at revision %d, reference names revision %d", ref.Key, record.Revision, ref.Revision) + } + if err := json.Unmarshal(record.Value, &bundle); err != nil { + return bundle, codexRoutingUnavailable("decode bundle %s: %v", ref.Key, err) + } + switch { + case bundle.Schema != extensionv1.CodexRoutingSchema: + return bundle, codexRoutingUnavailable("bundle %s has schema %d, expected %d", ref.Key, bundle.Schema, extensionv1.CodexRoutingSchema) + case !scope.SameOwner(bundle.Scope): + return bundle, codexRoutingUnavailable("bundle %s: %s", ref.Key, codexRoutingScopeChange(bundle.Scope, scope)) + case !now.Before(bundle.ExpiresAt): + return bundle, codexRoutingUnavailable("bundle %s expired at %s", ref.Key, bundle.ExpiresAt.UTC().Format(time.RFC3339)) + case !ref.ExpiresAt.Equal(bundle.ExpiresAt): + return bundle, codexRoutingUnavailable("bundle %s expiry %s differs from reference expiry %s", ref.Key, bundle.ExpiresAt.UTC().Format(time.RFC3339), ref.ExpiresAt.UTC().Format(time.RFC3339)) + case ref.ConnectionLeaseID != bundle.Scope.ConnectionLeaseID: + return bundle, codexRoutingUnavailable("bundle %s is bound to connection lease %q, reference names %q", ref.Key, bundle.Scope.ConnectionLeaseID, ref.ConnectionLeaseID) + case qualified && bundle.Status != "qualified": + return bundle, codexRoutingUnavailable("bundle %s status is %q, not qualified", ref.Key, bundle.Status) } clockRecord, err := store.ReadExtensionState(ctx, key, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: bundle.ClockKey}) var clock codexRoutingCookieClock - if err != nil || !clockRecord.Found || json.Unmarshal(clockRecord.Value, &clock) != nil || !clock.Scope.SameOwner(scope) { - return bundle, errCodexRoutingUnavailable + switch { + case err != nil: + return bundle, codexRoutingUnavailable("read cookie clock %s: %v", bundle.ClockKey, err) + case !clockRecord.Found: + return bundle, codexRoutingUnavailable("cookie clock %s not found", bundle.ClockKey) + } + if err := json.Unmarshal(clockRecord.Value, &clock); err != nil { + return bundle, codexRoutingUnavailable("decode cookie clock %s: %v", bundle.ClockKey, err) + } + if !clock.Scope.SameOwner(scope) { + return bundle, codexRoutingUnavailable("cookie clock %s: %s", bundle.ClockKey, codexRoutingScopeChange(clock.Scope, scope)) } for _, cookie := range bundle.Cookies { digest := codexRoutingDigest(cookie.Name, cookie.Value) @@ -267,11 +472,16 @@ func readCodexRoutingBundle(ctx context.Context, store NativeCodexStateStore, ke first, exists = seen.First, !seen.First.IsZero() } } - if !exists || !first.Equal(cookie.FirstSeen) || !time.Now().Before(cookie.ExpiresAt) { - return bundle, errCodexRoutingUnavailable + switch { + case !exists: + return bundle, codexRoutingUnavailable("bundle %s: first-seen record of cookie %s is missing", ref.Key, cookie.Name) + case !first.Equal(cookie.FirstSeen): + return bundle, codexRoutingUnavailable("bundle %s: cookie %s first seen at %s, bundle records %s", ref.Key, cookie.Name, first.UTC().Format(time.RFC3339), cookie.FirstSeen.UTC().Format(time.RFC3339)) + case !time.Now().Before(cookie.ExpiresAt): + return bundle, codexRoutingUnavailable("bundle %s: cookie %s expired at %s", ref.Key, cookie.Name, cookie.ExpiresAt.UTC().Format(time.RFC3339)) } if deleted, exists := clock.Tombstones[cookie.Name]; exists && !cookie.FirstSeen.After(deleted) { - return bundle, errCodexRoutingUnavailable + return bundle, codexRoutingUnavailable("bundle %s: upstream deleted cookie %s at %s", ref.Key, cookie.Name, deleted.UTC().Format(time.RFC3339)) } } return bundle, nil @@ -279,8 +489,13 @@ func readCodexRoutingBundle(ctx context.Context, store NativeCodexStateStore, ke func (s *OpenAIGatewayService) PrepareCodexRoutingScope(ctx context.Context, id int64, transport string) (extensionv1.CodexRoutingScope, error) { a, err := s.accountRepo.GetByID(ctx, id) - if err != nil || !isOpenAICodexTicketAccount(a) { - return extensionv1.CodexRoutingScope{}, errCodexRoutingUnavailable + switch { + case err != nil: + return extensionv1.CodexRoutingScope{}, codexRoutingUnavailable("read account %d: %v", id, err) + case a == nil: + return extensionv1.CodexRoutingScope{}, codexRoutingUnavailable("account %d not found", id) + case !isOpenAICodexTicketAccount(a): + return extensionv1.CodexRoutingScope{}, codexRoutingUnavailable("account %d (%s/%s, shadow=%t) is not an OpenAI OAuth/setup-token Codex account", a.ID, a.Platform, a.Type, a.IsShadow()) } identity, err := resolveCodexOutboundIdentityForAccountContext(ctx, a, s.codexIdentityOverrideUA(a)) if err != nil { @@ -289,7 +504,7 @@ func (s *OpenAIGatewayService) PrepareCodexRoutingScope(ctx context.Context, id seed, _ := codexFingerprintSeed(a.Extra) profile := codexRoutingDigest(identity.userAgent, identity.originator, identity.version, seed, resolveConvergedInstallationID(a, seed), string(a.GetCodexFingerprintMode()), "go-crypto-tls", "native-http1") config := s.openAICodexTicketConfig() - return extensionv1.CodexRoutingScope{AccountID: id, Identity: CodexTicketAccountIdentity(a), ProfileHash: profile, RouteHash: codexRoutingDigest(resolveAccountProxyURL(a), config.HarvestProxyURL), RouteEvidence: "connection_required", Transport: transport}, nil + return extensionv1.CodexRoutingScope{AccountID: id, Identity: CodexTicketAccountIdentity(a), ProfileHash: profile, RouteHash: codexRoutingDigest(resolveAccountProxyURL(a), config.HarvestProxyURL), RouteEvidence: "connection_required", Transport: transport, AccountProxyID: qualityProxyID(a)}, nil } func (s *OpenAIGatewayService) CheckCodexRoutingLease(ctx context.Context, scope extensionv1.CodexRoutingScope, expiresAt time.Time) error { @@ -310,19 +525,22 @@ func (s *OpenAIGatewayService) ExecuteCodexRoutingProbe(ctx context.Context, que } cfg := s.openAICodexTicketConfig() _, validation := codexValidationFromContext(ctx) - if !cfg.Enabled || (!slices.Contains(cfg.Models, query.Model) && !validation) { - return nil, scope, errCodexRoutingUnavailable + if !cfg.Enabled { + return nil, scope, codexRoutingUnavailable("Codex route acquisition is switched off") + } + if !slices.Contains(cfg.Models, query.Model) && !validation { + return nil, scope, codexRoutingUnavailable("model %q is not in the routing model list %v", query.Model, cfg.Models) } if query.Transport == "ws" { return s.executeCodexRoutingWSProbe(ctx, query, cookies, deadline, scope) } a, err := s.accountRepo.GetByID(ctx, query.AccountID) if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("read account %d: %v", query.AccountID, err) } token, _, err := s.GetAccessToken(withCodexTicketCredentials(ctx), a) if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, fmt.Errorf("%w: %v", errCodexRoutingToken, err) } request, err := s.buildCodexRoutingProbe(ctx, a, query.Model, token, cookies, query.ReasoningEffort) if err != nil { @@ -330,14 +548,14 @@ func (s *OpenAIGatewayService) ExecuteCodexRoutingProbe(ctx context.Context, que } if query.Stage == "acquire" { if cfg.HarvestProxyURL == "" { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("no acquisition proxy is configured") } response, err := s.doCodexRoutingAcquisition(request, cfg.HarvestProxyURL, a.ID) return response, scope, err } transport, ok := s.httpUpstream.(CodexConnectionLeaseUpstream) if !ok { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("the upstream transport cannot hold a verified connection lease") } if err := reserveCodexQualitySend(request, a, nil); err != nil { return nil, scope, err diff --git a/backend/internal/service/codex_routing_identity_observation.go b/backend/internal/service/codex_routing_identity_observation.go index ab1d4c352fb0..f24184a1136c 100644 --- a/backend/internal/service/codex_routing_identity_observation.go +++ b/backend/internal/service/codex_routing_identity_observation.go @@ -10,7 +10,10 @@ import ( type codexIdentityBodyKey struct{} +// Raw values are shown to administrators next to the stable digests. type CodexIdentityFieldObservation struct { + HeaderValue string `json:"header_value,omitempty"` + BodyValue string `json:"body_value,omitempty"` HeaderDigest string `json:"header_digest,omitempty"` BodyDigest string `json:"body_digest,omitempty"` Consistency string `json:"consistency"` @@ -53,7 +56,7 @@ func codexIdentityFieldObservations(headers http.Header, body codexIdentityBodyO for key, raw := range values { header := strings.TrimSpace(raw) value := body.Fields[key] - observation := CodexIdentityFieldObservation{Consistency: "uninspected"} + observation := CodexIdentityFieldObservation{Consistency: "uninspected", HeaderValue: header, BodyValue: value} if header != "" { observation.HeaderDigest = codexRoutingDigest(header)[:16] } diff --git a/backend/internal/service/codex_routing_identity_observation_test.go b/backend/internal/service/codex_routing_identity_observation_test.go index 1d46001bc0c9..b16600600245 100644 --- a/backend/internal/service/codex_routing_identity_observation_test.go +++ b/backend/internal/service/codex_routing_identity_observation_test.go @@ -25,6 +25,10 @@ func TestCodexRoutingIdentityObservationsAreBoundedAndRedacted(t *testing.T) { require.Equal(t, "missing", fields["turn"].Consistency) require.Len(t, fields["session"].HeaderDigest, 16) require.NotEqual(t, "same-session", fields["session"].HeaderDigest) + // Raw identifiers are shown next to the digests. + require.Equal(t, "same-session", fields["session"].HeaderValue) + require.Equal(t, "different-thread", fields["thread"].HeaderValue) + require.Equal(t, "body-thread", fields["thread"].BodyValue) request.GetBody = nil fields = codexIdentityFieldObservations(request.Header, inspectCodexIdentityBody(request)) require.Equal(t, "uninspected", fields["session"].Consistency) diff --git a/backend/internal/service/codex_routing_profile.go b/backend/internal/service/codex_routing_profile.go index 18965660e3fc..46cad46d8d50 100644 --- a/backend/internal/service/codex_routing_profile.go +++ b/backend/internal/service/codex_routing_profile.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "time" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" @@ -14,38 +15,44 @@ type CodexProfileSelection struct { func (s *OpenAIGatewayService) SelectCodexProfile(ctx context.Context, id int64, choice CodexProfileSelection) error { if choice.Profile != "captured_windows_cli" && choice.Profile != "keep_legacy" { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("unknown profile %q (expected captured_windows_cli or keep_legacy)", choice.Profile) } account, err := s.accountRepo.GetByID(ctx, id) - if err != nil || !isOpenAICodexTicketAccount(account) { - return errCodexRoutingUnavailable + switch { + case err != nil: + return codexRoutingUnavailable("read account %d: %v", id, err) + case !isOpenAICodexTicketAccount(account): + return codexRoutingUnavailable("account %d is not an OpenAI OAuth/setup-token Codex account", id) } if choice.Profile == "keep_legacy" { return nil } if choice.ExpectedRevision.IsZero() || !choice.ExpectedRevision.Equal(account.UpdatedAt) { - return ErrPluginStateChanged + return fmt.Errorf("%w: account revision is %s, the request expected %s", ErrPluginStateChanged, account.UpdatedAt.UTC().Format(time.RFC3339Nano), choice.ExpectedRevision.UTC().Format(time.RFC3339Nano)) } seed, valid := codexFingerprintSeed(account.Extra) if !valid { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("account %d has no valid Codex device seed", id) } result, err := invokeCodexIdentityPolicyForAccount(ctx, account, "codex.identity.derive", extensionv1.CodexIdentityQuery{Seed: seed, Preset: choice.Profile}) - if err != nil || !result.Valid || result.Profile.Version != 2 { - return errCodexRoutingUnavailable + if err != nil { + return codexRoutingUnavailable("derive profile %s: %v", choice.Profile, err) + } + if !result.Valid || result.Profile.Version != 2 { + return codexRoutingUnavailable("derived profile %s is not usable (valid=%t, version=%d)", choice.Profile, result.Valid, result.Profile.Version) } writer, ok := s.accountRepo.(interface { UpdateExtraIfRevision(context.Context, int64, time.Time, map[string]any) (bool, error) }) if !ok { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("account repository cannot update extra by revision") } applied, err := writer.UpdateExtraIfRevision(ctx, id, choice.ExpectedRevision, map[string]any{CodexClientIdentityExtraKey: codexClientIdentityExtraValue(codexClientIdentity(result.Profile), time.Now())}) if err != nil { return err } if !applied { - return ErrPluginStateChanged + return fmt.Errorf("%w: the account changed while the profile was applied", ErrPluginStateChanged) } return nil } diff --git a/backend/internal/service/codex_routing_proxy.go b/backend/internal/service/codex_routing_proxy.go index 48c4cb565ad1..f65a25bb02e5 100644 --- a/backend/internal/service/codex_routing_proxy.go +++ b/backend/internal/service/codex_routing_proxy.go @@ -21,29 +21,29 @@ type codexRoutingProxyTrust struct { func (s *OpenAIGatewayService) doCodexRoutingAcquisition(request *http.Request, proxyURL string, accountID int64) (*http.Response, error) { if request == nil || request.URL == nil || request.Method != http.MethodPost || request.URL.Scheme != "https" || request.URL.Host != "chatgpt.com" || request.URL.Path != "/backend-api/codex/responses" || request.URL.RawQuery != "" || request.URL.User != nil || s.nativeCodexRuntime == nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("acquisition accepts only POST https://chatgpt.com/backend-api/codex/responses with a loaded Codex runtime") } _, err := proxytransport.ParseEndpoint(proxyURL) if err != nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("acquisition proxy: %v", err) } normal := proxyURL installation := s.nativeCodexRuntime.metadata() store := s.nativeCodexRuntime.repo ok := store != nil if installation == nil || !ok { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("Codex runtime state is not loaded") } ctx := WithNativeCodexExecution(request.Context(), installation) record, err := store.ReadExtensionState(ctx, NativeCodexPluginKey, extensionv1.StateRequest{Namespace: "proxy-trust", Key: codexRoutingDigest(normal, "chatgpt.com")}) if err != nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("read pinned acquisition proxy certificate: %v", err) } wire := request.Clone(WithHTTPUpstreamProfile(request.Context(), HTTPUpstreamProfileOpenAIHarvest)) wire.Header.Del("Cookie") if !record.Found { if s.httpUpstream == nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("no upstream transport is configured") } if err := reserveCodexQualityAcquisition(wire, accountID); err != nil { return nil, err @@ -53,18 +53,27 @@ func (s *OpenAIGatewayService) doCodexRoutingAcquisition(request *http.Request, return response, err } var trust codexRoutingProxyTrust - if len(record.Value) > 128*1024 || json.Unmarshal(record.Value, &trust) != nil || len(trust.Certificates) != 1 { - return nil, errCodexRoutingUnavailable + if len(record.Value) > 128*1024 { + return nil, codexRoutingUnavailable("pinned proxy certificate record of %d bytes exceeds 128 KiB", len(record.Value)) + } + if err := json.Unmarshal(record.Value, &trust); err != nil { + return nil, codexRoutingUnavailable("decode pinned proxy certificate: %v", err) + } + if len(trust.Certificates) != 1 { + return nil, codexRoutingUnavailable("pinned proxy certificate record holds %d certificates, expected 1", len(trust.Certificates)) } anchor, err := x509.ParseCertificate(trust.Certificates[0]) - if err != nil || codexRoutingDigest(string(anchor.RawSubjectPublicKeyInfo)) != trust.Fingerprint { - return nil, errCodexRoutingUnavailable + if err != nil { + return nil, codexRoutingUnavailable("parse pinned proxy certificate: %v", err) + } + if digest := codexRoutingDigest(string(anchor.RawSubjectPublicKeyInfo)); digest != trust.Fingerprint { + return nil, codexRoutingUnavailable("pinned proxy certificate key digest %s does not match its record %s", digest, trust.Fingerprint) } roots := x509.NewCertPool() roots.AddCert(anchor) transport, err := codexRoutingAcquisitionTransport(normal, roots) if err != nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("acquisition transport: %v", err) } defer transport.CloseIdleConnections() client := &http.Client{Transport: transport, Timeout: 25 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} @@ -78,25 +87,27 @@ func (s *OpenAIGatewayService) doCodexRoutingAcquisition(request *http.Request, func verifyCodexRoutingProxyCertificate(state tls.ConnectionState, pinnedRoots *x509.CertPool) error { if state.ServerName == "" || len(state.PeerCertificates) == 0 { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("TLS handshake reported no server name or peer certificate") } intermediates := x509.NewCertPool() for _, certificate := range state.PeerCertificates[1:] { intermediates.AddCert(certificate) } opts := x509.VerifyOptions{DNSName: state.ServerName, Intermediates: intermediates} - if _, err := state.PeerCertificates[0].Verify(opts); err == nil { + _, systemErr := state.PeerCertificates[0].Verify(opts) + if systemErr == nil { return nil } // The explicit anchor cannot authorize an HTTPS proxy endpoint, another // origin, an expired certificate or a changed certificate chain. No IO in // this path learns or writes certificate trust, with or without OAuth. + leaf := state.PeerCertificates[0] if state.ServerName != "chatgpt.com" || pinnedRoots == nil { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("certificate %q (issuer %q) for %s is not trusted by the system roots and no pinned proxy certificate applies: %v", leaf.Subject.String(), leaf.Issuer.String(), state.ServerName, systemErr) } opts.Roots = pinnedRoots - if _, err := state.PeerCertificates[0].Verify(opts); err != nil { - return errCodexRoutingUnavailable + if _, err := leaf.Verify(opts); err != nil { + return codexRoutingUnavailable("certificate %q (issuer %q) for %s is trusted neither by the system roots (%v) nor by the pinned proxy certificate (%v); test the acquisition proxy again", leaf.Subject.String(), leaf.Issuer.String(), state.ServerName, systemErr, err) } return nil } @@ -104,7 +115,7 @@ func verifyCodexRoutingProxyCertificate(state tls.ConnectionState, pinnedRoots * func codexRoutingAcquisitionTransport(raw string, pinnedRoots *x509.CertPool) (*http.Transport, error) { address, err := proxytransport.ParseEndpoint(raw) if err != nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("acquisition proxy: %v", err) } transport := &http.Transport{ DialContext: (&net.Dialer{Timeout: 8 * time.Second}).DialContext, diff --git a/backend/internal/service/codex_routing_test.go b/backend/internal/service/codex_routing_test.go index d88b1637ff23..9d4140a46fa2 100644 --- a/backend/internal/service/codex_routing_test.go +++ b/backend/internal/service/codex_routing_test.go @@ -3,6 +3,8 @@ package service import ( "context" "encoding/json" + "errors" + "fmt" "github.com/Wei-Shaw/sub2api/internal/config" "github.com/coder/websocket" "io" @@ -81,6 +83,9 @@ type routingHostDirectoryFixture struct { account extensionv1.Account scope extensionv1.CodexRoutingScope response string + status int + headers http.Header + probeErr error requests int onSend func() } @@ -138,6 +143,9 @@ func (d *routingHostDirectoryFixture) ExecuteCodexRoutingProbe(_ context.Context } headers := http.Header{"Content-Type": []string{"text/event-stream"}, "Set-Cookie": []string{"__cflb=synthetic-route; Path=/; Secure; Max-Age=100", "__oailb=synthetic-lb; Path=/; Secure; Max-Age=100"}} scope := d.scope + if d.probeErr != nil { + return nil, scope, d.probeErr + } if q.Stage == "verify" { if cookies == "" { return nil, scope, errCodexRoutingUnavailable @@ -145,7 +153,14 @@ func (d *routingHostDirectoryFixture) ExecuteCodexRoutingProbe(_ context.Context scope.ConnectionLeaseID = "actual-connection" scope.RouteEvidence = "connection" } - return &http.Response{StatusCode: 200, Header: headers, Body: io.NopCloser(strings.NewReader(d.response))}, scope, nil + status := http.StatusOK + if d.status != 0 { + status = d.status + } + for name, values := range d.headers { + headers[name] = values + } + return &http.Response{StatusCode: status, Header: headers, Body: io.NopCloser(strings.NewReader(d.response))}, scope, nil } func routingHostFixture() (*nativeCodexHost, *routingHostDirectoryFixture, *routingMemoryStore) { @@ -220,8 +235,6 @@ func TestCodexRoutingHostSeparatesCandidateQualifiedAndPrivateMaterial(t *testin raw, _ := json.Marshal(query) result, err := host.Call(context.Background(), extensionv1.HostInvocation{Operation: extensionv1.HostCodexRoutingProbe, Payload: raw}) require.NoError(t, err) - require.NotContains(t, string(result.Payload), "synthetic-route") - require.NotContains(t, string(result.Payload), "synthetic-lb") var value extensionv1.CodexRoutingProbeResult require.NoError(t, json.Unmarshal(result.Payload, &value)) return value @@ -230,6 +243,9 @@ func TestCodexRoutingHostSeparatesCandidateQualifiedAndPrivateMaterial(t *testin candidate := call(query) require.True(t, candidate.Valid) require.Empty(t, candidate.Scope.ConnectionLeaseID) + // Administrators see every upstream response header as sent, Set-Cookie + // included; the bundle records themselves stay in the private namespace. + require.Contains(t, candidate.Observation.ResponseHeaders, extensionv1.CodexRoutingHeader{Name: "Set-Cookie", Value: "__cflb=synthetic-route; Path=/; Secure; Max-Age=100"}) query.Stage, query.Bundle = "verify", candidate.Bundle verified := call(query) require.True(t, verified.Valid) @@ -276,6 +292,65 @@ func TestCodexRoutingHostRejectsMismatchedModelAndChangedOwner(t *testing.T) { } } +func TestCodexRoutingProbeKeepsUpstreamErrorAndTransportCause(t *testing.T) { + probe := func(configure func(*routingHostDirectoryFixture)) extensionv1.CodexRoutingObservation { + host, directory, _ := routingHostFixture() + configure(directory) + raw, _ := json.Marshal(extensionv1.CodexRoutingQuery{AccountID: 7, Model: "gpt-6-astra", Transport: "http", OperationID: "one", Stage: "acquire"}) + result, err := host.Call(context.Background(), extensionv1.HostInvocation{Operation: extensionv1.HostCodexRoutingProbe, Payload: raw}) + require.NoError(t, err) + var value extensionv1.CodexRoutingProbeResult + require.NoError(t, json.Unmarshal(result.Payload, &value)) + return value.Observation + } + body := `{"error":{"type":"rate_limit","code":"rate_limit_exceeded","message":"Rate limit reached for gpt-6-astra"}}` + large := strings.Repeat("v", extensionv1.CodexRoutingHeaderLimit) + rejected := probe(func(d *routingHostDirectoryFixture) { + d.status, d.response = http.StatusTooManyRequests, body + d.headers = http.Header{"X-Request-Id": []string{"req_fixture"}, "Cf-Ray": []string{"ray-fixture"}, "X-Codex-Turn-State": []string{"raw-state-value"}, "X-Large": []string{large}} + }) + require.Equal(t, "raw-state-value", rejected.State) + require.Equal(t, len("raw-state-value"), rejected.StateLength) + require.Contains(t, rejected.ResponseHeaders, extensionv1.CodexRoutingHeader{Name: "X-Codex-Turn-State", Value: "raw-state-value"}) + require.Contains(t, rejected.ResponseHeaders, extensionv1.CodexRoutingHeader{Name: "Set-Cookie", Value: "__oailb=synthetic-lb; Path=/; Secure; Max-Age=100"}) + require.Equal(t, []string{"X-Large"}, rejected.ResponseHeadersOmitted, "a value over the 4 KiB bound is named, never cut") + require.Equal(t, "routing_upstream", rejected.Code) + require.Equal(t, http.StatusTooManyRequests, rejected.HTTPStatus) + require.Equal(t, "rate_limit", rejected.UpstreamErrorType) + require.Equal(t, "rate_limit_exceeded", rejected.UpstreamErrorCode) + require.Equal(t, "Rate limit reached for gpt-6-astra", rejected.UpstreamErrorMessage) + require.Equal(t, body, rejected.UpstreamBody) + require.Equal(t, "req_fixture", rejected.RequestID) + require.Equal(t, "ray-fixture", rejected.CFRay) + failed := probe(func(d *routingHostDirectoryFixture) { + d.response = "data: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"failed\",\"model\":\"gpt-6-astra\",\"instructions\":\"private instructions\",\"tools\":[{\"name\":\"private_tool\"}],\"output\":[{\"text\":\"private output\"}],\"error\":{\"code\":\"server_is_overloaded\",\"message\":\"Our servers are overloaded\"}}}\n\n" + }) + require.Equal(t, "routing_capacity", failed.Code) + require.Equal(t, "server_is_overloaded", failed.UpstreamErrorCode) + require.Equal(t, "Our servers are overloaded", failed.UpstreamErrorMessage) + require.JSONEq(t, `{"type":"response.failed","response":{"id":"resp_fixture","status":"failed","model":"gpt-6-astra","error":{"code":"server_is_overloaded","message":"Our servers are overloaded"}}}`, failed.UpstreamBody, "the failing event keeps only its error and response identity") + transport := probe(func(d *routingHostDirectoryFixture) { + d.probeErr = errors.New("proxyconnect tcp: dial tcp 192.0.2.1:8080: connect: connection refused") + }) + require.Equal(t, "routing_transport", transport.Code) + require.Contains(t, transport.Error, "connection refused") + token := probe(func(d *routingHostDirectoryFixture) { + d.probeErr = fmt.Errorf("%w: refresh token revoked", errCodexRoutingToken) + }) + require.Equal(t, "ticket_token", token.Code) + require.Contains(t, token.Error, "refresh token revoked") +} + +func TestCodexRoutingRefusalIsNeverAPersistentTransportFault(t *testing.T) { + for _, reason := range []string{"dial tcp 192.0.2.1:5432: connect: connection refused", "lookup proxy.example: no such host", "username/password authentication failed"} { + require.True(t, classifyUpstreamTransportError(errors.New(reason)).Persistent, "control: the same text from a real transport error is durable") + require.False(t, classifyUpstreamTransportError(codexRoutingUnavailable("read bundle bundle.live: %s", reason)).Persistent) + refusal := fmt.Errorf("%w: %w", ErrOpenAICodexTicketUnavailable, &NativeCodexResultError{Operation: "inject", Code: "ticket_missing", Message: reason}) + require.False(t, classifyUpstreamTransportError(refusal).Persistent) + require.Contains(t, refusal.Error(), reason, "Ops keeps the full reason") + } +} + func TestCodexRoutingValidationBudgetSevenStagesAndOneOwner(t *testing.T) { store := &routingMemoryStore{values: map[string]extensionv1.StateResult{}} ctx := context.Background() diff --git a/backend/internal/service/codex_routing_transport.go b/backend/internal/service/codex_routing_transport.go index 7a6e5c3d20b1..457572f2a60c 100644 --- a/backend/internal/service/codex_routing_transport.go +++ b/backend/internal/service/codex_routing_transport.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "slices" @@ -31,20 +32,30 @@ func (s *OpenAIGatewayService) codexRoutingApplies(account *Account, model strin func (m *NativeCodexRuntime) codexRoutingQualification(ctx context.Context, account *Account, model string) (*extensionv1.CodexRoutingQualification, *NativeCodexMetadata, error) { installation := m.metadata() if installation == nil { - return nil, nil, errCodexRoutingUnavailable + return nil, nil, codexRoutingUnavailable("native Codex runtime is not loaded") } raw, _ := json.Marshal(extensionv1.SchedulingRequest{Account: *extensionAccount(account), Model: model, Now: time.Now().UTC()}) result, err := m.Invoke(ctx, account.Platform, account.Type, extensionv1.Invocation{Capability: extensionv1.CapabilityRequest, Operation: "inject", AccountID: account.ID, Payload: raw}) + switch { + case err != nil: + return nil, installation, codexRoutingUnavailable("inject: %v", err) + case result.Code != "": + // The runtime's result code and reason stay inspectable (errors.As). + return nil, installation, fmt.Errorf("%w: %w", errCodexRoutingUnavailable, &NativeCodexResultError{Operation: "inject", Code: result.Code, Message: result.Message}) + } var injection extensionv1.CodexRoutingInjection - if err != nil || result.Code != "" || json.Unmarshal(result.Payload, &injection) != nil || !injection.Qualification.Valid(time.Now(), account.ID, CodexTicketAccountIdentity(account), model) { - return nil, installation, errCodexRoutingUnavailable + if err := json.Unmarshal(result.Payload, &injection); err != nil { + return nil, installation, codexRoutingUnavailable("decode the inject result: %v", err) + } + if problem := injection.Qualification.Problem(time.Now(), account.ID, CodexTicketAccountIdentity(account), model); problem != "" { + return nil, installation, codexRoutingUnavailable("%s", problem) } return injection.Qualification, installation, nil } func (s *OpenAIGatewayService) prepareQualifiedCodexRequest(request *http.Request, account *Account, model string) (*http.Request, *extensionv1.CodexRoutingQualification, *NativeCodexMetadata, error) { if request.URL == nil || request.URL.Scheme != "https" || request.URL.Hostname() != "chatgpt.com" || (request.URL.Path != "/backend-api/codex/responses" && request.URL.Path != "/backend-api/codex/responses/compact") { - return request, nil, nil, errCodexRoutingUnavailable + return request, nil, nil, codexRoutingUnavailable("a verified route only serves https://chatgpt.com/backend-api/codex/responses[/compact], not %v", request.URL) } var q *extensionv1.CodexRoutingQualification var installation *NativeCodexMetadata @@ -58,17 +69,29 @@ func (s *OpenAIGatewayService) prepareQualifiedCodexRequest(request *http.Reques return request, nil, installation, err } scope, err := s.PrepareCodexRoutingScope(request.Context(), account.ID, "http") - if err != nil || !q.Scope.SameOwner(scope) || q.Scope.Transport != "http" || q.Scope.ConnectionLeaseID == "" { - return request, q, installation, errCodexRoutingUnavailable + switch { + case err != nil: + return request, q, installation, codexRoutingUnavailable("routing scope: %v", err) + case !q.Scope.SameOwner(scope): + return request, q, installation, codexRoutingUnavailable("route qualification %s", codexRoutingScopeChange(q.Scope, scope)) + case q.Scope.Transport != "http" || q.Scope.ConnectionLeaseID == "": + return request, q, installation, codexRoutingUnavailable("route qualification is bound to transport %q connection %q; an HTTP connection lease is required", q.Scope.Transport, q.Scope.ConnectionLeaseID) } store := s.nativeCodexRuntime.repo bundle, err := readCodexRoutingBundle(request.Context(), store, NativeCodexPluginKey, q.Bundle, scope, true) - if err != nil || bundle.Model != model || bundle.Scope.ConnectionLeaseID != q.Scope.ConnectionLeaseID || bundle.Scope.Transport != q.Scope.Transport || q.ExpiresAt.After(bundle.ExpiresAt) { - return request, q, installation, errCodexRoutingUnavailable + switch { + case err != nil: + return request, q, installation, err + case bundle.Model != model: + return request, q, installation, codexRoutingUnavailable("route bundle %s is for model %q, not %q", q.Bundle.Key, bundle.Model, model) + case bundle.Scope.ConnectionLeaseID != q.Scope.ConnectionLeaseID || bundle.Scope.Transport != q.Scope.Transport: + return request, q, installation, codexRoutingUnavailable("route bundle %s is bound to %s connection %q, the qualification to %s connection %q", q.Bundle.Key, bundle.Scope.Transport, bundle.Scope.ConnectionLeaseID, q.Scope.Transport, q.Scope.ConnectionLeaseID) + case q.ExpiresAt.After(bundle.ExpiresAt): + return request, q, installation, codexRoutingUnavailable("route qualification (until %s) outlives bundle %s (until %s)", q.ExpiresAt.UTC().Format(time.RFC3339), q.Bundle.Key, bundle.ExpiresAt.UTC().Format(time.RFC3339)) } cookies, _ := codexRoutingCookieHeader(bundle.Cookies, time.Now()) if cookies == "" { - return request, q, installation, errCodexRoutingUnavailable + return request, q, installation, codexRoutingUnavailable("route bundle %s has no live routing cookie left (%d stored)", q.Bundle.Key, len(bundle.Cookies)) } wire := request.Clone(request.Context()) wire.Header.Set("Cookie", cookies) @@ -102,7 +125,7 @@ func (s *OpenAIGatewayService) doQualifiedCodexUpstream(request *http.Request, a }() transport, ok := s.httpUpstream.(CodexConnectionLeaseUpstream) if !ok { - return nil, true, errCodexRoutingUnavailable + return nil, true, codexRoutingUnavailable("the upstream transport cannot hold a verified connection lease") } start := time.Now() if err := reserveCodexQualitySend(wire, account, q); err != nil { @@ -117,7 +140,7 @@ func (s *OpenAIGatewayService) doQualifiedCodexUpstream(request *http.Request, a return response, true, err } if err != nil { - s.publishCodexRoutingObservation(request.Context(), account, installation, q, extensionv1.CodexRoutingObservation{Code: "routing_connection_expired", Stage: "business", RequestedModel: model, ObservedAt: time.Now().UTC(), Transport: "http", CookieSent: false}, nil) + s.publishCodexRoutingObservation(request.Context(), account, installation, q, extensionv1.CodexRoutingObservation{Code: "routing_connection_expired", Stage: "business", RequestedModel: model, ObservedAt: time.Now().UTC(), Transport: "http", CookieSent: false, Error: err.Error()}, nil) return nil, true, err } if observer := codexWireObserverFromContext(wire.Context()); observer != nil { @@ -125,7 +148,7 @@ func (s *OpenAIGatewayService) doQualifiedCodexUpstream(request *http.Request, a } observeCodexInfrastructureCookies(q.Scope, response.Header, q.ExpiresAt) s.observeCodexWire(request.Context(), account, wire, response, q) - observation := extensionv1.CodexRoutingObservation{Stage: "business", Code: "routing_incomplete", HTTPStatus: response.StatusCode, RequestedModel: model, StateLength: len(response.Header.Get(openAICodexTurnStateHeader)), ObservedAt: time.Now().UTC(), Transport: "http", CookieSent: true} + observation := extensionv1.CodexRoutingObservation{Stage: "business", Code: "routing_incomplete", HTTPStatus: response.StatusCode, RequestedModel: model, StateLength: len(response.Header.Get(openAICodexTurnStateHeader)), ObservedAt: time.Now().UTC(), Transport: "http", CookieSent: true, RequestID: response.Header.Get("x-request-id"), CFRay: response.Header.Get("cf-ray")} deleted := s.recordCodexRoutingDeletions(request.Context(), installation, q, response.Header, observation.ObservedAt) if wire.URL.Path == "/backend-api/codex/responses/compact" { // Compaction is not a model completion and cannot renew qualification. @@ -148,6 +171,9 @@ func (s *OpenAIGatewayService) doQualifiedCodexUpstream(request *http.Request, a observation.ModelMatched = observation.Completed && !completion.Mismatch && completion.Model == model observation.DurationMS = time.Since(start).Milliseconds() observation.Code = completion.observationCode(model, response.StatusCode) + if observation.Code != "routing_verified" { + completion.apply(&observation, response.StatusCode) + } if deleted { observation.Code = "routing_cookie_deleted" } @@ -340,7 +366,7 @@ func (s *OpenAIGatewayService) refreshObservedCodexCookies(ctx context.Context, } // A refreshed Cookie cannot extend a connection lease. A new connection // always needs a new complete validation even if cookies remain live. - bundle.Cookies, bundle.ClockRevision, bundle.Observation = clock.selected(bundle.Cookies, changes, time.Now()), saved.Revision, observation + bundle.Cookies, bundle.ClockRevision, bundle.Observation = clock.selected(bundle.Cookies, changes, time.Now()), saved.Revision, codexRoutingPrivateObservation(observation) _, expiry := codexRoutingCookieHeader(bundle.Cookies, time.Now()) if expiry.IsZero() { return nil diff --git a/backend/internal/service/codex_routing_validation.go b/backend/internal/service/codex_routing_validation.go index 43a823eff2b8..d9ce6797e29a 100644 --- a/backend/internal/service/codex_routing_validation.go +++ b/backend/internal/service/codex_routing_validation.go @@ -57,24 +57,29 @@ func consumeCodexValidationBudget(ctx context.Context, store NativeCodexStateSto return nil } if validation.AccountID != query.AccountID || validation.Model != query.Model || validation.Transport != query.Transport || !slices.Contains(codexValidationModels, query.Model) { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("validation %s is bound to account %d %s/%s, probe asked for account %d %s/%s", validation.ID, validation.AccountID, validation.Model, validation.Transport, query.AccountID, query.Model, query.Transport) } stage := query.Model + ":" + query.Transport + ":" + query.Stage if query.Transport == "ws" && query.Stage != "verify" { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("WebSocket validation only has a verify stage, not %q", query.Stage) } key := "validation." + validation.ID for range 3 { previous, err := store.ReadExtensionState(ctx, plugin, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: key}) if err != nil { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("read validation budget %s: %v", validation.ID, err) } budget := codexValidationBudget{AccountID: query.AccountID, Stages: map[string]bool{}} if previous.Found && (json.Unmarshal(previous.Value, &budget) != nil || budget.AccountID != query.AccountID || budget.Stages == nil) { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("validation budget %s belongs to account %d or is invalid: %s", validation.ID, budget.AccountID, string(previous.Value)) } - if budget.Used >= 7 || budget.Stages[stage] || (query.Transport == "ws" && budget.Stages["ws-spent"]) { - return errCodexRoutingUnavailable + switch { + case budget.Used >= 7: + return codexRoutingUnavailable("validation %s already used %d of 7 upstream calls", validation.ID, budget.Used) + case budget.Stages[stage]: + return codexRoutingUnavailable("validation %s already spent stage %s", validation.ID, stage) + case query.Transport == "ws" && budget.Stages["ws-spent"]: + return codexRoutingUnavailable("validation %s already spent its WebSocket stage", validation.ID) } budget.Used++ budget.Stages[stage] = true @@ -84,32 +89,44 @@ func consumeCodexValidationBudget(ctx context.Context, store NativeCodexStateSto raw, _ := json.Marshal(budget) next, err := store.CompareSwapExtensionState(ctx, plugin, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: key, ExpectedRevision: previous.Revision, Value: raw}) if err != nil { - return errCodexRoutingUnavailable + return codexRoutingUnavailable("save validation budget %s: %v", validation.ID, err) } if next.Applied { return nil } } - return errCodexRoutingUnavailable + return codexRoutingUnavailable("validation budget %s kept changing concurrently", validation.ID) } func (s *OpenAIGatewayService) ValidateCodexRouting(ctx context.Context, id int64, request CodexRoutingValidationRequest) (*CodexRoutingValidationResult, error) { - if uuid.Validate(request.ValidationID) != nil || !slices.Contains(codexValidationModels, request.Model) || (request.Transport != "http" && request.Transport != "ws") || s.nativeCodexRuntime == nil { - return nil, errCodexRoutingUnavailable + if err := uuid.Validate(request.ValidationID); err != nil { + return nil, codexRoutingUnavailable("validation_id %q is not a UUID: %v", request.ValidationID, err) + } + if !slices.Contains(codexValidationModels, request.Model) { + return nil, codexRoutingUnavailable("model %q is not one of %v", request.Model, codexValidationModels) + } + if request.Transport != "http" && request.Transport != "ws" { + return nil, codexRoutingUnavailable("transport must be http or ws, got %q", request.Transport) + } + if s.nativeCodexRuntime == nil { + return nil, codexRoutingUnavailable("native Codex runtime is not configured") } account, err := s.accountRepo.GetByID(ctx, id) - if err != nil || !isOpenAICodexTicketAccount(account) { - return nil, errCodexRoutingUnavailable + switch { + case err != nil: + return nil, codexRoutingUnavailable("read account %d: %v", id, err) + case !isOpenAICodexTicketAccount(account): + return nil, codexRoutingUnavailable("account %d is not an OpenAI OAuth/setup-token Codex account", id) } installation := s.nativeCodexRuntime.metadata() store := s.nativeCodexRuntime.repo ok := store != nil if installation == nil || !ok { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("native Codex runtime is not loaded") } snapshot := s.nativeCodexRuntime.current() if snapshot == nil { - return nil, errCodexRoutingUnavailable + return nil, codexRoutingUnavailable("native Codex runtime snapshot is not loaded") } ctx = WithNativeCodexExecution(ctx, installation) @@ -118,8 +135,11 @@ func (s *OpenAIGatewayService) ValidateCodexRouting(ctx context.Context, id int6 defer cancel() lease := extensionv1.LeaseRequest{Namespace: "routing-account", Key: fmt.Sprint(id), Owner: uuid.NewString(), TTLSeconds: 75} locked, err := store.AcquireExtensionLease(ctx, NativeCodexPluginKey, lease) - if err != nil || !locked.Acquired { - return nil, ErrCodexTicketBusy + if err != nil { + return nil, fmt.Errorf("%w: acquire account routing lease: %v", ErrCodexTicketBusy, err) + } + if !locked.Acquired { + return nil, fmt.Errorf("%w: another routing operation holds account %d", ErrCodexTicketBusy, id) } lease.Generation = locked.Generation defer func() { @@ -132,8 +152,14 @@ func (s *OpenAIGatewayService) ValidateCodexRouting(ctx context.Context, id int6 raw, _ := json.Marshal(query) response, err := snapshot.host.Call(ctx, extensionv1.HostInvocation{Operation: extensionv1.HostCodexRoutingProbe, Payload: raw}) var value extensionv1.CodexRoutingProbeResult - if err != nil || response.Code != "" || json.Unmarshal(response.Payload, &value) != nil { - return value, errCodexRoutingUnavailable + switch { + case err != nil: + return value, fmt.Errorf("%s probe: %w", query.Stage, err) + case response.Code != "": + return value, codexRoutingUnavailable("%s probe returned code %s: %s", query.Stage, response.Code, response.Message) + } + if err := json.Unmarshal(response.Payload, &value); err != nil { + return value, codexRoutingUnavailable("decode %s probe result: %v", query.Stage, err) } result.Observations = append(result.Observations, value.Observation) return value, nil @@ -146,8 +172,14 @@ func (s *OpenAIGatewayService) ValidateCodexRouting(ctx context.Context, id int6 acquired, err = call(query) } else { previous, readErr := store.ReadExtensionState(ctx, NativeCodexPluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: storedKey}) - if readErr != nil || !previous.Found || json.Unmarshal(previous.Value, &acquired) != nil { - return nil, errCodexRoutingUnavailable + switch { + case readErr != nil: + return nil, codexRoutingUnavailable("read the HTTP validation result of %s: %v", request.ValidationID, readErr) + case !previous.Found: + return nil, codexRoutingUnavailable("WebSocket validation needs a successful HTTP validation of model %s with the same validation_id first", request.Model) + } + if err := json.Unmarshal(previous.Value, &acquired); err != nil { + return nil, codexRoutingUnavailable("decode the HTTP validation result of %s: %v", request.ValidationID, err) } } if err == nil && acquired.Valid && acquired.Bundle != nil { @@ -156,12 +188,14 @@ func (s *OpenAIGatewayService) ValidateCodexRouting(ctx context.Context, id int6 verified, err = call(query) result.Success = err == nil && verified.Valid if result.Success && request.Transport == "http" { - raw, _ := json.Marshal(verified) + stored := verified + stored.Observation = codexRoutingPrivateObservation(stored.Observation) + raw, _ := json.Marshal(stored) _, _ = store.CompareSwapExtensionState(ctx, NativeCodexPluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: storedKey, Value: raw}) } } if err != nil { - result.Observations = append(result.Observations, extensionv1.CodexRoutingObservation{Stage: query.Stage, Code: "routing_validation_unavailable", RequestedModel: request.Model, Transport: request.Transport, ObservedAt: time.Now().UTC()}) + result.Observations = append(result.Observations, extensionv1.CodexRoutingObservation{Stage: query.Stage, Code: "routing_validation_unavailable", RequestedModel: request.Model, Transport: request.Transport, ObservedAt: time.Now().UTC(), Error: err.Error()}) } budgetRecord, readErr := store.ReadExtensionState(context.WithoutCancel(ctx), NativeCodexPluginKey, extensionv1.StateRequest{Namespace: codexRoutingPrivateNamespace, Key: "validation." + request.ValidationID}) if readErr == nil { diff --git a/backend/internal/service/codex_routing_ws_probe.go b/backend/internal/service/codex_routing_ws_probe.go index e3d57484cd4b..2fe076cbab5a 100644 --- a/backend/internal/service/codex_routing_ws_probe.go +++ b/backend/internal/service/codex_routing_ws_probe.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "time" @@ -18,15 +19,15 @@ import ( // socket or client session. Normal qualified WS ingress uses the HTTP bridge. func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, query extensionv1.CodexRoutingQuery, cookies string, deadline time.Time, scope extensionv1.CodexRoutingScope) (*http.Response, extensionv1.CodexRoutingScope, error) { if _, validation := codexValidationFromContext(ctx); !validation || query.Stage != "verify" || cookies == "" || !time.Now().Before(deadline) { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("a WebSocket probe is only a validation verify stage with live cookies (validation=%t stage=%q cookies=%t deadline=%s)", validation, query.Stage, cookies != "", deadline.UTC().Format(time.RFC3339)) } account, err := s.accountRepo.GetByID(ctx, query.AccountID) if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("read account %d: %v", query.AccountID, err) } token, _, err := s.GetAccessToken(withCodexTicketCredentials(ctx), account) if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, fmt.Errorf("%w: %v", errCodexRoutingToken, err) } request, err := s.buildCodexRoutingProbe(ctx, account, query.Model, token, cookies, query.ReasoningEffort) if err != nil { @@ -35,22 +36,22 @@ func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, q raw, err := io.ReadAll(io.LimitReader(request.Body, codexRoutingProbeReadLimit)) _ = request.Body.Close() if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("read probe body: %v", err) } if request.Header.Get("Content-Encoding") == "zstd" { decoder, decoderErr := zstd.NewReader(nil) if decoderErr != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("zstd decoder: %v", decoderErr) } raw, err = decoder.DecodeAll(raw, nil) decoder.Close() if err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("decode zstd probe body: %v", err) } } var payload map[string]any - if json.Unmarshal(raw, &payload) != nil { - return nil, scope, errCodexRoutingUnavailable + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, scope, codexRoutingUnavailable("decode probe body: %v", err) } delete(payload, "stream") payload["type"] = "response.create" @@ -61,8 +62,11 @@ func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, q headers.Del("Content-Length") headers.Set("OpenAI-Beta", openAIWSBetaV2Value) connection, status, handshake, err := s.getOpenAIWSPassthroughDialer().Dial(ctx, "wss://chatgpt.com/backend-api/codex/responses", headers, resolveAccountProxyURL(account)) - if err != nil || connection == nil { - return nil, scope, errCodexRoutingUnavailable + if err != nil { + return nil, scope, fmt.Errorf("dial Codex WebSocket (handshake status %d): %w", status, err) + } + if connection == nil { + return nil, scope, codexRoutingUnavailable("WebSocket dial returned no connection (handshake status %d)", status) } defer func() { _ = connection.Close() }() // The validation read cap covers every frame, not only the gaps between them. @@ -71,7 +75,7 @@ func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, q } scope.ConnectionLeaseID, scope.RouteEvidence, scope.Transport = "ws-validation-"+uuid.NewString(), "closed_validation_connection", "ws" if err := connection.WriteJSON(ctx, payload); err != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, fmt.Errorf("write response.create frame: %w", err) } completion := codexRoutingCompletion{RequestedModel: query.Model} completion.headers(handshake) @@ -80,11 +84,11 @@ func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, q for readBytes < codexRoutingProbeReadLimit { frame, readErr := connection.ReadMessage(ctx) if readErr != nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, fmt.Errorf("read WebSocket frame after %d bytes: %w", readBytes, readErr) } readBytes += len(frame) if readBytes > codexRoutingProbeReadLimit { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("WebSocket validation read %d bytes, over the %d-byte limit", readBytes, codexRoutingProbeReadLimit) } completion.json(frame) if completion.Failed || completion.Completed { @@ -93,16 +97,17 @@ func (s *OpenAIGatewayService) executeCodexRoutingWSProbe(ctx context.Context, q } } if terminal == nil { - return nil, scope, errCodexRoutingUnavailable + return nil, scope, codexRoutingUnavailable("no terminal WebSocket event within %d bytes", codexRoutingProbeReadLimit) } if status == 0 { status = http.StatusSwitchingProtocols } + // Only the upstream handshake headers are reported; the terminal frame is + // JSON, which the completion reader accepts without a Content-Type. response := &http.Response{StatusCode: status, Proto: "HTTP/1.1", Header: handshake.Clone(), Body: io.NopCloser(bytes.NewReader(terminal))} if response.Header == nil { response.Header = make(http.Header) } - response.Header.Set("Content-Type", "application/json") q := &extensionv1.CodexRoutingQualification{Scope: scope, Model: query.Model} request.Header = headers request = request.WithContext(context.WithValue(request.Context(), codexRoutingIngressKey{}, "ws")) diff --git a/backend/internal/service/codex_ticket_lifecycle.go b/backend/internal/service/codex_ticket_lifecycle.go index a109cbbfe93f..c47f76528d11 100644 --- a/backend/internal/service/codex_ticket_lifecycle.go +++ b/backend/internal/service/codex_ticket_lifecycle.go @@ -7,6 +7,8 @@ import ( "net/http" "strings" "time" + + extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" ) const CodexTicketRuntimeExtraPrefix = "codex_ticket_runtime:" @@ -29,16 +31,35 @@ type CodexTicketLifecycle struct { LastResult *CodexTicketResult `json:"last_result,omitempty"` } +// CodexTicketResult keeps the complete runtime outcome: Error is the original +// error text, Observation the recorded routing observation and Facts the +// display facts rendered by the account operation views. type CodexTicketResult struct { - Stage string `json:"stage,omitempty"` - Code string `json:"code"` - Success bool `json:"success"` - HTTPStatus int `json:"http_status,omitempty"` - ObservedLength int `json:"observed_length,omitempty"` - Message string `json:"message,omitempty"` - DurationMS int64 `json:"duration_ms,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` + Stage string `json:"stage,omitempty"` + Code string `json:"code"` + Success bool `json:"success"` + HTTPStatus int `json:"http_status,omitempty"` + ObservedLength int `json:"observed_length,omitempty"` + ResponseModel string `json:"response_model,omitempty"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Observation *extensionv1.CodexRoutingObservation `json:"observation,omitempty"` + Facts []extensionv1.DisplayFact `json:"facts,omitempty"` +} + +// Detail returns the original error text of the result, or "" when the +// outcome carries none (the catalog message then describes the code). +func (r CodexTicketResult) Detail() string { + if r.Error != "" { + return r.Error + } + if r.Observation != nil { + return r.Observation.Summary() + } + return "" } type CodexTicketRecord struct { @@ -79,60 +100,84 @@ func CodexTicketAccountEligible(a *Account) bool { return isOpenAICodexTicketAccount(a) } +var codexTicketMessages = map[string]string{ + "routing_verified": "路由验证记录：业务出口响应完整、模型声明一致；本项未验证质量", + "routing_candidate": "已取得候选 Cookie，尚未通过业务出口验证", + "routing_model_mismatch": "业务出口的上游模型声明不匹配，未发布路由资格", + "routing_incomplete": "响应未完整结束或缺少模型声明", + "routing_capacity": "上游容量不足或流内限流，未完成路由验证", + "routing_policy": "上游策略检查阻止请求，未完成路由验证", + "routing_cancelled": "客户端取消，本次响应未完成验证", + "routing_legacy_retired": "旧 STATE 材料已丢弃，尚未获得路由资格", + "routing_cookie_missing": "未取得可用路由 Cookie", + "routing_cookie_expired": "路由 Cookie 已到期", + "routing_cookie_deleted": "上游已撤销路由 Cookie", + "routing_transport": "采集或业务出口连接失败", + "routing_stale": "账号、设备、出口或 Cookie 代次已变化", + "routing_connection_expired": "已验证连接不可复用，需要重新验证", + "routing_upstream": "上游拒绝路由验证请求", + "routing_unavailable": "路由采集未能开始（账号出口范围不可用）", + "routing_persist": "路由 Cookie 记录保存失败", + "routing_connection_unknown": "无法绑定经过验证的实际连接", + "routing_observation_unavailable": "无法登记本次路由观测", + "routing_budget_spent": "本次验证的固定调用预算已使用", + "routing_validation_unavailable": "验证请求未能完成", + "ticket_plugin_unavailable": "Codex 路由插件未启用或暂不可用", + "ticket_result_invalid": "Codex 路由插件返回的结果无法解析", + "ticket_runtime_error": "Codex 路由运行时执行失败", + "ticket_disabled": "Codex 路由采集与验证未开启", + "ticket_proxy_missing": "未配置路由采集代理", + "ticket_ineligible": "仅 OpenAI OAuth/Setup Token 非影子账号支持路由采集", + "ticket_model_invalid": "模型不在路由配置范围内", + "ticket_operation_required": "手动采集缺少操作标识", + "ticket_busy": "该账号和模型已有路由采集任务", + "ticket_interrupted": "上次请求已开始，结果未确认；不会自动重复发送", + "ticket_attempt_in_progress": "该账号和模型的上一次尝试仍在进行", + "ticket_not_due": "未到续期时间", + "ticket_missing": "该模型尚无有效路由资格", + "ticket_skipped": "已有路由验证记录，本次跳过；本次未验证质量", + "ticket_ready": "历史 STATE 采集记录，不代表当前路由资格", + "ticket_token": "无法取得账号访问令牌", + "ticket_timeout": "请求超时", + "ticket_transport": "路由采集代理传输失败", + "ticket_upstream": "上游拒绝路由采集请求", + "ticket_length": "历史 STATE 长度检查失败（该规则已停用）", + "ticket_prefix": "历史 STATE 格式检查失败（该规则已停用）", + "ticket_missing_header": "历史采集响应未包含 STATE 头", + "ticket_persist": "路由验证记录持久化失败，未加入自动续期", + "ticket_stale": "账号身份、任务或续期状态已经变化", + "ticket_canceled": "任务已取消", + "ticket_stopped": "自动续期已停止", + "ticket_proxy_auth": "代理鉴权失败", + "ticket_proxy_protocol": "代理协议握手失败，请核对HTTP或SOCKS协议", + "ticket_proxy_dns": "代理主机DNS解析失败", + "ticket_proxy_connect": "代理连接被拒绝或CONNECT失败", + "ticket_proxy_eof": "连接被对端提前关闭（EOF）", + "ticket_proxy_reset": "连接被对端重置", + "ticket_proxy_tls": "代理TLS证书验证失败，请测试代理连接", + "proxy_reachable": "代理连接和证书验证正常", + "target_http_status": "代理已连通，目标返回非成功状态", + "proxy_connection_failed": "代理连接或证书验证失败", + "pinned_connection_failed": "固定证书验证失败", + "invalid_proxy": "代理地址或协议不正确", + "proxy_trust_unavailable": "无法读取已固定的代理证书", + "proxy_trust_changed": "固定代理证书在测试期间被修改", +} + +// CodexTicketFailure describes a result code. An unknown code keeps its own +// name instead of being folded into the transport category. func CodexTicketFailure(code string) CodexTicketResult { - messages := map[string]string{ - "routing_candidate": "已取得候选 Cookie，尚未通过业务出口验证", - "routing_model_mismatch": "业务出口的上游模型声明不匹配，未发布路由资格", - "routing_incomplete": "响应未完整结束或缺少模型声明", - "routing_capacity": "上游容量不足或流内限流，未完成路由验证", - "routing_policy": "上游策略检查阻止请求，未完成路由验证", - "routing_cancelled": "客户端取消，本次响应未完成验证", - "routing_legacy_retired": "旧 STATE 材料已丢弃，尚未获得路由资格", - "routing_cookie_missing": "未取得可用路由 Cookie", - "routing_cookie_expired": "路由 Cookie 已到期", - "routing_cookie_deleted": "上游已撤销路由 Cookie", - "routing_transport": "采集或业务出口连接失败", - "routing_stale": "账号、设备、出口或 Cookie 代次已变化", - "routing_connection_expired": "已验证连接不可复用，需要重新验证", - "routing_upstream": "上游拒绝路由验证请求", - "ticket_plugin_unavailable": "Codex 路由插件未启用或暂不可用", - "ticket_disabled": "Codex 路由采集与验证未开启", - "ticket_proxy_missing": "未配置路由采集代理", - "ticket_ineligible": "仅 OpenAI OAuth/Setup Token 非影子账号支持路由采集", - "ticket_model_invalid": "模型不在路由配置范围内", - "ticket_busy": "该账号和模型已有路由采集任务", - "ticket_interrupted": "上次请求已开始，结果未确认；不会自动重复发送", - "ticket_token": "无法取得账号访问令牌", - "ticket_timeout": "请求超时", - "ticket_transport": "路由采集代理传输失败", - "ticket_upstream": "上游拒绝路由采集请求", - "ticket_length": "历史 STATE 长度检查失败（该规则已停用）", - "ticket_prefix": "历史 STATE 格式检查失败（该规则已停用）", - "ticket_missing_header": "历史采集响应未包含 STATE 头", - "ticket_persist": "路由验证记录持久化失败，未加入自动续期", - "ticket_stale": "账号身份、任务或续期状态已经变化", - "ticket_canceled": "任务已取消", - "ticket_stopped": "自动续期已停止", - "ticket_proxy_auth": "代理鉴权失败", - "ticket_proxy_protocol": "代理协议握手失败，请核对HTTP或SOCKS协议", - "ticket_proxy_dns": "代理主机DNS解析失败", - "ticket_proxy_connect": "代理连接被拒绝或CONNECT失败", - "ticket_proxy_eof": "连接被对端提前关闭（EOF）", - "ticket_proxy_reset": "连接被对端重置", - "ticket_proxy_tls": "代理TLS证书验证失败，请测试代理连接", - } - message, ok := messages[code] + message, ok := codexTicketMessages[code] if !ok { - code = "ticket_transport" - message = messages[code] + message = "未归类的结果代码：" + code } stage := "request" switch code { - case "ticket_disabled", "ticket_proxy_missing", "ticket_ineligible", "ticket_model_invalid": + case "ticket_disabled", "ticket_proxy_missing", "ticket_ineligible", "ticket_model_invalid", "ticket_operation_required", "invalid_proxy", "routing_unavailable": stage = "configuration" - case "ticket_busy", "ticket_interrupted", "ticket_stale", "ticket_stopped", "ticket_canceled": + case "ticket_busy", "ticket_interrupted", "ticket_stale", "ticket_stopped", "ticket_canceled", "ticket_attempt_in_progress", "ticket_not_due": stage = "lifecycle" - case "routing_cancelled", "routing_legacy_retired": + case "routing_cancelled", "routing_legacy_retired", "routing_budget_spent": stage = "lifecycle" case "routing_capacity": stage = "upstream_stream" @@ -148,14 +193,16 @@ func CodexTicketFailure(code string) CodexTicketResult { stage = "proxy_dns" case "ticket_proxy_connect": stage = "proxy_connect" - case "ticket_proxy_tls": + case "ticket_proxy_tls", "proxy_trust_unavailable", "proxy_trust_changed", "pinned_connection_failed": stage = "tls" case "ticket_length", "ticket_prefix", "ticket_missing_header": stage = "ticket_validation" case "ticket_upstream": stage = "upstream_http" - case "ticket_persist": + case "ticket_persist", "routing_persist", "routing_observation_unavailable": stage = "persistence" + case "routing_upstream": + stage = "upstream_http" } return CodexTicketResult{Code: code, Message: message, Stage: stage} } diff --git a/backend/internal/service/codex_ticket_plugin.go b/backend/internal/service/codex_ticket_plugin.go index 42e6e5961ccb..97193f655ab8 100644 --- a/backend/internal/service/codex_ticket_plugin.go +++ b/backend/internal/service/codex_ticket_plugin.go @@ -45,7 +45,13 @@ func (s *OpenAIGatewayService) applyOpenAICodexTicket(ctx context.Context, accou return nil } if err := s.nativeCodexRuntime.ApplyRequestHeaders(ctx, account, model, headers); err != nil { - return fmt.Errorf("%w: extension request prerequisites", ErrOpenAICodexTicketUnavailable) + var result *NativeCodexResultError + if errors.As(err, &result) { + // The runtime's result code and reason stay inspectable (errors.As). + return fmt.Errorf("%w: %w", ErrOpenAICodexTicketUnavailable, result) + } + // Other causes are kept as text; only the sentinel stays in the chain. + return fmt.Errorf("%w: extension request prerequisites: %v", ErrOpenAICodexTicketUnavailable, err) } return nil } @@ -57,13 +63,37 @@ func (s *OpenAIGatewayService) openAICodexTicketBlocksAccount(account *Account, return s != nil && s.nativeCodexRuntime != nil && !s.nativeCodexRuntime.SchedulingDecision(account, model, time.Now()).Allowed } +func codexTicketFailureWithError(code, detail string) CodexTicketResult { + result := CodexTicketFailure(code) + result.Error = detail + return result +} + +// codexTicketInvokeFailure keeps the runtime error text and only chooses the +// catalog code that describes where it happened. Only the job's own context +// decides cancellation: a context error while the job is still running comes +// from the runtime stopping or reloading underneath it. +func codexTicketInvokeFailure(ctx context.Context, err error) CodexTicketResult { + switch { + case ctx.Err() != nil: + return codexTicketFailureWithError("ticket_canceled", err.Error()) + case errors.Is(err, context.Canceled): + return codexTicketFailureWithError("ticket_plugin_unavailable", "the Codex runtime stopped or reloaded during the operation: "+err.Error()) + case errors.Is(err, context.DeadlineExceeded): + return codexTicketFailureWithError("ticket_timeout", err.Error()) + case errors.Is(err, ErrNativeCodexRuntimeUnavailable), errors.Is(err, ErrNativeCodexPolicyDisabled), errors.Is(err, ErrNativeCodexRuntimeChanged): + return codexTicketFailureWithError("ticket_plugin_unavailable", err.Error()) + } + return codexTicketFailureWithError("ticket_runtime_error", err.Error()) +} + func (s *OpenAIGatewayService) HarvestCodexTicket(ctx context.Context, id int64, model, operation string, jobID int64, force bool) CodexTicketResult { if s == nil || s.nativeCodexRuntime == nil { - return CodexTicketFailure("ticket_plugin_unavailable") + return codexTicketFailureWithError("ticket_plugin_unavailable", "native Codex runtime is not configured") } installation := s.nativeCodexRuntime.metadata() if installation == nil { - return CodexTicketFailure("ticket_plugin_unavailable") + return codexTicketFailureWithError("ticket_plugin_unavailable", "native Codex runtime is not loaded") } if jobID > 0 { operation = fmt.Sprintf("job-%d", jobID) @@ -71,11 +101,25 @@ func (s *OpenAIGatewayService) HarvestCodexTicket(ctx context.Context, id int64, payload, _ := json.Marshal(map[string]any{"account_id": id, "model": model, "operation_id": operation, "force": force}) response, err := s.nativeCodexRuntime.Invoke(ctx, PlatformOpenAI, "", extensionv1.Invocation{Capability: extensionv1.CapabilityAdmin, Operation: "harvest", AccountID: id, Payload: payload}) if err != nil { - return CodexTicketFailure("ticket_plugin_unavailable") + return codexTicketInvokeFailure(ctx, err) } var result CodexTicketResult - if json.Unmarshal(response.Payload, &result) != nil { - return CodexTicketFailure("ticket_transport") + if err := json.Unmarshal(response.Payload, &result); err != nil { + return codexTicketFailureWithError("ticket_result_invalid", err.Error()) + } + if result.Stage == "" { + result.Stage = CodexTicketFailure(result.Code).Stage + } + if observation := result.Observation; observation != nil { + if result.ResponseModel == "" { + result.ResponseModel = observation.ResponseModel + } + if result.DurationMS == 0 { + result.DurationMS = observation.DurationMS + } + if result.Error == "" && !result.Success { + result.Error = observation.Summary() + } } return result } @@ -92,11 +136,14 @@ func (s *OpenAIGatewayService) StopCodexTicketRenewal(ctx context.Context, id in raw, _ := json.Marshal(map[string]any{"account_id": id, "model": model}) response, err := s.nativeCodexRuntime.Invoke(ctx, PlatformOpenAI, "", extensionv1.Invocation{Capability: extensionv1.CapabilityAdmin, Operation: "stop", AccountID: id, Payload: raw}) if err != nil { - return err + return fmt.Errorf("stop renewal of %s: %w", model, err) } var result CodexTicketResult - if json.Unmarshal(response.Payload, &result) != nil || !result.Success { - return errors.New("ticket stop failed") + if err := json.Unmarshal(response.Payload, &result); err != nil { + return fmt.Errorf("stop renewal of %s: decode runtime result %s: %w", model, string(response.Payload), err) + } + if !result.Success { + return fmt.Errorf("stop renewal of %s: runtime returned %s: %s", model, result.Code, string(response.Payload)) } } return nil @@ -121,26 +168,36 @@ func (s *OpenAIGatewayService) TestCodexTicketProxyWithProtocol(ctx context.Cont } var result CodexTicketProxyTestResult if err := json.Unmarshal(response.Payload, &result); err != nil { - return nil, err + return nil, fmt.Errorf("decode proxy test result: %w", err) } if result.NetworkReachable && (result.Code == "proxy_reachable" || result.Code == "target_http_status") { result.Message = fmt.Sprintf("连接和证书验证正常，目标返回HTTP %d；未发送账号凭据", result.HTTPStatus) } else { - result.Message = "代理连接或证书验证失败" + // The code names the failure category; FailureDetail and the stages + // keep the original error text. + result.Message = CodexTicketFailure(result.Code).Message } return &result, nil } +// OpenAICodexTicketStatuses projects the recorded routing history. With the +// switch off, models with a recorded observation stay visible (nothing is +// ready or blocked then); models without any history are omitted. func OpenAICodexTicketStatuses(account *Account, cfg config.OpenAICodexTicketConfig, now time.Time) []OpenAICodexTicketStatus { - if !cfg.Enabled || !isOpenAICodexTicketAccount(account) { + if !isOpenAICodexTicketAccount(account) { return nil } projection := nativeCodexAccountProjection(account) + owned := projection.Identity == CodexTicketAccountIdentity(account) out := make([]OpenAICodexTicketStatus, 0, len(cfg.Models)) for _, model := range cfg.Models { entry := OpenAICodexTicketStatus{Model: model, RenewalState: "idle"} - if projection.Identity == CodexTicketAccountIdentity(account) { - if observation, ok := projection.Observations[model]; ok { + observation, observed := projection.Observations[model] + if !cfg.Enabled && (!owned || !observed) { + continue + } + if owned { + if observed { entry.RenewalState = observation.State entry.NextAttemptAt = observation.NextAt entry.LastAttemptAt = observation.CheckedAt @@ -148,19 +205,22 @@ func OpenAICodexTicketStatuses(account *Account, cfg config.OpenAICodexTicketCon entry.Length = observation.Count if observation.Code != "" { result := CodexTicketFailure(observation.Code) - if observation.Code == "routing_verified" || observation.Code == "ticket_skipped" { - result = CodexTicketResult{Code: observation.Code, Success: true, Message: "路由验证记录：业务出口响应完整、模型声明一致；本项未验证质量"} - } + result.Success = observation.Code == "routing_verified" || observation.Code == "ticket_skipped" + result.HTTPStatus = observation.HTTPStatus + result.ObservedLength = observation.Count + result.ResponseModel = observation.ResponseModel + // Message stays the short catalog sentence; Error is the raw text. + result.Error = observation.Message entry.LastResult = &result } } - if grant, ok := projection.Scheduling[model]; ok && grant.Effect == "allow" && grant.Reason == "routing_verified" && grant.Until != nil && now.Before(*grant.Until) { + if grant, ok := projection.Scheduling[model]; cfg.Enabled && ok && grant.Effect == "allow" && grant.Reason == "routing_verified" && grant.Until != nil && now.Before(*grant.Until) { entry.Ready = true entry.ExpiresAt = grant.Until entry.RemainingSeconds = int64(grant.Until.Sub(now).Seconds()) } } - entry.Blocked = cfg.FailClosed && !entry.Ready + entry.Blocked = cfg.Enabled && cfg.FailClosed && !entry.Ready out = append(out, entry) } return out diff --git a/backend/internal/service/codex_ticket_proxy.go b/backend/internal/service/codex_ticket_proxy.go index 5f07084f2811..efa5a75bd267 100644 --- a/backend/internal/service/codex_ticket_proxy.go +++ b/backend/internal/service/codex_ticket_proxy.go @@ -1,6 +1,8 @@ package service import ( + "time" + proxytransport "github.com/Wei-Shaw/sub2api/internal/proxytransport" ) @@ -16,18 +18,36 @@ type CodexTicketProxyStage struct { Message string `json:"message,omitempty"` } +// CodexTicketProxyCertificate is one certificate presented during the test +// handshake (HTTPS proxy, target, or TLS-inspecting proxy chain). +type CodexTicketProxyCertificate struct { + ServerName string `json:"server_name,omitempty"` + Subject string `json:"subject"` + Issuer string `json:"issuer"` + SerialNumber string `json:"serial_number,omitempty"` + NotBefore time.Time `json:"not_before"` + NotAfter time.Time `json:"not_after"` + DNSNames []string `json:"dns_names,omitempty"` + SHA256 string `json:"sha256"` + SPKISHA256 string `json:"spki_sha256"` +} + +// FailureDetail is the original error of the failed request and every stage +// keeps its own original error text; certificate facts come from the handshake. type CodexTicketProxyTestResult struct { - FailureDetail string `json:"failure_detail,omitempty"` - Success bool `json:"success"` - NetworkReachable bool `json:"network_reachable"` - Protocol string `json:"protocol"` - HTTPStatus int `json:"http_status,omitempty"` - Code string `json:"code"` - Message string `json:"message"` - Stages []CodexTicketProxyStage `json:"stages"` - CertificateTrust string `json:"certificate_trust,omitempty"` - CertificateFingerprint string `json:"certificate_fingerprint,omitempty"` - ProtocolSuggestion string `json:"protocol_suggestion,omitempty"` + FailureDetail string `json:"failure_detail,omitempty"` + Success bool `json:"success"` + NetworkReachable bool `json:"network_reachable"` + Protocol string `json:"protocol"` + HTTPStatus int `json:"http_status,omitempty"` + Code string `json:"code"` + Message string `json:"message"` + Stages []CodexTicketProxyStage `json:"stages"` + CertificateTrust string `json:"certificate_trust,omitempty"` + CertificateFingerprint string `json:"certificate_fingerprint,omitempty"` + Certificates []CodexTicketProxyCertificate `json:"certificates,omitempty"` + CertificateError string `json:"certificate_error,omitempty"` + ProtocolSuggestion string `json:"protocol_suggestion,omitempty"` } // NormalizeCodexTicketProxy is shared by settings validation and the preview diff --git a/backend/internal/service/image_studio_job.go b/backend/internal/service/image_studio_job.go index 153ac5b7302e..dc18f5c45536 100644 --- a/backend/internal/service/image_studio_job.go +++ b/backend/internal/service/image_studio_job.go @@ -190,6 +190,9 @@ type ImageStudioExecutionRequest struct { ReferenceContentType string Mask []byte MaskContentType string + // Origin is the client that submitted or retried the job, when this + // process still knows it. + Origin ImageStudioRequestOrigin } type ImageStudioExecutionResult struct { diff --git a/backend/internal/service/image_studio_job_service.go b/backend/internal/service/image_studio_job_service.go index 2f553a0efc65..2002194442fa 100644 --- a/backend/internal/service/image_studio_job_service.go +++ b/backend/internal/service/image_studio_job_service.go @@ -42,6 +42,7 @@ type ImageStudioService struct { accounts AccountRepository store ImageStudioFileStorage now func() time.Time + origins imageStudioOrigins } func NewImageStudioService( @@ -158,6 +159,7 @@ func (s *ImageStudioService) Create( removeSaved() return nil, err } + s.origins.remember(job.ID, imageStudioRequestOriginFromContext(ctx), now) return job, nil } @@ -197,7 +199,9 @@ func (s *ImageStudioService) Retry(ctx context.Context, userID, jobID int64) (*I return nil, newImageStudioError(503, "studio_unavailable", "Image Studio is unavailable") } } - return s.repo.Retry(ctx, userID, jobID, s.now()) + now := s.now() + s.origins.remember(jobID, imageStudioRequestOriginFromContext(ctx), now) + return s.repo.Retry(ctx, userID, jobID, now) } func (s *ImageStudioService) OpenArtifact(ctx context.Context, userID, jobID, artifactID int64) (*ImageStudioArtifactDownload, error) { diff --git a/backend/internal/service/image_studio_job_test.go b/backend/internal/service/image_studio_job_test.go index 73e6be4ee30e..e7f2307effd8 100644 --- a/backend/internal/service/image_studio_job_test.go +++ b/backend/internal/service/image_studio_job_test.go @@ -3,6 +3,8 @@ package service import ( + "context" + "fmt" "testing" "github.com/stretchr/testify/require" @@ -109,3 +111,12 @@ func TestResolveImageStudioTerminalStatusPreservesPartialAndCanceledResults(t *t }) } } + +// A gateway failure keeps its status and body for administrators (server log +// and Ops); the job item shows the generic user-facing message. +func TestImageStudioGatewayErrorShowsGenericUserMessage(t *testing.T) { + err := fmt.Errorf("generate: %w", &ImageStudioGatewayError{StatusCode: 502, Body: `{"error":{"message":"upstream account 7 was banned"}}`}) + code, message := imageStudioSafeExecutionError(context.Background(), err) + require.Equal(t, "generation_failed", code) + require.Equal(t, "Image generation failed", message) +} diff --git a/backend/internal/service/image_studio_runtime.go b/backend/internal/service/image_studio_runtime.go index 97acac073871..49c3061595fc 100644 --- a/backend/internal/service/image_studio_runtime.go +++ b/backend/internal/service/image_studio_runtime.go @@ -2,12 +2,136 @@ package service import ( "context" + "crypto/rand" + "encoding/hex" "errors" "fmt" + "log/slog" + "strings" "sync" "time" ) +// ImageStudioRequestOrigin is the client address and User-Agent of the request +// that submitted or retried an Image Studio job. +type ImageStudioRequestOrigin struct { + ClientIP string + UserAgent string +} + +type imageStudioRequestOriginKey struct{} + +// WithImageStudioRequestOrigin records the submitting client for Create/Retry. +func WithImageStudioRequestOrigin(ctx context.Context, origin ImageStudioRequestOrigin) context.Context { + return context.WithValue(ctx, imageStudioRequestOriginKey{}, origin) +} + +func imageStudioRequestOriginFromContext(ctx context.Context) ImageStudioRequestOrigin { + if ctx == nil { + return ImageStudioRequestOrigin{} + } + origin, _ := ctx.Value(imageStudioRequestOriginKey{}).(ImageStudioRequestOrigin) + return origin +} + +// ImageStudioRequestOriginFromContext returns the Image Studio submitter that +// an in-process gateway request carries for its usage and Ops records. The +// gateway request itself carries no client headers, because the images +// handler forwards request headers to API-key upstreams. +func ImageStudioRequestOriginFromContext(ctx context.Context) (ImageStudioRequestOrigin, bool) { + origin := imageStudioRequestOriginFromContext(ctx) + return origin, strings.TrimSpace(origin.ClientIP) != "" || strings.TrimSpace(origin.UserAgent) != "" +} + +// ImageStudioClientRequestIDPrefix starts the client request ID of every Image +// Studio gateway request. It marks the Ops rows that stay admin-only. +const ImageStudioClientRequestIDPrefix = "image-studio-" + +// imageStudioClientRequestIDMaxLen keeps "client:"+ID, the usage billing +// dedupe key, within the 64-character request ID columns. +const imageStudioClientRequestIDMaxLen = 57 + +// NewImageStudioClientRequestID returns the client request ID of one gateway +// attempt of a job item. It ties Ops and usage rows to the item and is the +// usage billing dedupe key, so every attempt gets its own random suffix: a +// retried or re-run item is billed again instead of being deduplicated. +func NewImageStudioClientRequestID(jobID, itemID int64) string { + var nonce [6]byte + _, _ = rand.Read(nonce[:]) + id := fmt.Sprintf("%sjob-%d-item-%d-%s", ImageStudioClientRequestIDPrefix, jobID, itemID, hex.EncodeToString(nonce[:])) + if len(id) > imageStudioClientRequestIDMaxLen { + var wide [16]byte + _, _ = rand.Read(wide[:]) + id = ImageStudioClientRequestIDPrefix + hex.EncodeToString(wide[:]) + } + return id +} + +// imageStudioLogLimit bounds the error text an Image Studio failure writes to +// the server log; a response body can be an entire image. +const imageStudioLogLimit = 4 << 10 + +// ImageStudioLogText returns at most the first 4 KiB of a text for the server +// log, noting how long the full text was. +func ImageStudioLogText(text string) string { + if len(text) <= imageStudioLogLimit { + return strings.ToValidUTF8(text, "\uFFFD") + } + return strings.ToValidUTF8(text[:imageStudioLogLimit], "\uFFFD") + fmt.Sprintf("...<%d bytes in total>", len(text)) +} + +// imageStudioOrigins keeps the submitting client of recent jobs in memory, so +// the gateway request of a job item carries the real client IP and User-Agent +// into usage and Ops records. Without a database column it does not survive a +// restart, and a job processed by another instance falls back to no origin. +type imageStudioOrigins struct { + mu sync.Mutex + entries map[int64]imageStudioOriginEntry +} + +type imageStudioOriginEntry struct { + origin ImageStudioRequestOrigin + at time.Time +} + +func (o *imageStudioOrigins) remember(jobID int64, origin ImageStudioRequestOrigin, now time.Time) { + if o == nil || jobID <= 0 || (origin.ClientIP == "" && origin.UserAgent == "") { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if o.entries == nil { + o.entries = make(map[int64]imageStudioOriginEntry) + } + for id, entry := range o.entries { + if now.Sub(entry.at) > ImageStudioFileRetention { + delete(o.entries, id) + } + } + o.entries[jobID] = imageStudioOriginEntry{origin: origin, at: now} +} + +func (o *imageStudioOrigins) lookup(jobID int64) ImageStudioRequestOrigin { + if o == nil { + return ImageStudioRequestOrigin{} + } + o.mu.Lock() + defer o.mu.Unlock() + return o.entries[jobID].origin +} + +// ImageStudioGatewayError keeps the HTTP status and body the image gateway +// returned for a failed Image Studio request. The server log and the Ops error +// record keep them; the job item shows its user-facing message. +type ImageStudioGatewayError struct { + StatusCode int + Body string +} + +func (e *ImageStudioGatewayError) Error() string { + return fmt.Sprintf("image gateway returned %d: %s", e.StatusCode, e.Body) +} + type ImageStudioRuntimeOptions struct { Workers int PollInterval time.Duration @@ -164,7 +288,7 @@ func (r *ImageStudioRuntime) processClaim(ctx context.Context, claim *ImageStudi r.failClaim(ctx, job.ID, claim.Item.ID, err) return } - request := ImageStudioExecutionRequest{Job: job, Item: claim.Item, APIKey: key} + request := ImageStudioExecutionRequest{Job: job, Item: claim.Item, APIKey: key, Origin: r.studio.origins.lookup(job.ID)} for _, artifact := range claim.Inputs { data, readErr := r.store.Read(artifact.StorageKey) if readErr != nil { @@ -204,6 +328,11 @@ func (r *ImageStudioRuntime) processClaim(ctx context.Context, claim *ImageStudi func (r *ImageStudioRuntime) failClaim(ctx context.Context, jobID, itemID int64, err error) { code, message := imageStudioSafeExecutionError(ctx, err) + if err != nil { + // The job item keeps its user-facing message; administrators get the + // original error (for a gateway failure its status and body) here. + slog.Warn("image_studio.item_failed", "job_id", jobID, "item_id", itemID, "code", code, "error", ImageStudioLogText(err.Error())) + } finishCtx := context.WithoutCancel(ctx) if completeErr := r.repo.CompleteFailure(finishCtx, itemID, code, message); completeErr != nil { return diff --git a/backend/internal/service/native_feature_bootstrap.go b/backend/internal/service/native_feature_bootstrap.go index 84efa286bf51..972ba5c4a8a2 100644 --- a/backend/internal/service/native_feature_bootstrap.go +++ b/backend/internal/service/native_feature_bootstrap.go @@ -55,6 +55,9 @@ type NativeFeatureBootstrapRepository interface { // retirement must commit before a native feature or the plugin manager starts. type NativeFeatureBootstrap struct { Snapshot *NativeRetirementSnapshot + + encryptor SecretEncryptor + retired RetiredPluginSource } func ProvideNativeFeatureBootstrap(repo NativeFeatureBootstrapRepository, encryptor SecretEncryptor) (*NativeFeatureBootstrap, error) { @@ -66,7 +69,103 @@ func ProvideNativeFeatureBootstrap(repo NativeFeatureBootstrapRepository, encryp if err != nil { return nil, fmt.Errorf("initialize native features: %w", err) } - return &NativeFeatureBootstrap{Snapshot: snapshot}, nil + return &NativeFeatureBootstrap{Snapshot: snapshot, encryptor: encryptor}, nil +} + +// RetiredPluginSource reads the retired first-party installation rows. They +// stay outside the plugin manager, whose reconciliation would clean them up. +type RetiredPluginSource interface { + RetiredPluginInstallations(context.Context) ([]*PluginInstallation, error) +} + +// SetRetiredPluginSource is called by the plugin repository boundary. +func (b *NativeFeatureBootstrap) SetRetiredPluginSource(source RetiredPluginSource) { + if b != nil { + b.retired = source + } +} + +// RetiredPluginInstallation is the read-only admin view of one retired +// first-party installation row with its decrypted saved configuration. +type RetiredPluginInstallation struct { + ID int64 `json:"id"` + PluginKey string `json:"plugin_key"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author string `json:"author"` + Manifest PluginManifest `json:"manifest"` + ArtifactPath string `json:"artifact_path"` + InstallPath string `json:"install_path"` + BinaryPath string `json:"binary_path"` + BinarySHA256 string `json:"binary_sha256"` + SignatureStatus string `json:"signature_status"` + State string `json:"state"` + LastError string `json:"last_error"` + InstalledBy *int64 `json:"installed_by"` + InstalledAt time.Time `json:"installed_at"` + EnabledAt *time.Time `json:"enabled_at"` + UpdatedAt time.Time `json:"updated_at"` + Bindings []PluginBinding `json:"bindings"` + Config json.RawMessage `json:"config,omitempty"` + ConfigText string `json:"config_text,omitempty"` + ConfigError string `json:"config_error,omitempty"` +} + +// RetiredPluginsView pairs the retirement receipt with the current rows. +type RetiredPluginsView struct { + Receipt *NativeRetirementSnapshot `json:"receipt"` + Installations []RetiredPluginInstallation `json:"installations"` +} + +// RetiredPlugins lists the retired first-party installations read-only, with +// their saved configuration decrypted as the plugin configuration API did. +func (b *NativeFeatureBootstrap) RetiredPlugins(ctx context.Context) (*RetiredPluginsView, error) { + view := &RetiredPluginsView{Installations: []RetiredPluginInstallation{}} + if b == nil { + return view, nil + } + view.Receipt = b.Snapshot + if b.retired == nil { + return view, nil + } + rows, err := b.retired.RetiredPluginInstallations(ctx) + if err != nil { + return nil, err + } + for _, row := range rows { + if row == nil { + continue + } + item := RetiredPluginInstallation{ + ID: row.ID, PluginKey: row.PluginKey, Name: row.Name, Version: row.Version, + Description: row.Description, Author: row.Author, Manifest: row.Manifest, + ArtifactPath: row.ArtifactPath, InstallPath: row.InstallPath, BinaryPath: row.BinaryPath, + BinarySHA256: row.BinarySHA256, SignatureStatus: row.SignatureStatus, State: row.State, + LastError: row.LastError, InstalledBy: row.InstalledBy, InstalledAt: row.InstalledAt, + EnabledAt: row.EnabledAt, UpdatedAt: row.UpdatedAt, Bindings: row.Bindings, + } + if item.Bindings == nil { + item.Bindings = []PluginBinding{} + } + switch { + case row.ConfigEncrypted == "": + case b.encryptor == nil: + item.ConfigError = "configuration decryptor is unavailable" + default: + plain, decryptErr := b.encryptor.Decrypt(row.ConfigEncrypted) + switch { + case decryptErr != nil: + item.ConfigError = decryptErr.Error() + case json.Valid([]byte(plain)): + item.Config = json.RawMessage(plain) + default: + item.ConfigText = plain + } + } + view.Installations = append(view.Installations, item) + } + return view, nil } func nativeFeatureSettings(plugin NativeRetirementPlugin, encryptor SecretEncryptor) (map[string]json.RawMessage, error) { diff --git a/backend/internal/service/native_feature_bootstrap_test.go b/backend/internal/service/native_feature_bootstrap_test.go index eff22930c9d4..5c0dc1d82aaf 100644 --- a/backend/internal/service/native_feature_bootstrap_test.go +++ b/backend/internal/service/native_feature_bootstrap_test.go @@ -1,6 +1,7 @@ package service import ( + "context" "encoding/json" "errors" "testing" @@ -98,3 +99,26 @@ func TestNativeFeatureBootstrapPreservesEffectiveConfiguration(t *testing.T) { require.Error(t, ValidateNativeFeatureSetting(SettingKeyImageToolsConfig, []byte(`{"studio_enabled":"false"}`))) require.Error(t, ValidateNativeFeatureSetting(SettingKeyImageToolsConfig, nil)) } + +type retiredPluginSourceStub []*PluginInstallation + +func (s retiredPluginSourceStub) RetiredPluginInstallations(context.Context) ([]*PluginInstallation, error) { + return s, nil +} + +func TestNativeFeatureBootstrapListsRetiredPluginsWithDecryptedConfig(t *testing.T) { + snapshot := &NativeRetirementSnapshot{Version: 1, Completed: true, Plugins: map[string]NativeRetirementPlugin{ + "codexrip.image-tools": {ID: 3, Key: "codexrip.image-tools", State: "enabled"}, + }} + bootstrap := &NativeFeatureBootstrap{Snapshot: snapshot, encryptor: retirementTestEncryptor{}} + bootstrap.SetRetiredPluginSource(retiredPluginSourceStub{{ + ID: 3, PluginKey: "codexrip.image-tools", State: "disabled", LastError: "stopped by retirement", + ConfigEncrypted: `{"studio_enabled":true,"api_key":"saved-secret"}`, + }}) + view, err := bootstrap.RetiredPlugins(context.Background()) + require.NoError(t, err) + require.Same(t, snapshot, view.Receipt) + require.Len(t, view.Installations, 1) + require.Equal(t, "stopped by retirement", view.Installations[0].LastError) + require.JSONEq(t, `{"studio_enabled":true,"api_key":"saved-secret"}`, string(view.Installations[0].Config)) +} diff --git a/backend/internal/service/openai_account_runtime_block_fastpath.go b/backend/internal/service/openai_account_runtime_block_fastpath.go index 1c45ab174017..d4d1437bcd3e 100644 --- a/backend/internal/service/openai_account_runtime_block_fastpath.go +++ b/backend/internal/service/openai_account_runtime_block_fastpath.go @@ -502,12 +502,15 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont case http.StatusUnauthorized, http.StatusForbidden, http.StatusTooManyRequests: // Authentication and quota responses are handled exclusively by the // request failover state: no same-account replay, runtime cooldown, then - // switch. Do not persist legacy schedulable/error state here. + // switch. Do not persist legacy schedulable/error state here; only the + // upstream's own error text is kept in a non-scheduling field. + s.recordOpenAILastUpstreamError(ctx, account, statusCode, responseBody) return false } } if isOpenAIOAuthAccount(account) && (statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden) && strings.EqualFold(strings.TrimSpace(extractUpstreamErrorMessage(responseBody)), "credential or quota failure") { + s.recordOpenAILastUpstreamError(ctx, account, statusCode, responseBody) return false } stateCtx, cancel := openAIAccountStateContext(ctx) diff --git a/backend/internal/service/openai_budget_exceeded.go b/backend/internal/service/openai_budget_exceeded.go index 5d545642e389..c06e7131f0c1 100644 --- a/backend/internal/service/openai_budget_exceeded.go +++ b/backend/internal/service/openai_budget_exceeded.go @@ -1,12 +1,16 @@ package service import ( + "bytes" "context" + "encoding/json" + "fmt" "log/slog" "net/http" "strings" "time" + "github.com/gin-gonic/gin" "github.com/tidwall/gjson" ) @@ -64,12 +68,44 @@ func openAIBudgetExceededMessage(payload []byte) string { return buildForbiddenErrorMessage("", "", payload, openAIBudgetExceededKind) } +// openAIBudgetExceededAccountError is the account error text: the upstream's +// own message followed by the structured type/code/param that identify the +// relay's budget rule. Without a message the complete error object is kept. +func openAIBudgetExceededAccountError(payload []byte) string { + errorObject := gjson.GetBytes(payload, "error") + if !errorObject.IsObject() { + errorObject = gjson.GetBytes(payload, "response.error") + } + message := strings.TrimSpace(errorObject.Get("message").String()) + if message == "" { + raw := []byte(errorObject.Raw) + if !errorObject.IsObject() { + raw = bytes.TrimSpace(payload) + } + var compact bytes.Buffer + if json.Compact(&compact, raw) == nil && compact.Len() > 0 { + return compact.String() + } + return openAIBudgetExceededMessage(payload) + } + var fields []string + for _, name := range [...]string{"type", "code", "param"} { + if value := errorObject.Get(name); value.Exists() && value.Type != gjson.Null && strings.TrimSpace(value.String()) != "" { + fields = append(fields, fmt.Sprintf("%s=%s", name, strings.TrimSpace(value.String()))) + } + } + if len(fields) == 0 { + return message + } + return message + " (" + strings.Join(fields, ", ") + ")" +} + // handleOpenAIBudgetExceeded moves the account to the error state. func (s *RateLimitService) handleOpenAIBudgetExceeded(ctx context.Context, account *Account, payload []byte) { if s == nil || account == nil { return } - message := openAIBudgetExceededMessage(payload) + message := openAIBudgetExceededAccountError(payload) s.notifyAccountSchedulingBlocked(account, time.Time{}, openAIBudgetExceededKind) if s.accountRepo == nil { return @@ -105,11 +141,13 @@ func (s *OpenAIGatewayService) handleOpenAIBudgetExceededTerminalEvent(ctx conte } // openAIBudgetExceededTerminalFailover converts a budget terminal event into -// an account-scoped failover that never retries the same account. -func (s *OpenAIGatewayService) openAIBudgetExceededTerminalFailover(ctx context.Context, account *Account, headers http.Header, payload []byte) (*UpstreamFailoverError, bool) { +// an account-scoped failover that never retries the same account. replayable +// tells whether the caller can still move the request to another account. +func (s *OpenAIGatewayService) openAIBudgetExceededTerminalFailover(ctx context.Context, c *gin.Context, account *Account, headers http.Header, payload []byte, webSocket bool, replayable bool) (*UpstreamFailoverError, bool) { if !s.handleOpenAIBudgetExceededTerminalEvent(ctx, account, payload) { return nil, false } + s.recordOpenAIBudgetExceededAttempt(ctx, c, account, http.StatusTooManyRequests, headers, payload, webSocket, replayable) return &UpstreamFailoverError{ StatusCode: http.StatusTooManyRequests, ResponseBody: append([]byte(nil), payload...), @@ -122,22 +160,62 @@ func (s *OpenAIGatewayService) openAIBudgetExceededTerminalFailover(ctx context. // openAIBudgetExceededHTTPResponseTerminalFailover accepts a terminal event // only on an exact HTTP-200 response, so a 201/202 body cannot masquerade as // an in-band budget event. -func (s *OpenAIGatewayService) openAIBudgetExceededHTTPResponseTerminalFailover(ctx context.Context, account *Account, statusCode int, headers http.Header, payload []byte) (*UpstreamFailoverError, bool) { +func (s *OpenAIGatewayService) openAIBudgetExceededHTTPResponseTerminalFailover(ctx context.Context, c *gin.Context, account *Account, statusCode int, headers http.Header, payload []byte) (*UpstreamFailoverError, bool) { if statusCode != http.StatusOK { return nil, false } - return s.openAIBudgetExceededTerminalFailover(ctx, account, headers, payload) + // Once client output is committed the request cannot move to another account. + return s.openAIBudgetExceededTerminalFailover(ctx, c, account, headers, payload, false, !openAIStreamClientOutputStarted(c, false)) } // handleOpenAIBudgetExceededHTTPFailover consumes a budget_exceeded HTTP 429 // before any generic retry, recovery or rewriting can reinterpret it. -func (s *OpenAIGatewayService) handleOpenAIBudgetExceededHTTPFailover(ctx context.Context, account *Account, statusCode int, headers http.Header, body []byte) (*UpstreamFailoverError, bool) { +func (s *OpenAIGatewayService) handleOpenAIBudgetExceededHTTPFailover(ctx context.Context, c *gin.Context, account *Account, statusCode int, headers http.Header, body []byte) (*UpstreamFailoverError, bool) { if account == nil || !account.IsOpenAICompatible() || !isOpenAIBudgetExceededResponse(statusCode, body) { return nil, false } s.handleOpenAIBudgetExceeded(ctx, account, body) + s.recordOpenAIBudgetExceededAttempt(ctx, c, account, statusCode, headers, body, false, true) failoverErr := newOpenAIUpstreamFailoverError(statusCode, headers, body, openAIBudgetExceededMessage(body), false) failoverErr.Scope = GatewayFailureScopeAccount failoverErr.NextAccountAction = NextAccountRetry return failoverErr, true } + +// recordOpenAIBudgetExceededAttempt records the budget rejection like any other +// upstream 429 attempt: the upstream message, the bounded body and the body log +// line. It counts as a failover only while the request can still move to +// another account. +func (s *OpenAIGatewayService) recordOpenAIBudgetExceededAttempt(ctx context.Context, c *gin.Context, account *Account, statusCode int, headers http.Header, payload []byte, webSocket bool, replayable bool) { + // Keep the upstream Agent Identity credential redaction for recorded bodies. + payload = s.redactAgentIdentitySensitiveBody(ctx, account, payload) + s.logOpenAIUpstreamErrorBody(account, statusCode, payload) + if c == nil { + return + } + message := sanitizeUpstreamErrorMessage(strings.TrimSpace(openAIBudgetExceededMessage(payload))) + detail := s.openAIUpstreamErrorDetail(payload) + setOpsUpstreamError(c, statusCode, message, detail) + proxyID, proxyName := opsUpstreamProxyAttribution(account) + if webSocket { + proxyID, proxyName = opsUpstreamWSProxyAttribution(account) + } + kind := "failover" + if !replayable { + kind = "stream_error" + } + event := OpsUpstreamErrorEvent{ + ProxyID: proxyID, + ProxyName: proxyName, + UpstreamStatusCode: statusCode, + UpstreamRequestID: headers.Get("x-request-id"), + Kind: kind, + Reason: openAIBudgetExceededKind, + Message: message, + Detail: detail, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} diff --git a/backend/internal/service/openai_codex_ticket_test.go b/backend/internal/service/openai_codex_ticket_test.go index 9f76da379d50..ae6cb97466a2 100644 --- a/backend/internal/service/openai_codex_ticket_test.go +++ b/backend/internal/service/openai_codex_ticket_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/Wei-Shaw/sub2api/internal/codexruntime/tickets" "github.com/Wei-Shaw/sub2api/internal/config" extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi" "github.com/stretchr/testify/require" @@ -73,7 +74,28 @@ func TestCodexTicketNativePolicyFailureDoesNotFallBack(t *testing.T) { }) service := &OpenAIGatewayService{nativeCodexRuntime: manager} require.ErrorIs(t, service.applyOpenAICodexTicket(context.Background(), ticketTestAccount(41), "gpt-6-astra", http.Header{}), ErrOpenAICodexTicketUnavailable) + // A runtime refusal keeps its own result code and reason. + service.nativeCodexRuntime = nativeTicketTestRuntime(t, config.OpenAICodexTicketConfig{Enabled: true, FailClosed: true}, func(extensionv1.Invocation) (extensionv1.Result, error) { + return extensionv1.Result{Code: "ticket_missing", Message: "no route qualification is recorded (routing phase stopped; last result routing_upstream)"}, nil + }) + err := service.applyOpenAICodexTicket(context.Background(), ticketTestAccount(41), "gpt-6-astra", http.Header{}) + require.ErrorIs(t, err, ErrOpenAICodexTicketUnavailable) + require.ErrorIs(t, err, ErrNativeCodexRuntimeUnavailable) + var refusal *NativeCodexResultError + require.ErrorAs(t, err, &refusal) + require.Equal(t, "ticket_missing", refusal.Code) + require.Contains(t, err.Error(), "last result routing_upstream") +} +func TestCodexTicketInvokeFailureUsesTheJobContext(t *testing.T) { + running := context.Background() + canceled, cancel := context.WithCancel(context.Background()) + cancel() + require.Equal(t, "ticket_plugin_unavailable", codexTicketInvokeFailure(running, tickets.ErrStopping).Code, "a runtime reload is not a job cancellation") + require.Equal(t, "ticket_plugin_unavailable", codexTicketInvokeFailure(running, context.Canceled).Code) + require.Equal(t, "ticket_canceled", codexTicketInvokeFailure(canceled, context.Canceled).Code) + require.Equal(t, "ticket_timeout", codexTicketInvokeFailure(running, context.DeadlineExceeded).Code) } + func TestOpenAICodexTicketStatusesUsesSanitizedProjection(t *testing.T) { account := ticketTestAccount(41) now := time.Now() @@ -88,6 +110,26 @@ func TestOpenAICodexTicketStatusesUsesSanitizedProjection(t *testing.T) { require.True(t, OpenAICodexTicketStatuses(account, cfg, now)[0].Blocked) cfg.Enabled = false require.Empty(t, OpenAICodexTicketStatuses(account, cfg, now)) + // With the switch off, recorded history stays visible with its real code, + // HTTP status, response model and original message; nothing is ready or blocked. + account = ticketTestAccount(41) + setTicketTestProjection(account, "gpt-6-astra", now.Add(time.Hour)) + plugins, ok := account.Extra[NativeCodexAccountProjectionKey].(map[string]any) + require.True(t, ok) + projection, ok := plugins[codexRuntimePluginKey].(map[string]any) + require.True(t, ok) + projection["observations"] = map[string]any{"gpt-6-astra": map[string]any{"key": "gpt-6-astra", "kind": "codex_routing", "state": "retry", "code": "routing_persist", "http_status": 503, "response_model": "gpt-6-luna", "message": "save cookie clock: database is locked"}} + statuses = OpenAICodexTicketStatuses(account, cfg, now) + require.Len(t, statuses, 1) + require.False(t, statuses[0].Ready) + require.False(t, statuses[0].Blocked) + require.Equal(t, "retry", statuses[0].RenewalState) + require.NotNil(t, statuses[0].LastResult) + require.Equal(t, "routing_persist", statuses[0].LastResult.Code, "a code outside the old catalog is not rewritten") + require.Equal(t, 503, statuses[0].LastResult.HTTPStatus) + require.Equal(t, "gpt-6-luna", statuses[0].LastResult.ResponseModel) + require.Equal(t, CodexTicketFailure("routing_persist").Message, statuses[0].LastResult.Message, "the short catalog sentence") + require.Equal(t, "save cookie clock: database is locked", statuses[0].LastResult.Error, "the raw text") } func TestExtractOpenAICodexTicketModel(t *testing.T) { require.Equal(t, "gpt-6-astra", extractOpenAICodexTicketModel([]byte(`{"model":" gpt-6-astra "}`))) diff --git a/backend/internal/service/openai_continuation_diagnostic.go b/backend/internal/service/openai_continuation_diagnostic.go index 64bc2b17e118..9f5fb16d2b7e 100644 --- a/backend/internal/service/openai_continuation_diagnostic.go +++ b/backend/internal/service/openai_continuation_diagnostic.go @@ -8,7 +8,6 @@ import ( "hash" "io" "net/http" - "regexp" "strings" "unicode/utf8" @@ -19,11 +18,21 @@ import ( const ( openAIContinuationDiagnosticBodyLimit = 8 << 20 openAIContinuationDiagnosticItemLimit = 1024 - openAIContinuationDiagnosticJSONLimit = 6 << 10 + // The serialized diagnostic stays inside the per-attempt Ops budget. When a + // document is too large its plaintext values are truncated, never dropped. + openAIContinuationDiagnosticJSONLimit = 16 << 10 + + // Per-field bounds for the original value kept next to each fingerprint. + openAIContinuationDiagnosticIDValueLimit = 512 + openAIContinuationDiagnosticInstructionsValueLimit = 4 << 10 + openAIContinuationDiagnosticMessageValueLimit = 2 << 10 + openAIContinuationDiagnosticCodeValueLimit = 256 + openAIContinuationDiagnosticParamValueLimit = 512 ) // The diagnostic is an observation of an existing failed attempt. It is not -// a retry signal or a request replay record, and contains no arbitrary text. +// a retry signal or a request replay record. Administrators see the bounded +// original values next to their complete-value fingerprints. type OpenAIContinuationDiagnostic struct { Version int `json:"version"` Classification string `json:"classification"` @@ -40,12 +49,16 @@ type openAIContinuationRecoveryShape struct { NotAttemptedReason string `json:"not_attempted_reason,omitempty"` } +// Bytes, Characters and SHA256 always describe the complete original value. +// Value is that original (JSON text for non-string values), cut to the field's +// byte bound; Truncated marks a partial value. type openAIContinuationFingerprint struct { Kind string `json:"kind"` Bytes int `json:"bytes"` Characters int `json:"characters"` SHA256 string `json:"sha256,omitempty"` Value string `json:"value,omitempty"` + Truncated bool `json:"truncated,omitempty"` } type openAIContinuationErrorShape struct { @@ -147,7 +160,7 @@ func continuationDiagnosticWireBody(req *http.Request, fallback []byte) ([]byte, return body, "actual_request", len(body) > openAIContinuationDiagnosticBodyLimit } -func continuationDiagnosticFingerprint(value gjson.Result) openAIContinuationFingerprint { +func continuationDiagnosticFingerprint(value gjson.Result, valueLimit int) openAIContinuationFingerprint { if !value.Exists() { return openAIContinuationFingerprint{Kind: "missing"} } @@ -169,7 +182,24 @@ func continuationDiagnosticFingerprint(value gjson.Result) openAIContinuationFin } } digest := sha256.Sum256([]byte(raw)) - return openAIContinuationFingerprint{Kind: kind, Bytes: len(raw), Characters: utf8.RuneCountInString(raw), SHA256: hex.EncodeToString(digest[:])} + fingerprint := openAIContinuationFingerprint{Kind: kind, Bytes: len(raw), Characters: utf8.RuneCountInString(raw), SHA256: hex.EncodeToString(digest[:])} + if kind != "null" { + fingerprint.Value, fingerprint.Truncated = continuationDiagnosticBoundValue(raw, valueLimit) + } + return fingerprint +} + +// continuationDiagnosticBoundValue keeps a valid UTF-8 prefix of at most limit +// bytes and reports whether anything was cut. +func continuationDiagnosticBoundValue(value string, limit int) (string, bool) { + value = strings.ToValidUTF8(value, "�") + if limit <= 0 { + return "", value != "" + } + if len(value) <= limit { + return strings.Clone(value), false + } + return strings.Clone(truncateString(value, limit)), true } func continuationDiagnosticHeader(headers http.Header, name string) openAIContinuationFingerprint { @@ -177,7 +207,7 @@ func continuationDiagnosticHeader(headers http.Header, name string) openAIContin if len(values) == 0 { return openAIContinuationFingerprint{Kind: "missing"} } - return continuationDiagnosticFingerprint(gjson.Result{Type: gjson.String, Str: values[0]}) + return continuationDiagnosticFingerprint(gjson.Result{Type: gjson.String, Str: values[0]}, openAIContinuationDiagnosticIDValueLimit) } func continuationDiagnosticRequest(body []byte, headers http.Header, source string) openAIContinuationRequestShape { @@ -192,10 +222,10 @@ func continuationDiagnosticRequest(body []byte, headers http.Header, source stri return shape } fields := gjson.GetManyBytes(body, "prompt_cache_key", "instructions", "previous_response_id", "client_metadata.session_id", "input") - shape.PromptCache = continuationDiagnosticFingerprint(fields[0]) - shape.Instructions = continuationDiagnosticFingerprint(fields[1]) - shape.PreviousResponse = continuationDiagnosticFingerprint(fields[2]) - shape.ClientMetadataSession = continuationDiagnosticFingerprint(fields[3]) + shape.PromptCache = continuationDiagnosticFingerprint(fields[0], openAIContinuationDiagnosticIDValueLimit) + shape.Instructions = continuationDiagnosticFingerprint(fields[1], openAIContinuationDiagnosticInstructionsValueLimit) + shape.PreviousResponse = continuationDiagnosticFingerprint(fields[2], openAIContinuationDiagnosticIDValueLimit) + shape.ClientMetadataSession = continuationDiagnosticFingerprint(fields[3], openAIContinuationDiagnosticIDValueLimit) shape.History = continuationDiagnosticHistory(fields[4]) return shape } @@ -286,18 +316,6 @@ func continuationDiagnosticHistory(input gjson.Result) openAIContinuationHistory return shape } -var continuationDiagnosticParam = regexp.MustCompile(`^(?:model|instructions|prompt_cache_key|previous_response_id|store|stream|input|tools|reasoning(?:\.(?:effort|mode|context))?|(?:input|tools)(?:\[[0-9]{1,6}\]|\.[0-9]{1,6})(?:\.(?:id|call_id|type|role|content|encrypted_content|name|namespace|arguments|output|function)(?:\.(?:name|arguments))?)?)$`) - -func continuationDiagnosticKnownError(value string) string { - switch value { - case "invalid_request_error", "invalid_argument", "bad_request", "upstream_error", "server_error", "validation_error", "invalid_request", "BadRequest", - "previous_response_not_found", "invalid_encrypted_content", "thinking_signature_invalid", "model_not_found", "unsupported_parameter", "unknown_parameter", "invalid_value", "missing_required_parameter", "context_length_exceeded", "400": - return strings.Clone(value) - default: - return "" - } -} - var continuationDiagnosticHints = []string{ "previous_response_not_found", "previous response not found", "invalid_encrypted_content", "encrypted content could not be verified", "thinking_signature_invalid", "unknown parameter", "unsupported", "missing", "function_call_output", "call_id", "instructions", "prompt_cache_key", @@ -323,17 +341,10 @@ func continuationDiagnosticError(body []byte) openAIContinuationErrorShape { message = root } shape := openAIContinuationErrorShape{ - ErrorType: continuationDiagnosticFingerprint(typ), ErrorCode: continuationDiagnosticFingerprint(code), - ErrorParam: continuationDiagnosticFingerprint(param), Message: continuationDiagnosticFingerprint(message), - } - if typ.Type == gjson.String { - shape.ErrorType.Value = continuationDiagnosticKnownError(typ.String()) - } - if code.Type == gjson.String { - shape.ErrorCode.Value = continuationDiagnosticKnownError(code.String()) - } - if param.Type == gjson.String && len(param.String()) <= 96 && continuationDiagnosticParam.MatchString(param.String()) { - shape.ErrorParam.Value = strings.Clone(param.String()) + ErrorType: continuationDiagnosticFingerprint(typ, openAIContinuationDiagnosticCodeValueLimit), + ErrorCode: continuationDiagnosticFingerprint(code, openAIContinuationDiagnosticCodeValueLimit), + ErrorParam: continuationDiagnosticFingerprint(param, openAIContinuationDiagnosticParamValueLimit), + Message: continuationDiagnosticFingerprint(message, openAIContinuationDiagnosticMessageValueLimit), } if message.Type == gjson.String { lower := strings.ToLower(message.String()) @@ -357,16 +368,38 @@ func continuationDiagnosticClassification(value string) string { } } -// Queue-bound validation is independent of best-effort text redaction. No -// producer can smuggle arbitrary strings through this typed metadata field. +// sanitizeOpenAIContinuationDiagnostic normalizes the structural enums and +// bounds every retained original value. A document that is still larger than +// the Ops budget keeps all fingerprints, lengths and counters and has its +// values shortened further; it is never discarded. func sanitizeOpenAIContinuationDiagnostic(in *OpenAIContinuationDiagnostic) *OpenAIContinuationDiagnostic { if in == nil { return nil } + var out *OpenAIContinuationDiagnostic + for divisor := 1; divisor <= 64; divisor *= 2 { + out = sanitizeOpenAIContinuationDiagnosticValues(in, divisor) + if encoded, err := json.Marshal(out); err == nil && len(encoded) <= openAIContinuationDiagnosticJSONLimit { + return out + } + } + // The fixed structure alone always fits; values are the only variable part. + return sanitizeOpenAIContinuationDiagnosticValues(in, 0) +} + +// sanitizeOpenAIContinuationDiagnosticValues divides every per-field value +// bound by divisor; divisor 0 keeps no plaintext values. +func sanitizeOpenAIContinuationDiagnosticValues(in *OpenAIContinuationDiagnostic, divisor int) *OpenAIContinuationDiagnostic { out := *in out.Version = 1 out.Classification = continuationDiagnosticClassification(in.Classification) - cleanFingerprint := func(f openAIContinuationFingerprint) openAIContinuationFingerprint { + bound := func(limit int) int { + if divisor <= 0 { + return 0 + } + return limit / divisor + } + cleanFingerprint := func(f openAIContinuationFingerprint, limit int) openAIContinuationFingerprint { switch f.Kind { case "missing", "null", "string", "number", "boolean", "array", "object", "uninspected": default: @@ -379,7 +412,8 @@ func sanitizeOpenAIContinuationDiagnostic(in *OpenAIContinuationDiagnostic) *Ope f.Characters = 0 } f.SHA256 = continuationDiagnosticSafeHash(f.SHA256) - f.Value = "" + value, cut := continuationDiagnosticBoundValue(f.Value, bound(limit)) + f.Value, f.Truncated = value, f.Truncated || cut return f } cleanRequest := func(r openAIContinuationRequestShape) openAIContinuationRequestShape { @@ -391,8 +425,12 @@ func sanitizeOpenAIContinuationDiagnostic(in *OpenAIContinuationDiagnostic) *Ope if r.BodyBytes < 0 { r.BodyBytes = 0 } - r.PromptCache, r.Instructions, r.PreviousResponse = cleanFingerprint(r.PromptCache), cleanFingerprint(r.Instructions), cleanFingerprint(r.PreviousResponse) - r.Session, r.Conversation, r.ClientMetadataSession = cleanFingerprint(r.Session), cleanFingerprint(r.Conversation), cleanFingerprint(r.ClientMetadataSession) + r.PromptCache = cleanFingerprint(r.PromptCache, openAIContinuationDiagnosticIDValueLimit) + r.Instructions = cleanFingerprint(r.Instructions, openAIContinuationDiagnosticInstructionsValueLimit) + r.PreviousResponse = cleanFingerprint(r.PreviousResponse, openAIContinuationDiagnosticIDValueLimit) + r.Session = cleanFingerprint(r.Session, openAIContinuationDiagnosticIDValueLimit) + r.Conversation = cleanFingerprint(r.Conversation, openAIContinuationDiagnosticIDValueLimit) + r.ClientMetadataSession = cleanFingerprint(r.ClientMetadataSession, openAIContinuationDiagnosticIDValueLimit) r.History.CallIDsSHA256 = continuationDiagnosticSafeHash(r.History.CallIDsSHA256) r.History.OutputIDsSHA256 = continuationDiagnosticSafeHash(r.History.OutputIDsSHA256) r.History.EncryptedSHA256 = continuationDiagnosticSafeHash(r.History.EncryptedSHA256) @@ -419,14 +457,10 @@ func sanitizeOpenAIContinuationDiagnostic(in *OpenAIContinuationDiagnostic) *Ope } out.Recovery = &recovery } - out.UpstreamError.ErrorType, out.UpstreamError.ErrorCode = cleanFingerprint(in.UpstreamError.ErrorType), cleanFingerprint(in.UpstreamError.ErrorCode) - out.UpstreamError.ErrorType.Value = continuationDiagnosticKnownError(in.UpstreamError.ErrorType.Value) - out.UpstreamError.ErrorCode.Value = continuationDiagnosticKnownError(in.UpstreamError.ErrorCode.Value) - out.UpstreamError.ErrorParam, out.UpstreamError.Message = cleanFingerprint(in.UpstreamError.ErrorParam), cleanFingerprint(in.UpstreamError.Message) - param := in.UpstreamError.ErrorParam.Value - if len(param) <= 96 && continuationDiagnosticParam.MatchString(param) { - out.UpstreamError.ErrorParam.Value = strings.Clone(param) - } + out.UpstreamError.ErrorType = cleanFingerprint(in.UpstreamError.ErrorType, openAIContinuationDiagnosticCodeValueLimit) + out.UpstreamError.ErrorCode = cleanFingerprint(in.UpstreamError.ErrorCode, openAIContinuationDiagnosticCodeValueLimit) + out.UpstreamError.ErrorParam = cleanFingerprint(in.UpstreamError.ErrorParam, openAIContinuationDiagnosticParamValueLimit) + out.UpstreamError.Message = cleanFingerprint(in.UpstreamError.Message, openAIContinuationDiagnosticMessageValueLimit) out.UpstreamError.Hints = nil for _, allowed := range continuationDiagnosticHints { for _, hint := range in.UpstreamError.Hints { @@ -436,10 +470,6 @@ func sanitizeOpenAIContinuationDiagnostic(in *OpenAIContinuationDiagnostic) *Ope } } } - encoded, err := json.Marshal(out) - if err != nil || len(encoded) > openAIContinuationDiagnosticJSONLimit { - return nil - } return &out } diff --git a/backend/internal/service/openai_continuation_diagnostic_forward_test.go b/backend/internal/service/openai_continuation_diagnostic_forward_test.go index ad4c702ca7a4..21f4fde46e87 100644 --- a/backend/internal/service/openai_continuation_diagnostic_forward_test.go +++ b/backend/internal/service/openai_continuation_diagnostic_forward_test.go @@ -125,6 +125,8 @@ func TestOpenAIContinuationDiagnosticForwardPreservesTerminalBehavior(t *testing Body: io.NopCloser(strings.NewReader(tt.upstreamError)), }} cfg := &config.Config{} + cfg.Gateway.LogUpstreamErrorBody = true + cfg.Gateway.LogUpstreamErrorBodyMaxBytes = 2048 svc := &OpenAIGatewayService{ cfg: cfg, httpUpstream: upstream, @@ -150,6 +152,9 @@ func TestOpenAIContinuationDiagnosticForwardPreservesTerminalBehavior(t *testing var terminal *UpstreamFailoverError require.ErrorAs(t, err, &terminal) expectedMessage, expectedKind := OpenAIContinuationStateUnavailableClientMessage, "continuation_state" + // Ops keeps the upstream's own message and bounded body; only the + // client terminal is fixed. + expectedOpsMessage := gjson.Get(tt.upstreamError, "error.message").String() if tt.requestRejected { expectedMessage, expectedKind = OpenAIRequestRejectedClientMessage, "request_rejected" require.True(t, terminal.IsOpenAIRequestRejected()) @@ -186,8 +191,14 @@ func TestOpenAIContinuationDiagnosticForwardPreservesTerminalBehavior(t *testing require.Equal(t, expectedKind, event.Kind) require.Equal(t, tt.passthrough, event.Passthrough) require.Equal(t, tt.status, event.UpstreamStatusCode) - require.Equal(t, expectedMessage, event.Message) - require.Empty(t, event.Detail, "diagnostics must not change passthrough-rule keyword input") + require.Equal(t, expectedOpsMessage, event.Message) + require.Equal(t, tt.upstreamError, event.Detail) + if tt.passthrough { + require.Equal(t, tt.upstreamError, event.UpstreamResponseBody) + } + require.Equal(t, "direct/no_proxy", event.ProxyName) + topMessage, _ := c.Get(OpsUpstreamErrorMessageKey) + require.Equal(t, expectedOpsMessage, topMessage, "the top-level upstream message is never the client sentence") require.NotNil(t, event.ContinuationDiagnostic) diagnostic := event.ContinuationDiagnostic require.Equal(t, tt.classification, diagnostic.Classification) @@ -205,7 +216,9 @@ func TestOpenAIContinuationDiagnosticForwardPreservesTerminalBehavior(t *testing encoded, encodeErr := json.Marshal(event) require.NoError(t, encodeErr) require.True(t, gjson.GetBytes(encoded, "continuation_diagnostic").IsObject()) - require.NotContains(t, string(encoded), "private-validation-message-51937") + if tt.upstreamError == missingNamespace { + require.Contains(t, string(encoded), "private-validation-message-51937") + } }) } } diff --git a/backend/internal/service/openai_continuation_diagnostic_test.go b/backend/internal/service/openai_continuation_diagnostic_test.go index b2a0e2b3dfb5..7e0a40bf7ddc 100644 --- a/backend/internal/service/openai_continuation_diagnostic_test.go +++ b/backend/internal/service/openai_continuation_diagnostic_test.go @@ -53,7 +53,7 @@ func continuationDiagnosticTestEncoded(t *testing.T, diagnostic *OpenAIContinuat t.Fatal("diagnostic is missing") } encoded := continuationDiagnosticTestJSON(t, diagnostic) - if len(encoded) > 6*1024 { + if len(encoded) > openAIContinuationDiagnosticJSONLimit { t.Fatalf("diagnostic exceeds its fixed serialization budget: %d bytes", len(encoded)) } root := gjson.ParseBytes(encoded) @@ -72,14 +72,17 @@ func continuationDiagnosticTestNoPlaintext(t *testing.T, encoded string, values } } -func TestOpenAIContinuationDiagnosticPrivacy(t *testing.T) { - t.Run("known_enums_and_protocol_param_only", func(t *testing.T) { - const message = "encrypted content could not be verified: private-upstream-message-69317" +// Administrators see the upstream error fields verbatim (bounded) next to the +// fingerprints of the complete values. This replaces the earlier contract that +// kept only known enum values and hashes. +func TestOpenAIContinuationDiagnosticRetainsBoundedErrorValues(t *testing.T) { + t.Run("original_error_fields", func(t *testing.T) { + const message = "encrypted content could not be verified: upstream-message-69317" upstream := continuationDiagnosticTestJSON(t, map[string]any{ "error": map[string]any{ "type": "invalid_request_error", "code": "invalid_encrypted_content", "param": "input[12].encrypted_content", "message": message, - "debug": map[string]any{"credentials": "unlisted-debug-secret-49217"}, + "debug": map[string]any{"credentials": "unlisted-debug-field-49217"}, }, }) body := []byte(`{"input":[]}`) @@ -90,71 +93,55 @@ func TestOpenAIContinuationDiagnosticPrivacy(t *testing.T) { "upstream_error.error_type.value": "invalid_request_error", "upstream_error.error_code.value": "invalid_encrypted_content", "upstream_error.error_param.value": "input[12].encrypted_content", + "upstream_error.message.value": message, "upstream_error.message.sha256": continuationDiagnosticTestDigest(message), } { if root.Get(path).String() != expected { t.Errorf("diagnostic contract mismatch at %s", path) } } - if root.Get("upstream_error.message.value").Exists() { - t.Fatal("upstream message must never have a plaintext value") - } - continuationDiagnosticTestNoPlaintext(t, encoded, "private-upstream-message-69317", "unlisted-debug-secret-49217") + // Only the error fields are projected; the complete body stays in the + // attempt's upstream detail. + continuationDiagnosticTestNoPlaintext(t, encoded, "unlisted-debug-field-49217") }) - t.Run("namespace_param_whitelist", func(t *testing.T) { - for _, tc := range []struct { - param string - allowed bool - }{ - {"input[11].namespace", true}, - {"input.11.namespace", true}, - {"tools[0].namespace", true}, - {"input[11].namespace.private-field-51937", false}, - {"input[11].namespace?token=private-token-71359", false}, - } { - t.Run(tc.param, func(t *testing.T) { + t.Run("every_param_is_kept", func(t *testing.T) { + for _, param := range []string{"input[11].namespace", "input.11.namespace", "tools[0].namespace", "input[11].namespace.custom-field-51937", "input[11].namespace?token=query-71359"} { + t.Run(param, func(t *testing.T) { body := []byte(`{"input":[]}`) upstream := continuationDiagnosticTestJSON(t, map[string]any{"error": map[string]any{ - "type": "invalid_request_error", "param": tc.param, - "message": "Missing namespace: private-upstream-message-31957", + "type": "invalid_request_error", "param": param, + "message": "Missing namespace: upstream-message-31957", }}) diagnostic := buildOpenAIContinuationDiagnostic(continuationDiagnosticTestContext(body), body, nil, body, upstream, "request_validation") for _, candidate := range []*OpenAIContinuationDiagnostic{diagnostic, sanitizeOpenAIContinuationDiagnostic(diagnostic)} { - encoded, root := continuationDiagnosticTestEncoded(t, candidate) - param := root.Get("upstream_error.error_param") - if param.Get("sha256").String() != continuationDiagnosticTestDigest(tc.param) { - t.Fatal("namespace parameter must retain its complete fingerprint") + _, root := continuationDiagnosticTestEncoded(t, candidate) + value := root.Get("upstream_error.error_param") + if value.Get("sha256").String() != continuationDiagnosticTestDigest(param) || value.Get("value").String() != param { + t.Fatal("the upstream param must keep its value and complete fingerprint") } - if tc.allowed { - if param.Get("value").String() != tc.param { - t.Fatal("safe indexed namespace parameter was lost") - } - } else { - if param.Get("value").Exists() { - t.Fatal("non-protocol namespace suffix retained a plaintext value") - } - continuationDiagnosticTestNoPlaintext(t, encoded, tc.param) + if root.Get("upstream_error.message.value").String() != "Missing namespace: upstream-message-31957" { + t.Fatal("the upstream message must be kept") } - continuationDiagnosticTestNoPlaintext(t, encoded, "private-upstream-message-31957") } }) } }) t.Run("non_json", func(t *testing.T) { - const message = "unknown parameter: prompt_cache_key; private-non-json-secret-71241" + const message = "unknown parameter: prompt_cache_key; upstream-non-json-71241" body := []byte(`{"input":[]}`) diagnostic := buildOpenAIContinuationDiagnostic(continuationDiagnosticTestContext(body), body, nil, body, []byte(message), "unclassified") - encoded, root := continuationDiagnosticTestEncoded(t, diagnostic) + _, root := continuationDiagnosticTestEncoded(t, diagnostic) if root.Get("upstream_error.message.sha256").String() != continuationDiagnosticTestDigest(message) || root.Get("upstream_error.message.bytes").Int() != int64(len(message)) || - root.Get("upstream_error.message.characters").Int() != int64(utf8.RuneCountInString(message)) { - t.Fatal("non-JSON upstream error must retain complete message fingerprint and lengths") + root.Get("upstream_error.message.characters").Int() != int64(utf8.RuneCountInString(message)) || + root.Get("upstream_error.message.value").String() != message { + t.Fatal("a non-JSON upstream error must keep its text, complete fingerprint and lengths") } - for _, field := range []string{"error_type", "error_code", "error_param", "message"} { + for _, field := range []string{"error_type", "error_code", "error_param"} { if root.Get("upstream_error." + field + ".value").Exists() { - t.Errorf("non-JSON upstream %s must not contain a plaintext value", field) + t.Errorf("a non-JSON upstream error has no structured %s value", field) } } hints := make(map[string]bool) @@ -165,47 +152,55 @@ func TestOpenAIContinuationDiagnosticPrivacy(t *testing.T) { if !hints["unknown parameter"] || !hints["prompt_cache_key"] { t.Fatal("non-JSON upstream error lost the fixed diagnostic hint categories") } - continuationDiagnosticTestNoPlaintext(t, encoded, message, "private-non-json-secret-71241") }) for _, tc := range []struct { name string value any }{ - {"api_key", "sk-privatecredential481735"}, - {"unknown_identifier", "private_credential_identifier_58493"}, - {"bearer", "Bearer private-bearer-material-59314"}, - {"email", "diagnostic-private-481735@example.invalid"}, - {"url", "https://example.invalid/error?access_token=private-query-59317"}, - {"crlf", "private-first-line-51973\r\nAuthorization: Bearer private-second-line-71935"}, + {"api_key", "sk-credential481735"}, + {"unknown_identifier", "credential_identifier_58493"}, + {"bearer", "Bearer bearer-material-59314"}, + {"email", "diagnostic-481735@example.invalid"}, + {"url", "https://example.invalid/error?access_token=query-59317"}, + {"crlf", "first-line-51973\r\nAuthorization: Bearer second-line-71935"}, {"overlong_unicode", strings.Repeat("私密字段", 4096)}, - {"object", map[string]any{"unexpected": "private-object-value-59317"}}, - {"array", []any{"private-array-value-59317"}}, + {"object", map[string]any{"unexpected": "object-value-59317"}}, + {"array", []any{"array-value-59317"}}, } { t.Run(tc.name, func(t *testing.T) { upstream := continuationDiagnosticTestJSON(t, map[string]any{"error": map[string]any{ "type": tc.value, "code": tc.value, "param": tc.value, "message": tc.value, }}) body := []byte(`{"input":[]}`) - diagnostic := buildOpenAIContinuationDiagnostic(continuationDiagnosticTestContext(body), body, nil, body, upstream, "private-untrusted-classification-59317") - encoded, root := continuationDiagnosticTestEncoded(t, diagnostic) + diagnostic := buildOpenAIContinuationDiagnostic(continuationDiagnosticTestContext(body), body, nil, body, upstream, "untrusted-classification-59317") + _, root := continuationDiagnosticTestEncoded(t, diagnostic) if root.Get("classification").String() != "unclassified" { - t.Fatal("untrusted classification must become a fixed fallback") + t.Fatal("an unknown classification must become the fixed fallback") } - for _, field := range []string{"error_type", "error_code", "error_param", "message"} { - if root.Get("upstream_error." + field + ".value").Exists() { - t.Errorf("unknown upstream %s retained a plaintext value", field) + value, ok := tc.value.(string) + if !ok { + if root.Get("upstream_error.error_code.value").String() != string(continuationDiagnosticTestJSON(t, tc.value)) { + t.Fatal("a non-string error field must keep its JSON text") } + return } - continuationDiagnosticTestNoPlaintext(t, encoded, "private-untrusted-classification-59317", "private-object-value-59317", "private-array-value-59317") - if value, ok := tc.value.(string); ok { - continuationDiagnosticTestNoPlaintext(t, encoded, value) - if root.Get("upstream_error.error_code.bytes").Int() != int64(len(value)) || - root.Get("upstream_error.error_code.characters").Int() != int64(utf8.RuneCountInString(value)) { - t.Fatal("unknown string length evidence was lost") + if root.Get("upstream_error.error_code.bytes").Int() != int64(len(value)) || + root.Get("upstream_error.error_code.characters").Int() != int64(utf8.RuneCountInString(value)) || + root.Get("upstream_error.error_code.sha256").String() != continuationDiagnosticTestDigest(value) { + t.Fatal("fingerprint and lengths must describe the complete original value") + } + for field, limit := range map[string]int{"error_code": openAIContinuationDiagnosticCodeValueLimit, "message": openAIContinuationDiagnosticMessageValueLimit} { + kept := root.Get("upstream_error." + field + ".value").String() + truncated := root.Get("upstream_error." + field + ".truncated").Bool() + if len(value) <= limit { + if kept != value || truncated { + t.Fatalf("%s must be kept verbatim", field) + } + continue } - if root.Get("upstream_error.error_code.sha256").String() != continuationDiagnosticTestDigest(value) { - t.Fatal("unknown string fingerprint must cover its complete original value") + if !truncated || kept == "" || len(kept) > limit || !strings.HasPrefix(value, kept) || !utf8.ValidString(kept) { + t.Fatalf("%s over its bound must keep a marked, valid prefix", field) } } }) @@ -277,6 +272,18 @@ func TestOpenAIContinuationDiagnosticStructure(t *testing.T) { "wire.previous_response.sha256": continuationDiagnosticTestDigest(previous), "incoming.session.sha256": continuationDiagnosticTestDigest(incomingSession), "wire.session.sha256": continuationDiagnosticTestDigest(wireSession), + // The bounded original values sit next to their fingerprints. + "incoming.prompt_cache.value": sourceKey, + "wire.prompt_cache.value": wireKey, + "incoming.instructions.value": sourceInstructions, + "wire.instructions.value": wireInstructions, + "wire.previous_response.value": previous, + "incoming.session.value": incomingSession, + "wire.session.value": wireSession, + "incoming.conversation.value": "private-incoming-conversation-71359", + "wire.conversation.value": "private-wire-conversation-73915", + "upstream_error.message.value": "private-failure-message-71359", + "incoming.previous_response.kind": "string", } { if root.Get(path).String() != expected { t.Errorf("diagnostic contract mismatch at %s", path) @@ -304,12 +311,13 @@ func TestOpenAIContinuationDiagnosticStructure(t *testing.T) { t.Errorf("history fingerprint %s is missing or not fixed-size", field) } } - continuationDiagnosticTestNoPlaintext(t, encoded, sourceKey, wireKey, sourceInstructions, wireInstructions, - previous, incomingSession, wireSession, encrypted, compacted, output, + // History content and request headers other than session/conversation are + // summarized by counts and set fingerprints only. + continuationDiagnosticTestNoPlaintext(t, encoded, encrypted, compacted, output, "private-authentication-53197", "private-user-content-719357", "private-reference-719357", "call-private-paired-51397", "call-private-unpaired-71395", "call-private-orphan-15397", "private-function-57391", "private-arguments-19357", "private-unknown-item-type-59137", - "private-unknown-content-71359", "private-failure-message-71359") + "private-unknown-content-71359") if !bytes.Equal(incoming, incomingBefore) || !bytes.Equal(wire, wireBefore) || !bytes.Equal(prepared, preparedBefore) || !reflect.DeepEqual(c.Request.Header, incomingHeaders) || !reflect.DeepEqual(req.Header, upstreamHeaders) || liveBody.reads != 0 || liveBody.closed { @@ -397,7 +405,34 @@ func TestOpenAIContinuationDiagnosticResourceBounds(t *testing.T) { if liveBody.reads != 0 || liveBody.closed { t.Fatal("clone failure must not fall back to consuming the live request body") } - continuationDiagnosticTestNoPlaintext(t, encoded, "private-live-body-cache-57319", "private-clone-error-with-credential-71935") + // The entry snapshot keeps its value; the unreadable wire has none, and + // the clone error is not request evidence. + if root.Get("incoming.prompt_cache.value").String() != "private-live-body-cache-57319" || root.Get("wire.prompt_cache.value").Exists() { + t.Fatal("only the inspected entry body can supply values") + } + continuationDiagnosticTestNoPlaintext(t, encoded, "private-clone-error-with-credential-71935") + }) + + t.Run("oversized_values_are_truncated_not_dropped", func(t *testing.T) { + instructions := strings.Repeat("instruction-text-", 64<<10) + key := strings.Repeat("k", 64<<10) + body := continuationDiagnosticTestJSON(t, map[string]any{"instructions": instructions, "prompt_cache_key": key, "previous_response_id": key, "input": []any{}}) + upstream := continuationDiagnosticTestJSON(t, map[string]any{"error": map[string]any{"message": strings.Repeat("", 64<<10), "code": key}}) + c := continuationDiagnosticTestContext(body) + c.Request.Header.Set("session_id", key) + c.Request.Header.Set("conversation_id", key) + diagnostic := buildOpenAIContinuationDiagnostic(c, body, nil, body, upstream, "request_validation") + _, root := continuationDiagnosticTestEncoded(t, diagnostic) + value := root.Get("incoming.instructions") + if value.Get("sha256").String() != continuationDiagnosticTestDigest(instructions) || value.Get("bytes").Int() != int64(len(instructions)) { + t.Fatal("the fingerprint must still describe the complete instructions") + } + if !value.Get("truncated").Bool() || value.Get("value").String() == "" || !strings.HasPrefix(instructions, value.Get("value").String()) { + t.Fatal("oversized instructions must keep a marked prefix instead of disappearing") + } + if !root.Get("upstream_error.message.truncated").Bool() || root.Get("upstream_error.message.value").String() == "" { + t.Fatal("an oversized upstream message must keep a marked prefix") + } }) } @@ -428,25 +463,25 @@ func TestOpenAIContinuationDiagnosticOpsQueueRoundTrip(t *testing.T) { if len(events) != total { t.Fatal("diagnostics unexpectedly changed the retained event count") } - for i, event := range events { - if i < total-opsUpstreamErrorsBodyWindow { - if event.ContinuationDiagnostic != nil { - t.Fatal("old attempts retained diagnostic payloads outside the body window") - } - continue - } + // Every retained attempt keeps its diagnostic, including attempts older + // than the body window; the entry byte budget still bounds the array. + for _, event := range events { _, after := continuationDiagnosticTestEncoded(t, event.ContinuationDiagnostic) for _, path := range []string{ "classification", "wire.prompt_cache.sha256", "incoming.prompt_cache.sha256", "wire.instructions.sha256", "upstream_error.error_code.value", "upstream_error.error_param.value", + "wire.prompt_cache.value", "wire.instructions.value", "upstream_error.message.value", } { if !before.Get(path).Exists() || after.Get(path).String() != before.Get(path).String() { t.Errorf("diagnostic field %s was lost in the Ops storage roundtrip", path) } } } - continuationDiagnosticTestNoPlaintext(t, *entry.UpstreamErrorsJSON, - "private-queue-cache-57319", "private-queue-instructions-73519", "private-queue-message-57319") + for _, value := range []string{"private-queue-cache-57319", "private-queue-instructions-73519", "private-queue-message-57319"} { + if !strings.Contains(*entry.UpstreamErrorsJSON, value) { + t.Fatal("stored diagnostics must keep the bounded original values") + } + } _, stillOriginal := continuationDiagnosticTestEncoded(t, diagnostic) if stillOriginal.Raw != before.Raw { t.Fatal("queue sanitization modified the shared source diagnostic") @@ -504,8 +539,11 @@ func TestOpenAIContinuationDiagnosticRequestRejectionOpsQueueRoundTrip(t *testin t.Errorf("diagnostic field %s changed in request-rejection queue roundtrip", path) } } - continuationDiagnosticTestNoPlaintext(t, *entry.UpstreamErrorsJSON, - "private-rejection-cache-57319", "private-rejection-instructions-73519", "private-rejection-message-57319") + for _, value := range []string{"private-rejection-cache-57319", "private-rejection-instructions-73519", "private-rejection-message-57319"} { + if !strings.Contains(*entry.UpstreamErrorsJSON, value) { + t.Fatal("stored diagnostics must keep the bounded original values") + } + } stillOriginal, _ := continuationDiagnosticTestEncoded(t, diagnostic) if stillOriginal != before { t.Fatal("queue sanitization changed the shared source diagnostic") diff --git a/backend/internal/service/openai_gateway_cc_pipeline.go b/backend/internal/service/openai_gateway_cc_pipeline.go index 338ddc55c2d9..533d67190a55 100644 --- a/backend/internal/service/openai_gateway_cc_pipeline.go +++ b/backend/internal/service/openai_gateway_cc_pipeline.go @@ -107,7 +107,7 @@ func (s *OpenAIGatewayService) failoverOpenAIUpstreamHTTPError( return nil } if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( - ctx, account, resp.StatusCode, resp.Header, classificationBody, + ctx, c, account, resp.StatusCode, resp.Header, classificationBody, ); ok { return failoverErr } @@ -335,7 +335,7 @@ func (s *OpenAIGatewayService) scanCCStream( st.Usage = *u } - chunk, err := s.decodeCCStreamChunk(c, account, resp.Header, payload) + chunk, err := s.decodeCCStreamChunk(c, account, resp.Header, payload, openAIStreamClientOutputStarted(c, false)) if err != nil { st.Err = err break @@ -378,9 +378,20 @@ func ccStreamProtocolFailure(code, message string) *UpstreamFailoverError { return &UpstreamFailoverError{StatusCode: http.StatusBadGateway, ResponseBody: body, Reason: GatewayFailureReason("openai_upstream_protocol_error"), ClientStatusCode: http.StatusBadGateway, ClientErrorCode: code, ClientMessage: message} } +// ccStreamPayloadIsErrorEnvelope reports the upstream error chunk that +// decodeCCStreamChunk records in Ops. +func ccStreamPayloadIsErrorEnvelope(payload string) bool { + var envelope struct { + Error json.RawMessage `json:"error"` + } + return json.Unmarshal([]byte(payload), &envelope) == nil && len(envelope.Error) > 0 && string(envelope.Error) != "null" +} + // decodeCCStreamChunk is shared by raw forwarding and protocol conversion. // Error envelopes and malformed data must never look like empty completions. -func (s *OpenAIGatewayService) decodeCCStreamChunk(c *gin.Context, account *Account, headers http.Header, payload string) (*apicompat.ChatCompletionsChunk, error) { +// clientOutputStarted tells whether the caller can still hand an error chunk +// to failover. +func (s *OpenAIGatewayService) decodeCCStreamChunk(c *gin.Context, account *Account, headers http.Header, payload string, clientOutputStarted bool) (*apicompat.ChatCompletionsChunk, error) { var envelope struct { Error json.RawMessage `json:"error"` } @@ -396,7 +407,29 @@ func (s *OpenAIGatewayService) decodeCCStreamChunk(c *gin.Context, account *Acco failure.Scope = GatewayFailureScopeRequest failure.SuppressAccountHealthPenalty = true } - setOpsUpstreamError(c, status, "upstream_stream_error", "") + // Record the error chunk itself; the raw forwarder no longer describes + // it as a truncated stream. A chunk that moves the request to another + // account counts as a failover. + kind := "stream_error" + if !clientOutputStarted && failure.ShouldRetryNextAccount() { + kind = "failover" + } + detail := s.openAIUpstreamErrorDetail([]byte(payload)) + setOpsUpstreamError(c, status, message, detail) + event := OpsUpstreamErrorEvent{ + ProxyID: opsUpstreamProxyID(account), + ProxyName: opsUpstreamProxyName(account), + Platform: PlatformOpenAI, + UpstreamStatusCode: status, + UpstreamRequestID: headers.Get("x-request-id"), + Kind: kind, + Message: message, + Detail: detail, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) return nil, failure } var chunk apicompat.ChatCompletionsChunk diff --git a/backend/internal/service/openai_gateway_chat_completions.go b/backend/internal/service/openai_gateway_chat_completions.go index 5ce0dbbf7a15..aba2c956e6c6 100644 --- a/backend/internal/service/openai_gateway_chat_completions.go +++ b/backend/internal/service/openai_gateway_chat_completions.go @@ -907,7 +907,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse( recovery := openAIChatReasoningRecoveryFromContext(c) if recovery == nil || !recovery.RecoveryAttempt() { if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover( - c.Request.Context(), account, resp.StatusCode, resp.Header, rawPayloadBytes, + c.Request.Context(), c, account, resp.StatusCode, resp.Header, rawPayloadBytes, ); ok { if parsedUsage, parsed := extractOpenAIUsageFromJSONBytes(rawPayloadBytes); parsed { usage = parsedUsage diff --git a/backend/internal/service/openai_gateway_chat_completions_raw.go b/backend/internal/service/openai_gateway_chat_completions_raw.go index cbd0fce0942b..173c33ec4e37 100644 --- a/backend/internal/service/openai_gateway_chat_completions_raw.go +++ b/backend/internal/service/openai_gateway_chat_completions_raw.go @@ -315,6 +315,7 @@ func (s *OpenAIGatewayService) streamRawChatCompletions( clientOutputStarted := false semanticOutputReady := false var protocolErr error + upstreamErrorChunkRecorded := false pendingLines := make([]string, 0, 8) refusalDetector := newOpenAIChatSilentRefusalDetector(requestBodyLen) var terminal openAIRawStreamTerminalState @@ -370,9 +371,12 @@ func (s *OpenAIGatewayService) streamRawChatCompletions( break } if trimmedPayload != "[DONE]" { - chunk, err := s.decodeCCStreamChunk(c, account, resp.Header, trimmedPayload) + chunk, err := s.decodeCCStreamChunk(c, account, resp.Header, trimmedPayload, openAIStreamClientOutputStarted(c, clientOutputStarted)) if err != nil { protocolErr = err + // decodeCCStreamChunk already recorded an upstream error + // chunk with its own message and payload. + upstreamErrorChunkRecorded = ccStreamPayloadIsErrorEnvelope(trimmedPayload) break } if terminal.err != nil { @@ -466,7 +470,9 @@ func (s *OpenAIGatewayService) streamRawChatCompletions( ) if !clientOutputStarted { if protocolErr != nil { - recordOpenAIRawStreamTruncation(c, account, requestID, cause, "failover") + if !upstreamErrorChunkRecorded { + recordOpenAIRawStreamTruncation(c, account, requestID, cause, "failover") + } return nil, protocolErr } // 响应头尚未提交：可以透明换号重试，客户端不会看到半截流。 @@ -474,7 +480,9 @@ func (s *OpenAIGatewayService) streamRawChatCompletions( } // 已写出语义字节：无法再 failover，改为带类型的上游错误。handler 会据此 // 补发 SSE error 帧并把本次请求计入 SLA 失败。 - recordOpenAIRawStreamTruncation(c, account, requestID, cause, "http_error") + if !upstreamErrorChunkRecorded { + recordOpenAIRawStreamTruncation(c, account, requestID, cause, "http_error") + } if protocolErr != nil { // The original error can be a pre-output failover candidate. Never // expose it through errors.As once semantic output has been sent. diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index c99acbf6df56..84f1f610d977 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -861,6 +861,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco // by the WS forwarder), so a reconnect retry must not swallow them. var modelNotSupportedErr *UpstreamFailoverError if errors.As(wsErr, &modelNotSupportedErr) && modelNotSupportedErr.IsOpenAIModelNotSupported() { + s.recordOpenAIWSModelNotSupportedAttempt(c, account, modelNotSupportedErr.ResponseHeaders, modelNotSupportedErr.ResponseBody) return nil, modelNotSupportedErr } var taskRecoveredErr *agentIdentityTaskRecoveredError @@ -966,15 +967,11 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco continuationReason, _ := classifyOpenAIWSReconnectReason(wsErr) switch strings.TrimPrefix(continuationReason, "prewarm_") { case "invalid_encrypted_content", "previous_response_not_found": - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: http.StatusBadRequest, - Kind: "continuation_state", - Message: OpenAIContinuationStateUnavailableClientMessage, - }) + s.recordOpenAIWSContinuationStateError(c, account, wsErr) return nil, NewOpenAIContinuationStateUnavailableError(http.StatusBadRequest, nil, nil) + case "thinking_signature_invalid": + // Ops keeps the rejected event; the client path below is unchanged. + s.recordOpenAIWSContinuationStateError(c, account, wsErr) } s.writeOpenAIWSFallbackErrorResponse(c, account, wsErr) return nil, wsErr @@ -1104,7 +1101,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, respBody) } if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( - ctx, account, resp.StatusCode, resp.Header, respBody, + ctx, c, account, resp.StatusCode, resp.Header, respBody, ); ok { return nil, failoverErr } @@ -1126,18 +1123,10 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco // The failure describes request history, not account health. Do not // let a compatibility proxy's 4xx/5xx wrapper fan this one request // out across the remaining scheduler pool. - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: resp.StatusCode, - UpstreamRequestID: resp.Header.Get("x-request-id"), - Kind: "continuation_state", - Message: OpenAIContinuationStateUnavailableClientMessage, - ContinuationDiagnostic: buildOpenAIContinuationDiagnostic( - c, diagnosticIncomingBody, upstreamReq, body, respBody, string(continuationStateError), - ), - }) + s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, respBody, + "continuation_state", false, + buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, respBody, string(continuationStateError)), + ) return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, respBody) } if retryBody, reason, changed, retryErr := normalizeSystemPromptRejectedFieldRetryBody(c, resp.StatusCode, body, respBody); retryErr != nil { @@ -1162,18 +1151,10 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco continue } if classification := classifyOpenAIRequestRejection(resp.StatusCode, upstreamMsg, respBody); classification != "" { - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: resp.StatusCode, - UpstreamRequestID: resp.Header.Get("x-request-id"), - Kind: "request_rejected", - Message: OpenAIRequestRejectedClientMessage, - ContinuationDiagnostic: buildOpenAIContinuationDiagnostic( - c, diagnosticIncomingBody, upstreamReq, body, respBody, classification, - ), - }) + s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, respBody, + "request_rejected", false, + buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, respBody, classification), + ) return nil, NewOpenAIRequestRejectedError(resp.StatusCode, resp.Header) } shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) diff --git a/backend/internal/service/openai_gateway_messages.go b/backend/internal/service/openai_gateway_messages.go index 4727d8947744..18bb67991201 100644 --- a/backend/internal/service/openai_gateway_messages.go +++ b/backend/internal/service/openai_gateway_messages.go @@ -1056,7 +1056,7 @@ func (s *OpenAIGatewayService) handleAnthropicStreamingResponse( rawEventType := strings.TrimSpace(gjson.GetBytes(rawPayloadBytes, "type").String()) if rawEventType == "response.failed" || rawEventType == "error" { if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover( - c.Request.Context(), account, resp.StatusCode, resp.Header, rawPayloadBytes, + c.Request.Context(), c, account, resp.StatusCode, resp.Header, rawPayloadBytes, ); ok { if parsedUsage, parsed := extractOpenAIUsageFromJSONBytes(rawPayloadBytes); parsed { usage = parsedUsage diff --git a/backend/internal/service/openai_gateway_ops_visibility.go b/backend/internal/service/openai_gateway_ops_visibility.go new file mode 100644 index 000000000000..45b7465473fd --- /dev/null +++ b/backend/internal/service/openai_gateway_ops_visibility.go @@ -0,0 +1,369 @@ +package service + +import ( + "context" + "errors" + "net/http" + "strings" + "sync" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// These helpers keep the administrator-facing record of an upstream failure +// complete where a downstream client receives a fixed terminal message. They +// follow handleErrorResponse / handleErrorResponsePassthrough: the sanitized +// upstream message and the configured bounded body reach the attempt event and +// the top-level Ops fields, and the body is logged when body logging is on. + +// openAIUpstreamErrorDetail bounds an upstream error body exactly like +// handleErrorResponse (gateway.log_upstream_error_body[_max_bytes]). +func (s *OpenAIGatewayService) openAIUpstreamErrorDetail(body []byte) string { + if s == nil || s.cfg == nil || !s.cfg.Gateway.LogUpstreamErrorBody || len(body) == 0 { + return "" + } + maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes + if maxBytes <= 0 { + maxBytes = 2048 + } + return truncateString(string(body), maxBytes) +} + +// logOpenAIUpstreamErrorBody writes handleErrorResponse's body log line. +func (s *OpenAIGatewayService) logOpenAIUpstreamErrorBody(account *Account, statusCode int, body []byte) { + if s == nil || s.cfg == nil || !s.cfg.Gateway.LogUpstreamErrorBody || account == nil { + return + } + logger.LegacyPrintf("service.openai_gateway", + "OpenAI upstream error %d (account=%d platform=%s type=%s): %s", + statusCode, + account.ID, + account.Platform, + account.Type, + truncateForLog(body, s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes), + ) +} + +// openAIOpsUpstreamErrorMessage extracts the upstream message the way +// handleErrorResponse does and also reads Responses envelopes +// (response.error.message) that that extractor does not cover. +func openAIOpsUpstreamErrorMessage(body []byte) string { + if message := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(body))); message != "" { + return message + } + return extractOpenAISSEErrorMessage(body) +} + +// recordOpenAIRequestTerminalUpstreamError records a request-scoped terminal +// (continuation state or request rejection) with the upstream's own message +// and bounded body. Only the client response is replaced by the fixed message. +func (s *OpenAIGatewayService) recordOpenAIRequestTerminalUpstreamError( + ctx context.Context, + c *gin.Context, + account *Account, + statusCode int, + headers http.Header, + requestBody []byte, + body []byte, + kind string, + passthrough bool, + diagnostic *OpenAIContinuationDiagnostic, +) { + upstreamMsg := openAIOpsUpstreamErrorMessage(body) + upstreamDetail := s.openAIUpstreamErrorDetail(body) + setOpsUpstreamError(c, statusCode, upstreamMsg, upstreamDetail) + logOpenAIInstructionsRequiredDebug(ctx, c, account, statusCode, upstreamMsg, requestBody, body) + s.logOpenAIUpstreamErrorBody(account, statusCode, body) + event := OpsUpstreamErrorEvent{ + ProxyID: opsUpstreamProxyID(account), + ProxyName: opsUpstreamProxyName(account), + UpstreamStatusCode: statusCode, + UpstreamRequestID: headers.Get("x-request-id"), + Passthrough: passthrough, + Kind: kind, + Message: upstreamMsg, + Detail: upstreamDetail, + ContinuationDiagnostic: diagnostic, + } + if passthrough { + event.UpstreamResponseBody = upstreamDetail + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +// OpenAIFailoverUpstreamMessage returns the upstream's own error text carried by +// a failover terminal (an HTTP error body or a Responses/WebSocket event), +// sanitized like every other Ops upstream message. Handlers record it for +// administrators; it is never written to the client from here. +func OpenAIFailoverUpstreamMessage(failoverErr *UpstreamFailoverError) string { + if failoverErr == nil || len(failoverErr.ResponseBody) == 0 { + return "" + } + return extractOpenAISSEErrorMessage(failoverErr.ResponseBody) +} + +// OpenAIModelNotSupportedNoAccountOpsMessage describes, for Ops only, why a +// request that reached no upstream still ends as model_not_supported. +const OpenAIModelNotSupportedNoAccountOpsMessage = "No upstream request was sent: every account that supports this model is in the model_not_supported cooldown set by an earlier upstream 400 response; see the account's model rate limits and the earlier Ops attempt for the upstream text." + +// openAIWSUpstreamEventError carries the upstream WebSocket event that ended an +// attempt. Error() is the upstream message, which is what fallback handling +// already used; the payload is kept only for Ops. +type openAIWSUpstreamEventError struct { + message string + payload []byte +} + +func newOpenAIWSUpstreamEventError(message string, payload []byte) *openAIWSUpstreamEventError { + return &openAIWSUpstreamEventError{message: message, payload: append([]byte(nil), payload...)} +} + +func (e *openAIWSUpstreamEventError) Error() string { + if e == nil { + return "" + } + return e.message +} + +// recordOpenAIWSContinuationStateError records the rejected WebSocket event: +// the upstream's own message from the event (or the handshake response), the +// bounded payload, and the status writeOpenAIWSFallbackErrorResponse uses. +func (s *OpenAIGatewayService) recordOpenAIWSContinuationStateError(c *gin.Context, account *Account, wsErr error) { + statusCode, _, _, fallbackMessage, ok := resolveOpenAIWSFallbackErrorResponse(wsErr) + if !ok || statusCode <= 0 { + statusCode = http.StatusBadRequest + } + upstreamMessage := "" + var payload []byte + var eventErr *openAIWSUpstreamEventError + var dialErr *openAIWSDialError + if errors.As(wsErr, &eventErr) && eventErr != nil { + payload = eventErr.payload + upstreamMessage = extractOpenAISSEErrorMessage(payload) + if upstreamMessage == "" { + upstreamMessage = sanitizeUpstreamErrorMessage(strings.TrimSpace(eventErr.message)) + } + } else if errors.As(wsErr, &dialErr) && dialErr != nil { + payload = dialErr.ResponseBody + upstreamMessage = extractOpenAISSEErrorMessage(payload) + } + // resolveOpenAIWSFallbackErrorResponse falls back to fixed defaults; use + // them only when the upstream supplied no text of its own. + if upstreamMessage == "" { + upstreamMessage = strings.TrimSpace(fallbackMessage) + } + if upstreamMessage == "" && wsErr != nil { + upstreamMessage = sanitizeUpstreamErrorMessage(strings.TrimSpace(wsErr.Error())) + } + detail := s.openAIUpstreamErrorDetail(payload) + setOpsUpstreamError(c, statusCode, upstreamMessage, detail) + proxyID, proxyName := opsUpstreamWSProxyAttribution(account) + event := OpsUpstreamErrorEvent{ + ProxyID: proxyID, + ProxyName: proxyName, + UpstreamStatusCode: statusCode, + Kind: "continuation_state", + Message: upstreamMessage, + Detail: detail, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +// recordOpenAIModelNotSupportedAttempt keeps the upstream's own model +// capability rejection on the attempt; the client still receives the fixed +// model_not_supported terminal once every account has refused the model. +func (s *OpenAIGatewayService) recordOpenAIModelNotSupportedAttempt(c *gin.Context, account *Account, headers http.Header, payload []byte, passthrough bool) { + upstreamMsg := extractOpenAISSEErrorMessage(payload) + upstreamDetail := s.openAIUpstreamErrorDetail(payload) + setOpsUpstreamError(c, http.StatusBadRequest, upstreamMsg, upstreamDetail) + event := OpsUpstreamErrorEvent{ + ProxyID: opsUpstreamProxyID(account), + ProxyName: opsUpstreamProxyName(account), + UpstreamStatusCode: http.StatusBadRequest, + UpstreamRequestID: headers.Get("x-request-id"), + Passthrough: passthrough, + Kind: "failover", + Reason: string(openAIModelNotSupportedReason), + Message: upstreamMsg, + Detail: upstreamDetail, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +// recordOpenAIWSModelNotSupportedAttempt is the WebSocket variant; a missing +// managed proxy on a WS dial is unknown rather than direct. +func (s *OpenAIGatewayService) recordOpenAIWSModelNotSupportedAttempt(c *gin.Context, account *Account, headers http.Header, payload []byte) { + upstreamMsg := extractOpenAISSEErrorMessage(payload) + upstreamDetail := s.openAIUpstreamErrorDetail(payload) + setOpsUpstreamError(c, http.StatusBadRequest, upstreamMsg, upstreamDetail) + proxyID, proxyName := opsUpstreamWSProxyAttribution(account) + event := OpsUpstreamErrorEvent{ + ProxyID: proxyID, + ProxyName: proxyName, + UpstreamStatusCode: http.StatusBadRequest, + UpstreamRequestID: headers.Get("x-request-id"), + Kind: "failover", + Reason: string(openAIModelNotSupportedReason), + Message: upstreamMsg, + Detail: upstreamDetail, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +// openAIWSDeliveryMarksFailuresKey is set while a WebSocket refusal output +// marks delivered failures itself, so writers above it do not mark frames the +// output may still hold back or replace. +const openAIWSDeliveryMarksFailuresKey = "openai_ws_delivery_marks_failures" + +func openAIWSDeliveryMarksFailures(c *gin.Context) bool { + return c != nil && c.GetBool(openAIWSDeliveryMarksFailuresKey) +} + +// markOpenAIWSDeliveredFailure marks an error or response.failed event the +// client received, as upstream v0.2.8 does, except a cyber-policy block: that +// already gets its own dedicated Ops row. +func markOpenAIWSDeliveredFailure(c *gin.Context, payload []byte) { + eventType, _, _ := parseOpenAIWSEventEnvelope(payload) + if eventType != "error" && eventType != "response.failed" { + return + } + if hit, _, _ := detectOpenAICyberPolicy(payload); hit { + return + } + markOpenAIWSClientVisibleFailure(c, eventType, payload) +} + +// recordOpenAIWSBridgeTransportError keeps the transport error behind the +// fixed "Upstream request failed" event a later bridge turn sends the client. +func recordOpenAIWSBridgeTransportError(c *gin.Context, account *Account, safeErr string) { + setOpsUpstreamError(c, 0, safeErr, "") + event := OpsUpstreamErrorEvent{ + ProxyID: opsUpstreamProxyID(account), + ProxyName: opsUpstreamProxyName(account), + Passthrough: true, + Kind: "request_error", + Message: safeErr, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsUpstreamError(c, event) +} + +// OpenAILastUpstreamErrorExtraKey keeps the most recent upstream 401/403/429 that +// the request failover state handled without persisting account error state. +// Administrators read it; scheduling never consults it. +const OpenAILastUpstreamErrorExtraKey = "last_upstream_error" + +const ( + openAILastUpstreamErrorMaxBytes = 8 << 10 + // One write per account and status inside this window, counted from the + // last successful write, so a burst of rejections costs one account write. + openAILastUpstreamErrorRepeatWindow = 30 * time.Second + // Background writes in flight at once; beyond this a record is dropped. + openAILastUpstreamErrorMaxInflight = 16 +) + +type openAILastUpstreamErrorKey struct { + accountID int64 + status int +} + +var ( + openAILastUpstreamErrorWritten sync.Map // openAILastUpstreamErrorKey -> time.Time of the last successful write + openAILastUpstreamErrorInflight sync.Map // openAILastUpstreamErrorKey -> struct{} + openAILastUpstreamErrorSlots = make(chan struct{}, openAILastUpstreamErrorMaxInflight) +) + +// recordOpenAILastUpstreamError stores the upstream's verbatim error body in +// accounts.extra.last_upstream_error = {status, message, at, source}. The +// write runs in the background and is best effort: failover never waits for +// it, and a record is dropped while the same account and status is being +// written, was written inside the repeat window, or too many writes are in +// flight. It does not change status, schedulability or any cooldown. +func (s *OpenAIGatewayService) recordOpenAILastUpstreamError(ctx context.Context, account *Account, statusCode int, body []byte) { + if s == nil || s.accountRepo == nil || account == nil || account.ID <= 0 { + return + } + key := openAILastUpstreamErrorKey{accountID: account.ID, status: statusCode} + now := time.Now() + if last, ok := openAILastUpstreamErrorWritten.Load(key); ok { + if at, ok := last.(time.Time); ok && now.Sub(at) < openAILastUpstreamErrorRepeatWindow { + return + } + } + if _, busy := openAILastUpstreamErrorInflight.LoadOrStore(key, struct{}{}); busy { + return + } + select { + case openAILastUpstreamErrorSlots <- struct{}{}: + default: + openAILastUpstreamErrorInflight.Delete(key) + return + } + // Detach everything the write needs from the request. A shadow account's + // credential owner is resolved in the background, never on this path. + body = append([]byte(nil), body...) + var shadow *Account + if account.IsShadow() { + parentID := *account.ParentAccountID + shadow = &Account{ID: account.ID, Platform: account.Platform, Type: account.Type, ParentAccountID: &parentID} + } else { + body = redactAgentIdentitySensitiveBodyForAccount(ctx, nil, account, body) + } + repo := s.accountRepo + stateCtx, cancel := openAIAccountStateContext(ctx) + go func() { + defer func() { + cancel() + openAILastUpstreamErrorInflight.Delete(key) + <-openAILastUpstreamErrorSlots + }() + // A best-effort record must never take the process down. + defer func() { + if recovered := recover(); recovered != nil { + logger.FromContext(stateCtx).Error("openai.last_upstream_error_panic", + zap.Int64("account_id", key.accountID), + zap.Any("recover", recovered), + ) + } + }() + if shadow != nil { + body = redactAgentIdentitySensitiveBodyForAccount(stateCtx, repo, shadow, body) + } + message := strings.TrimSpace(strings.ToValidUTF8(string(body), "\uFFFD")) + if message == "" { + message = http.StatusText(statusCode) + } + value := map[string]any{ + "status": statusCode, + "message": truncateString(message, openAILastUpstreamErrorMaxBytes), + "at": now.UTC().Format(time.RFC3339), + "source": "gateway", + } + if err := repo.UpdateExtra(stateCtx, key.accountID, map[string]any{OpenAILastUpstreamErrorExtraKey: value}); err != nil { + logger.FromContext(stateCtx).Warn("openai.last_upstream_error_persist_failed", + zap.Int64("account_id", key.accountID), + zap.Int("upstream_status", statusCode), + zap.Error(err), + ) + return + } + openAILastUpstreamErrorWritten.Store(key, time.Now()) + }() +} diff --git a/backend/internal/service/openai_gateway_ops_visibility_test.go b/backend/internal/service/openai_gateway_ops_visibility_test.go new file mode 100644 index 000000000000..78822ddf68b0 --- /dev/null +++ b/backend/internal/service/openai_gateway_ops_visibility_test.go @@ -0,0 +1,309 @@ +//go:build unit + +package service + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func opsVisibilityTestContext() *gin.Context { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + return c +} + +func opsVisibilityTestEvents(t *testing.T, c *gin.Context) []*OpsUpstreamErrorEvent { + t.Helper() + value, ok := c.Get(OpsUpstreamErrorsKey) + require.True(t, ok) + events, ok := value.([]*OpsUpstreamErrorEvent) + require.True(t, ok) + return events +} + +func opsVisibilityTestConfig() *config.Config { + cfg := &config.Config{} + cfg.Gateway.LogUpstreamErrorBody = true + cfg.Gateway.LogUpstreamErrorBodyMaxBytes = 2048 + return cfg +} + +// OPS-8: a budget_exceeded 429 is recorded like any other upstream failover +// attempt, and the account error keeps the structured type and code. +func TestOpenAIBudgetExceededHTTPFailoverRecordsUpstreamAttempt(t *testing.T) { + const body = `{"error":{"message":"ExceededBudget: key over budget","type":"budget_exceeded","param":null,"code":"429"}}` + cfg := opsVisibilityTestConfig() + repo := &budgetExceededAccountRepoStub{} + svc := &OpenAIGatewayService{cfg: cfg, rateLimitService: NewRateLimitService(repo, nil, cfg, nil, nil)} + c := opsVisibilityTestContext() + + failoverErr, ok := svc.handleOpenAIBudgetExceededHTTPFailover(context.Background(), c, newBudgetRelayAccount(97101, false), + http.StatusTooManyRequests, http.Header{"X-Request-Id": []string{"req_budget"}}, []byte(body)) + + require.True(t, ok) + require.Equal(t, NextAccountRetry, failoverErr.NextAccountAction) + events := opsVisibilityTestEvents(t, c) + require.Len(t, events, 1) + require.Equal(t, "failover", events[0].Kind) + require.Equal(t, openAIBudgetExceededKind, events[0].Reason) + require.Equal(t, http.StatusTooManyRequests, events[0].UpstreamStatusCode) + require.Equal(t, "req_budget", events[0].UpstreamRequestID) + require.Equal(t, "ExceededBudget: key over budget", events[0].Message) + require.Equal(t, body, events[0].Detail) + require.Equal(t, "ExceededBudget: key over budget (type=budget_exceeded, code=429)", repo.lastErrorMsg) +} + +type lastUpstreamErrorAccountRepo struct { + stubOpenAIAccountRepo + mu sync.Mutex + updates []map[string]any +} + +func (r *lastUpstreamErrorAccountRepo) UpdateExtra(_ context.Context, _ int64, updates map[string]any) error { + r.mu.Lock() + defer r.mu.Unlock() + r.updates = append(r.updates, updates) + return nil +} + +// OPS-9: the upstream 401/403/429 handled only by the request failover state is +// kept verbatim in extra.last_upstream_error; no account state changes. +func TestOpenAIAPIKeyAuthAndQuotaErrorsRecordLastUpstreamError(t *testing.T) { + for index, statusCode := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusTooManyRequests} { + t.Run(http.StatusText(statusCode), func(t *testing.T) { + body := `{"error":{"message":"Request blocked by the relay policy","type":"requests","code":"relay_blocked"}}` + repo := &lastUpstreamErrorAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ID: int64(97201 + index), Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Status: StatusActive, Schedulable: true} + key := openAILastUpstreamErrorKey{accountID: account.ID, status: statusCode} + openAILastUpstreamErrorWritten.Delete(key) + + require.False(t, svc.handleOpenAIAccountUpstreamError(context.Background(), account, statusCode, http.Header{}, []byte(body))) + require.Eventually(t, func() bool { + _, written := openAILastUpstreamErrorWritten.Load(key) + return written + }, 5*time.Second, 5*time.Millisecond, "the background write completes") + // Same account and status inside the window: skipped whatever the body. + require.False(t, svc.handleOpenAIAccountUpstreamError(context.Background(), account, statusCode, http.Header{}, []byte(`{"error":{"message":"different text"}}`))) + _, inflight := openAILastUpstreamErrorInflight.Load(key) + require.False(t, inflight) + + repo.mu.Lock() + defer repo.mu.Unlock() + require.Len(t, repo.updates, 1, "one write per account and status inside the window") + record, ok := repo.updates[0][OpenAILastUpstreamErrorExtraKey].(map[string]any) + require.True(t, ok) + require.Equal(t, statusCode, record["status"]) + require.Equal(t, body, record["message"]) + require.Equal(t, "gateway", record["source"]) + _, err := time.Parse(time.RFC3339, record["at"].(string)) + require.NoError(t, err) + require.Equal(t, StatusActive, account.Status) + require.True(t, account.Schedulable) + require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) + }) + } +} + +// OPS-5: an upstream error chunk inside a Chat Completions stream is recorded +// with its own message and payload instead of a truncation description. +func TestForwardAsRawChatCompletionsRecordsUpstreamErrorChunk(t *testing.T) { + for _, tc := range []struct { + name string + payload string + message string + kind string + }{ + // The chunk ends this request: no other account is tried. + {"request_scoped", `{"error":{"message":"upstream model crashed","type":"server_error","code":"internal"}}`, "upstream model crashed", "stream_error"}, + // Nothing reached the client and the chunk moves the request on. + {"switches_account", `{"error":{"message":"Rate limit reached","type":"rate_limit_error","code":"rate_limit_exceeded"}}`, "Rate limit reached", "failover"}, + } { + t.Run(tc.name, func(t *testing.T) { + c := opsVisibilityTestContext() + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{}`)) + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}, "X-Request-Id": []string{"req_chunk"}}, + Body: io.NopCloser(strings.NewReader("data: " + tc.payload + "\n\n")), + }} + cfg := rawChatCompletionsTestConfig() + cfg.Gateway.LogUpstreamErrorBody = true + cfg.Gateway.LogUpstreamErrorBodyMaxBytes = 2048 + svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"hi"}],"stream":true}`) + + _, err := svc.forwardAsRawChatCompletions(context.Background(), c, rawChatCompletionsTestAccount(), body, "") + + require.Error(t, err) + events := opsVisibilityTestEvents(t, c) + require.Len(t, events, 1, "the error chunk is not re-described as a truncated stream") + require.Equal(t, tc.kind, events[0].Kind) + require.Equal(t, tc.message, events[0].Message) + require.Equal(t, tc.payload, events[0].Detail) + require.Equal(t, "req_chunk", events[0].UpstreamRequestID) + message, _ := c.Get(OpsUpstreamErrorMessageKey) + require.Equal(t, tc.message, message) + }) + } +} + +// OPS-6: the upstream's own model_not_supported text reaches the attempt. +func TestOpenAIModelNotSupportedAttemptKeepsUpstreamText(t *testing.T) { + c := opsVisibilityTestContext() + payload := []byte(`{"error":{"type":"model_not_supported","code":"400","message":"The model 'gpt-x' is not supported for this key"}}`) + svc := &OpenAIGatewayService{cfg: opsVisibilityTestConfig()} + + svc.recordOpenAIModelNotSupportedAttempt(c, &Account{ID: 7, Name: "relay", Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, http.Header{}, payload, false) + + events := opsVisibilityTestEvents(t, c) + require.Len(t, events, 1) + require.Equal(t, string(openAIModelNotSupportedReason), events[0].Reason) + require.Equal(t, "The model 'gpt-x' is not supported for this key", events[0].Message) + require.Equal(t, string(payload), events[0].Detail) +} + +// OPS-7: a refusal rewrite or prompt retry records the matched keyword and the +// original leading text for administrators. +func TestOpenAIRefusalRecoveryRecordsKeywordAndOriginalText(t *testing.T) { + matcher, err := NewOpenAIRefusalMatcher([]string{"I cannot"}, "继续当前任务") + require.NoError(t, err) + body := []byte(`{"id":"resp_1","status":"completed","output":[{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"output_text","text":"I cannot help with that request."}]}]}`) + _, matched, evidence, err := rewriteOpenAIResponsesJSONWithEvidence(body, matcher) + require.NoError(t, err) + require.True(t, matched) + require.Equal(t, "I cannot", evidence.Keyword) + require.Equal(t, "I cannot help with that request.", evidence.Text) + + c := opsVisibilityTestContext() + recordOpenAIRefusalRecovery(context.Background(), c, &Account{ID: 8, Platform: PlatformOpenAI}, "http", false, false, openAIRefusalActionPromptRetry, evidence) + // A handled refusal is not upstream error context: a request that then + // succeeds leaves no Ops row, and nothing classifies on it. + _, recordedAsUpstream := c.Get(OpsUpstreamErrorsKey) + require.False(t, recordedAsUpstream) + events := OpsRefusalRecoveryEvents(c) + require.Len(t, events, 1) + require.Equal(t, "refusal_recovery", events[0].Kind) + require.Equal(t, openAIRefusalActionPromptRetry, events[0].Reason) + require.Contains(t, events[0].Message, `"I cannot"`) + require.Equal(t, "I cannot help with that request.", events[0].Detail) + + // WebSocket refusals are logged only. + wsContext := opsVisibilityTestContext() + recordOpenAIRefusalRecovery(context.Background(), wsContext, &Account{ID: 8, Platform: PlatformOpenAI}, "websocket", true, false, openAIRefusalActionRewritten, evidence) + _, recordedAsUpstream = wsContext.Get(OpsUpstreamErrorsKey) + require.False(t, recordedAsUpstream) + require.Empty(t, OpsRefusalRecoveryEvents(wsContext)) +} + +// OPS-11: image transport failures keep the upstream endpoint on the attempt. +func TestOpenAIImagesTransportErrorKeepsUpstreamURL(t *testing.T) { + c := opsVisibilityTestContext() + svc := &OpenAIGatewayService{} + account := &Account{ID: 9, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} + + err := svc.handleOpenAIImagesUpstreamTransportError(context.Background(), c, account, errors.New("dial tcp: connection refused"), + "https://api.example.test/v1/images/generations?api-version=1") + + require.Error(t, err) + events := opsVisibilityTestEvents(t, c) + require.Len(t, events, 1) + require.Equal(t, "request_error", events[0].Kind) + require.Equal(t, "https://api.example.test/v1/images/generations", events[0].UpstreamURL) +} + +// C-13: the model's complete words behind an image failure reach Ops and the +// connection test; the client keeps the bounded text it always received. +func TestOpenAIImagesModelTextKeepsFullTextForAdministrators(t *testing.T) { + text := "Blocked by our content policy. " + strings.Repeat("详细原因 ", 300) + body := []byte("data: {\"type\":\"response.output_text.delta\",\"delta\":\"" + text + "\"}\n\n") + + require.Equal(t, strings.TrimSpace(text), extractOpenAIImagesModelText(body)) + upstreamErr := openAIImagesTextFallbackError(body) + require.NotNil(t, upstreamErr) + require.Equal(t, "content_policy_violation", upstreamErr.Code) + // The client text is computed exactly as before: trim, cut at 600 bytes, + // trim again. + legacy := strings.TrimSpace(text)[:600] + require.Equal(t, sanitizeUpstreamErrorMessage(strings.TrimSpace(legacy)), upstreamErr.Message) + require.Equal(t, strings.TrimSpace(text), upstreamErr.ModelText) + require.Equal(t, strings.TrimSpace(text), upstreamErr.opsMessage()) + + // Streamed text keeps its raw first 600 bytes, leading whitespace included. + streamed := "\n\n " + text + streamErr := openAIImagesTextFallbackErrorForWindow(openAIImagesModelTextClientWindow(streamed), streamed) + require.NotNil(t, streamErr) + require.Equal(t, sanitizeUpstreamErrorMessage(strings.TrimSpace(streamed[:600])), streamErr.Message) + require.Equal(t, strings.TrimSpace(streamed), streamErr.ModelText) +} + +// OPS-10: the reasoning-recovery detail is bounded by its encoded size, so the +// Ops queue never shrinks it to an object without its fields. +func TestOpenAIReasoningRecoveryDetailFitsOpsBudget(t *testing.T) { + payload := []byte(`{"type":"error","error":{"code":"invalid_encrypted_content","message":"` + strings.Repeat("", 2000) + `"}}`) + detail := map[string]any{"action": "retry_without_encrypted_content", "items": 3, "usage_status": "unavailable", "payload_sha256": openAIReasoningDigest(payload)} + + encoded := encodeOpenAIReasoningRecoveryDetail(detail, extractOpenAISSEErrorMessage(payload), payload) + + require.LessOrEqual(t, len(encoded), openAIReasoningRecoveryDetailEncodedLimit) + stored, _ := sanitizeErrorBodyForStorage(string(encoded), OpsErrorLogQueueBodyMaxBytes) + for _, key := range []string{"action", "items", "usage_status", "payload_sha256", "upstream_message", "payload"} { + require.True(t, gjson.Get(stored, key).Exists(), key) + } + require.True(t, gjson.Get(stored, "payload_truncated").Bool()) +} + +// OPS-4: a delivered cyber-policy block keeps only its dedicated row; other +// delivered failures are marked as client-visible. +func TestOpenAIWSDeliveredFailureSkipsCyberPolicy(t *testing.T) { + for _, payload := range [][]byte{ + []byte(`{"type":"error","error":{"type":"invalid_request_error","code":"cyber_policy","message":"blocked"}}`), + []byte(`{"type":"response.failed","response":{"error":{"code":"cyber_policy","message":"blocked"}}}`), + } { + c := opsVisibilityTestContext() + markOpenAIWSDeliveredFailure(c, payload) + require.Empty(t, GetOpsStreamErrors(c)) + } + c := opsVisibilityTestContext() + markOpenAIWSDeliveredFailure(c, []byte(`{"type":"error","error":{"type":"server_error","code":"server_error","message":"boom"}}`)) + require.Len(t, GetOpsStreamErrors(c), 1) +} + +// A budget terminal counts as a failover only while the request can still move +// to another account. +func TestOpenAIBudgetExceededTerminalKindFollowsReplayability(t *testing.T) { + payload := []byte(`{"type":"response.failed","response":{"error":{"type":"budget_exceeded","message":"ExceededBudget: key over budget"}}}`) + cfg := opsVisibilityTestConfig() + svc := &OpenAIGatewayService{cfg: cfg, rateLimitService: NewRateLimitService(&budgetExceededAccountRepoStub{}, nil, cfg, nil, nil)} + for _, tc := range []struct { + name string + started bool + kind string + }{{"before_output", false, "failover"}, {"after_output", true, "stream_error"}} { + t.Run(tc.name, func(t *testing.T) { + c := opsVisibilityTestContext() + if tc.started { + c.Writer.WriteHeaderNow() + } + _, ok := svc.openAIBudgetExceededHTTPResponseTerminalFailover(context.Background(), c, newBudgetRelayAccount(97301, false), http.StatusOK, http.Header{}, payload) + require.True(t, ok) + events := opsVisibilityTestEvents(t, c) + require.Len(t, events, 1) + require.Equal(t, tc.kind, events[0].Kind) + }) + } +} diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 5a6d482e1b47..91aea59ade62 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -448,7 +448,7 @@ retryUpstream: return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, probeBody) } if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( - ctx, account, resp.StatusCode, resp.Header, probeBody, + ctx, c, account, resp.StatusCode, resp.Header, probeBody, ); ok { return nil, failoverErr } @@ -471,19 +471,10 @@ retryUpstream: // health. Classify them before agent recovery, failover, response // commitment, or any scheduler/rate-limit side effect. redactedBody := s.redactAgentIdentitySensitiveBody(ctx, account, probeBody) - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: resp.StatusCode, - UpstreamRequestID: resp.Header.Get("x-request-id"), - Passthrough: true, - Kind: "continuation_state", - Message: OpenAIContinuationStateUnavailableClientMessage, - ContinuationDiagnostic: buildOpenAIContinuationDiagnostic( - c, diagnosticIncomingBody, upstreamReq, body, probeBody, string(continuationStateError), - ), - }) + s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, redactedBody, + "continuation_state", true, + buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, redactedBody, string(continuationStateError)), + ) return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, redactedBody) } if !agentTaskRecoveryTried && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, probeBody) { @@ -496,19 +487,11 @@ retryUpstream: } if classification := classifyOpenAIRequestRejection(resp.StatusCode, upstreamMsg, probeBody); classification != "" { - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: resp.StatusCode, - UpstreamRequestID: resp.Header.Get("x-request-id"), - Passthrough: true, - Kind: "request_rejected", - Message: OpenAIRequestRejectedClientMessage, - ContinuationDiagnostic: buildOpenAIContinuationDiagnostic( - c, diagnosticIncomingBody, upstreamReq, body, probeBody, classification, - ), - }) + redactedBody := s.redactAgentIdentitySensitiveBody(ctx, account, probeBody) + s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, redactedBody, + "request_rejected", true, + buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, redactedBody, classification), + ) return nil, NewOpenAIRequestRejectedError(resp.StatusCode, resp.Header) } @@ -2287,7 +2270,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( pendingErrorEventHeader = false continue } - if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, account, resp.StatusCode, resp.Header, rawDataBytes); ok { + if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, rawDataBytes); ok { // Classification must happen on the untouched event, before error // rules, namespace restoration or the client stream see it. The // handler can switch accounts while pendingLines are still @@ -2542,6 +2525,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( continue case openAIRefusalStreamReplace: if openAIRefusalShouldPromptRetry(c, refusalRuntime) { + recordOpenAIRefusalRecovery(ctx, c, account, "sse", false, refusalEarlyEmitted, openAIRefusalActionPromptRetry, refusalStream.evidence()) return resultWithUsage(), NewOpenAIRefusalRecoveryFailoverError(resp.Header) } pendingLines = pendingLines[:0] @@ -2559,7 +2543,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( firstTokenMs = &ms } s.clearOpenAIProxyStreamDisconnect(account, ctx) - logger.FromContext(ctx).Info("openai.refusal_recovery_rewritten", zap.String("transport", "sse"), zap.Bool("early", refusalEarlyEmitted)) + recordOpenAIRefusalRecovery(ctx, c, account, "sse", false, refusalEarlyEmitted, openAIRefusalActionRewritten, refusalStream.evidence()) return resultWithUsage(), nil case openAIRefusalStreamHold: lineStartsClientOutput = false @@ -2703,10 +2687,10 @@ func (s *OpenAIGatewayService) handleNonStreamingResponsePassthrough( observer.ObserveOpenAI(body, strings.TrimSpace(gjson.GetBytes(body, "type").String())) } if account != nil { - failoverErr, budgetExceeded := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, account, resp.StatusCode, resp.Header, body) + failoverErr, budgetExceeded := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, body) forEachOpenAISSEDataPayload(string(body), func(payload []byte) { if !budgetExceeded { - failoverErr, budgetExceeded = s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, account, resp.StatusCode, resp.Header, payload) + failoverErr, budgetExceeded = s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, payload) } }) if budgetExceeded { @@ -2719,6 +2703,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponsePassthrough( model = canonicalOpenAIAccountSchedulingModel(account, originalModel) } _ = s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusBadRequest, resp.Header, body, model) + s.recordOpenAIModelNotSupportedAttempt(c, account, resp.Header, body, true) return nil, newOpenAIModelNotSupportedFailoverError(resp.Header, body) } body = restoreSystemPromptEcho(c, body) @@ -2774,14 +2759,15 @@ func (s *OpenAIGatewayService) handleNonStreamingResponsePassthrough( if !cyberHit && isOpenAIRefusalRecoveryResponsesRequest(c) { runtime := s.openAIRefusalRecoveryRuntime(ctx) if runtime.RewriteEnabled() { - if rewritten, matched, _, rewriteErr := RewriteOpenAIResponsesJSON(body, runtime.Matcher); rewriteErr != nil { + if rewritten, matched, evidence, rewriteErr := rewriteOpenAIResponsesJSONWithEvidence(body, runtime.Matcher); rewriteErr != nil { logger.FromContext(ctx).Warn("openai.refusal_recovery_rewrite_failed", zap.String("transport", "http"), zap.Error(rewriteErr)) } else if matched { if openAIRefusalShouldPromptRetry(c, runtime) { + recordOpenAIRefusalRecovery(ctx, c, account, "http", false, false, openAIRefusalActionPromptRetry, evidence) return nil, NewOpenAIRefusalRecoveryFailoverError(resp.Header) } body = rewritten - logger.FromContext(ctx).Info("openai.refusal_recovery_rewritten", zap.String("transport", "http")) + recordOpenAIRefusalRecovery(ctx, c, account, "http", false, false, openAIRefusalActionRewritten, evidence) } } } @@ -2867,14 +2853,15 @@ func (s *OpenAIGatewayService) handlePassthroughSSEToJSON(resp *http.Response, c if !cyberHit && isOpenAIRefusalRecoveryResponsesRequest(c) { runtime := s.openAIRefusalRecoveryRuntime(c.Request.Context()) if runtime.RewriteEnabled() { - if rewritten, matched, _, rewriteErr := RewriteOpenAIResponsesJSON(body, runtime.Matcher); rewriteErr != nil { + if rewritten, matched, evidence, rewriteErr := rewriteOpenAIResponsesJSONWithEvidence(body, runtime.Matcher); rewriteErr != nil { logger.FromContext(c.Request.Context()).Warn("openai.refusal_recovery_rewrite_failed", zap.String("transport", "http_sse_to_json"), zap.Error(rewriteErr)) } else if matched { if openAIRefusalShouldPromptRetry(c, runtime) { + recordOpenAIRefusalRecovery(c.Request.Context(), c, account, "http_sse_to_json", false, false, openAIRefusalActionPromptRetry, evidence) return nil, NewOpenAIRefusalRecoveryFailoverError(resp.Header) } body = rewritten - logger.FromContext(c.Request.Context()).Info("openai.refusal_recovery_rewritten", zap.String("transport", "http_sse_to_json")) + recordOpenAIRefusalRecovery(c.Request.Context(), c, account, "http_sse_to_json", false, false, openAIRefusalActionRewritten, evidence) } } } diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index a021eaba7c83..313730e086a8 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -670,7 +670,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. return } if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover( - ctx, account, resp.StatusCode, resp.Header, rawDataBytes, + ctx, c, account, resp.StatusCode, resp.Header, rawDataBytes, ); ok { s.parseSSEUsageBytes(rawDataBytes, usage) sawFailedEvent = true @@ -995,6 +995,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } if refusalAction == openAIRefusalStreamReplace { if openAIRefusalShouldPromptRetry(c, refusalRuntime) { + recordOpenAIRefusalRecovery(ctx, c, account, "sse", false, refusalEarlyEmitted, openAIRefusalActionPromptRetry, refusalStream.evidence()) streamEarlyErr = NewOpenAIRefusalRecoveryFailoverError(resp.Header) return } @@ -1019,7 +1020,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. firstOutputScanGuard.Store(false) stopFirstOutputTimer() } - logger.FromContext(ctx).Info("openai.refusal_recovery_rewritten", zap.String("transport", "sse"), zap.Bool("early", refusalEarlyEmitted)) + recordOpenAIRefusalRecovery(ctx, c, account, "sse", false, refusalEarlyEmitted, openAIRefusalActionRewritten, refusalStream.evidence()) return } startsClientOutput := forceFlushFailedEvent || openAIStreamDataStartsClientOutput(data, eventType) @@ -1796,10 +1797,10 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r // Classify the untouched upstream payload before any response rewriting. // A non-stream request may still receive a JSON or SSE terminal event with // HTTP 200. - budgetFailoverErr, budgetExceeded := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, account, resp.StatusCode, resp.Header, body) + budgetFailoverErr, budgetExceeded := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, body) forEachOpenAISSEDataPayload(string(body), func(payload []byte) { if !budgetExceeded { - budgetFailoverErr, budgetExceeded = s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, account, resp.StatusCode, resp.Header, payload) + budgetFailoverErr, budgetExceeded = s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, payload) } }) if budgetExceeded { @@ -1811,6 +1812,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r model = canonicalOpenAIAccountSchedulingModel(account, originalModel) } _ = s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusBadRequest, resp.Header, body, model) + s.recordOpenAIModelNotSupportedAttempt(c, account, resp.Header, body, false) return nil, newOpenAIModelNotSupportedFailoverError(resp.Header, body) } body = restoreSystemPromptEcho(c, body) @@ -1864,15 +1866,16 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r if !cyberPolicyHit && isOpenAIRefusalRecoveryResponsesRequest(c) { runtime := s.openAIRefusalRecoveryRuntime(ctx) if runtime.RewriteEnabled() { - rewritten, matched, _, rewriteErr := RewriteOpenAIResponsesJSON(body, runtime.Matcher) + rewritten, matched, evidence, rewriteErr := rewriteOpenAIResponsesJSONWithEvidence(body, runtime.Matcher) if rewriteErr != nil { logger.FromContext(ctx).Warn("openai.refusal_recovery_rewrite_failed", zap.String("transport", "http"), zap.Error(rewriteErr)) } else if matched { if openAIRefusalShouldPromptRetry(c, runtime) { + recordOpenAIRefusalRecovery(ctx, c, account, "http", false, false, openAIRefusalActionPromptRetry, evidence) return nil, NewOpenAIRefusalRecoveryFailoverError(resp.Header) } body = rewritten - logger.FromContext(ctx).Info("openai.refusal_recovery_rewritten", zap.String("transport", "http")) + recordOpenAIRefusalRecovery(ctx, c, account, "http", false, false, openAIRefusalActionRewritten, evidence) } } } @@ -2146,14 +2149,15 @@ func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Conte if isOpenAIRefusalRecoveryResponsesRequest(c) { runtime := s.openAIRefusalRecoveryRuntime(c.Request.Context()) if runtime.RewriteEnabled() { - if rewritten, matched, _, rewriteErr := RewriteOpenAIResponsesJSON(body, runtime.Matcher); rewriteErr != nil { + if rewritten, matched, evidence, rewriteErr := rewriteOpenAIResponsesJSONWithEvidence(body, runtime.Matcher); rewriteErr != nil { logger.FromContext(c.Request.Context()).Warn("openai.refusal_recovery_rewrite_failed", zap.String("transport", "http_sse_to_json"), zap.Error(rewriteErr)) } else if matched { if openAIRefusalShouldPromptRetry(c, runtime) { + recordOpenAIRefusalRecovery(c.Request.Context(), c, account, "http_sse_to_json", false, false, openAIRefusalActionPromptRetry, evidence) return nil, NewOpenAIRefusalRecoveryFailoverError(resp.Header) } body = rewritten - logger.FromContext(c.Request.Context()).Info("openai.refusal_recovery_rewritten", zap.String("transport", "http_sse_to_json")) + recordOpenAIRefusalRecovery(c.Request.Context(), c, account, "http_sse_to_json", false, false, openAIRefusalActionRewritten, evidence) } } } diff --git a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go index 400c71d45cdb..417c9e439a51 100644 --- a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go +++ b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go @@ -493,18 +493,21 @@ func TestOpenAIGatewayService_Forward_RecordsSafeInstructionsRequiredDiagnostic( require.True(t, ok) require.Len(t, events, 1) require.Equal(t, "request_rejected", events[0].Kind) - require.Equal(t, OpenAIRequestRejectedClientMessage, events[0].Message) + // The client terminal stays fixed; Ops keeps the upstream's own message. + require.Equal(t, "Missing required parameter: 'instructions'", events[0].Message) require.Equal(t, http.StatusBadRequest, events[0].UpstreamStatusCode) encoded, diagnostic := continuationDiagnosticTestEncoded(t, events[0].ContinuationDiagnostic) require.Equal(t, "request_validation", diagnostic.Get("classification").String()) require.Equal(t, "invalid_request_error", diagnostic.Get("upstream_error.error_type.value").String()) require.Equal(t, "missing_required_parameter", diagnostic.Get("upstream_error.error_code.value").String()) require.Equal(t, "instructions", diagnostic.Get("upstream_error.error_param.value").String()) - require.False(t, diagnostic.Get("upstream_error.message.value").Exists()) - require.NotContains(t, encoded, "Missing required parameter") - require.NotContains(t, encoded, "secret-token") + require.Equal(t, "Missing required parameter: 'instructions'", diagnostic.Get("upstream_error.message.value").String()) + require.Equal(t, "pc-forward", diagnostic.Get("incoming.prompt_cache.value").String()) + require.NotContains(t, encoded, "secret-token", "fields outside the diagnostic projection are not copied") - require.False(t, logSink.ContainsMessageAtLevel("OpenAI 上游返回 Instructions are required，已记录请求详情用于排查", "warn")) + // Same server log as upstream v0.2.8 for an instructions-required rejection. + require.True(t, logSink.ContainsMessageAtLevel("OpenAI 上游返回 Instructions are required，已记录请求详情用于排查", "warn")) + require.True(t, logSink.ContainsFieldValue("request_model", "gpt-5.1-codex")) require.False(t, logSink.ContainsField("request_body_preview")) } diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index 495941a39e0e..4d409f9d7d6f 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -464,8 +464,9 @@ func openAIContinuationStateErrorFromFailedEvent(statusCode int, responseHeaders } // A residual HTTP 400 is a rejected request, not evidence that its history is -// permanently unavailable. Keep the original upstream details in bounded Ops -// diagnostics only; the client receives a stable, non-sensitive terminal. +// permanently unavailable. Ops records the upstream message, the bounded body +// and the diagnostic at the rejection site; the client receives a stable +// terminal. const OpenAIRequestRejectedCode = "upstream_request_rejected" const OpenAIRequestRejectedClientMessage = "The upstream rejected this request. Check the request parameters before trying again." const openAIRequestRejectedReason = GatewayFailureReason("openai_request_rejected") @@ -524,7 +525,9 @@ func classifyOpenAIRequestRejection(statusCode int, upstreamMsg string, upstream } // NewOpenAIRequestRejectedError stops the current request without retrying or -// changing selected-account health. It deliberately carries no upstream body. +// changing selected-account health. It carries no upstream body: the caller +// has already recorded the upstream text for Ops, and nothing downstream may +// render it to the client. func NewOpenAIRequestRejectedError( statusCode int, responseHeaders http.Header, @@ -695,7 +698,7 @@ func (s *OpenAIGatewayService) handleErrorResponse( requestedModel ...string, ) (*OpenAIForwardResult, error) { body := s.readUpstreamErrorBody(resp) - if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover(ctx, account, resp.StatusCode, resp.Header, body); ok { + if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover(ctx, c, account, resp.StatusCode, resp.Header, body); ok { return nil, failoverErr } body = s.redactAgentIdentitySensitiveBody(ctx, account, body) @@ -945,7 +948,7 @@ func (s *OpenAIGatewayService) handleCompatErrorResponse( ) (*OpenAIForwardResult, error) { body := s.readUpstreamErrorBody(resp) if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( - c.Request.Context(), account, resp.StatusCode, resp.Header, body, + c.Request.Context(), c, account, resp.StatusCode, resp.Header, body, ); ok { return nil, failoverErr } diff --git a/backend/internal/service/openai_images.go b/backend/internal/service/openai_images.go index f54356238f8e..1d7d83ee0dbe 100644 --- a/backend/internal/service/openai_images.go +++ b/backend/internal/service/openai_images.go @@ -696,7 +696,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesAPIKey( resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency) SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds()) if err != nil { - return nil, s.handleOpenAIUpstreamTransportError(upstreamCtx, c, account, err, false) + return nil, s.handleOpenAIImagesUpstreamTransportError(upstreamCtx, c, account, err, upstreamReq.URL.String()) } if resp.StatusCode >= 400 { respBody := s.readUpstreamErrorBody(resp) @@ -1815,3 +1815,33 @@ func dedupeStrings(values []string) []string { } return out } + +// handleOpenAIImagesUpstreamTransportError restores the upstream_url that +// v0.2.8 recorded on image transport failures; the shared transport helper +// records the attempt but does not know the endpoint. +func (s *OpenAIGatewayService) handleOpenAIImagesUpstreamTransportError(ctx context.Context, c *gin.Context, account *Account, err error, upstreamURL string) error { + recordedBefore := openAIOpsUpstreamErrorCount(c) + transportErr := s.handleOpenAIUpstreamTransportError(ctx, c, account, err, false) + if c != nil { + if value, ok := c.Get(OpsUpstreamErrorsKey); ok { + if events, ok := value.([]*OpsUpstreamErrorEvent); ok && len(events) > recordedBefore { + if last := events[len(events)-1]; last != nil && last.UpstreamURL == "" { + last.UpstreamURL = safeUpstreamURL(upstreamURL) + } + } + } + } + return transportErr +} + +func openAIOpsUpstreamErrorCount(c *gin.Context) int { + if c == nil { + return 0 + } + if value, ok := c.Get(OpsUpstreamErrorsKey); ok { + if events, ok := value.([]*OpsUpstreamErrorEvent); ok { + return len(events) + } + } + return 0 +} diff --git a/backend/internal/service/openai_images_incomplete_test.go b/backend/internal/service/openai_images_incomplete_test.go index 8aab3120dded..c9a448f10b87 100644 --- a/backend/internal/service/openai_images_incomplete_test.go +++ b/backend/internal/service/openai_images_incomplete_test.go @@ -75,7 +75,7 @@ func TestSummarizeNoOutputBody_ExtractsDiagnostics(t *testing.T) { } } -// 摘要应能抓到 incomplete_reason 并对超长 body 截断。 +// 摘要应能抓到 incomplete_reason；body 按 Ops 能存下的上限保留，只截断超出部分。 func TestSummarizeNoOutputBody_IncompleteReasonAndTruncation(t *testing.T) { long := strings.Repeat("x", 2000) body := "data: {\"type\":\"response.incomplete\",\"response\":{\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"junk\":\"" + long + "\"}}\n\n" @@ -83,8 +83,12 @@ func TestSummarizeNoOutputBody_IncompleteReasonAndTruncation(t *testing.T) { if !strings.Contains(summary, "incomplete_reason=max_output_tokens") { t.Fatalf("should capture incomplete reason, got %q", summary[:120]) } - if !strings.Contains(summary, "truncated") { - t.Fatalf("oversized body should be truncated, len=%d", len(summary)) + if !strings.Contains(summary, long) || strings.Contains(summary, "truncated") { + t.Fatalf("a body within the Ops detail budget must be kept whole, len=%d", len(summary)) + } + oversized := summarizeOpenAIImagesNoOutputBody([]byte(strings.Repeat("y", opsMaxStoredErrorBodyBytes+10))) + if !strings.Contains(oversized, "truncated") { + t.Fatalf("a body above the Ops detail budget should be truncated, len=%d", len(oversized)) } } diff --git a/backend/internal/service/openai_images_responses.go b/backend/internal/service/openai_images_responses.go index 54ea0a797327..a37c3dd733d2 100644 --- a/backend/internal/service/openai_images_responses.go +++ b/backend/internal/service/openai_images_responses.go @@ -45,8 +45,19 @@ type OpenAIImagesUpstreamError struct { // with words instead of an image"), not the account — see // shouldCoolOpenAIImagesToolForError. SynthesizedFromModelText bool + // ModelText is the model's complete plain-text answer behind an error + // inferred from it. Message keeps the bounded client text; Ops and the + // administrator's connection test see ModelText. + ModelText string } +// openAIImagesModelTextClientLimit is the window of model text that is +// classified and returned to the client; administrators get the full text. +const openAIImagesModelTextClientLimit = 600 + +// openAIImagesModelTextOpsLimit bounds the streamed model text kept for Ops. +const openAIImagesModelTextOpsLimit = 64 << 10 + func (e *OpenAIImagesUpstreamError) Error() string { if e == nil { return "" @@ -88,6 +99,17 @@ func (e *OpenAIImagesUpstreamError) clientErrorType() string { return "upstream_error" } +// opsMessage is what administrators see for this error: the model's complete +// words when the error was inferred from them, otherwise the client message. +func (e *OpenAIImagesUpstreamError) opsMessage() string { + if e != nil { + if text := strings.TrimSpace(e.ModelText); text != "" { + return sanitizeUpstreamErrorMessage(text) + } + } + return e.clientMessage() +} + func (e *OpenAIImagesUpstreamError) clientMessage() string { if e == nil { return "Upstream request failed" @@ -685,13 +707,9 @@ func extractOpenAIImagesModelText(body []byte) string { } } }) - refusal := strings.TrimSpace(b.String()) - // 截断过长文本，避免把整段模型输出塞进错误响应。 - const maxRefusal = 600 - if len(refusal) > maxRefusal { - refusal = refusal[:maxRefusal] - } - return refusal + // The complete text: callers that answer the client bound it themselves + // (openAIImagesTextFallbackErrorForWindow); Ops and connection tests keep it. + return strings.TrimSpace(b.String()) } func isOpenAIImagesContentPolicyRefusal(text string) bool { @@ -708,10 +726,19 @@ func isOpenAIImagesContentPolicyRefusal(text string) bool { return false } +// openAIImagesModelTextClientWindow cuts model text to the byte window the +// client answer has always used; callers trim it afterwards, as before. +func openAIImagesModelTextClientWindow(text string) string { + if len(text) > openAIImagesModelTextClientLimit { + return text[:openAIImagesModelTextClientLimit] + } + return text +} + // extractOpenAIImagesModelRefusal returns only text with an explicit safety or // moderation signal. Plain prompt suggestions are capability failures instead. func extractOpenAIImagesModelRefusal(body []byte) string { - text := extractOpenAIImagesModelText(body) + text := openAIImagesModelTextClientWindow(extractOpenAIImagesModelText(body)) if !isOpenAIImagesContentPolicyRefusal(text) { return "" } @@ -719,20 +746,26 @@ func extractOpenAIImagesModelRefusal(body []byte) string { } func openAIImagesTextFallbackError(body []byte) *OpenAIImagesUpstreamError { - return openAIImagesTextFallbackErrorForText(extractOpenAIImagesModelText(body)) + modelText := extractOpenAIImagesModelText(body) + return openAIImagesTextFallbackErrorForWindow(openAIImagesModelTextClientWindow(modelText), modelText) } -func openAIImagesTextFallbackErrorForText(text string) *OpenAIImagesUpstreamError { - text = strings.TrimSpace(text) +// openAIImagesTextFallbackErrorForWindow classifies and answers the client from +// window exactly as the gateway always has; modelText is the complete text +// kept in ModelText for administrators. +func openAIImagesTextFallbackErrorForWindow(window, modelText string) *OpenAIImagesUpstreamError { + text := strings.TrimSpace(window) if text == "" { return nil } + modelText = strings.TrimSpace(modelText) if isOpenAIImagesContentPolicyRefusal(text) { return &OpenAIImagesUpstreamError{ StatusCode: http.StatusBadRequest, ErrorType: "image_generation_user_error", Code: "content_policy_violation", Message: sanitizeUpstreamErrorMessage(text), + ModelText: modelText, } } return &OpenAIImagesUpstreamError{ @@ -744,6 +777,7 @@ func openAIImagesTextFallbackErrorForText(text string) *OpenAIImagesUpstreamErro // good enough to fail this turn over to another account, not evidence that // this account's image tool is down for the next 30 minutes. SynthesizedFromModelText: true, + ModelText: modelText, } } @@ -779,11 +813,10 @@ func summarizeOpenAIImagesNoOutputBody(body []byte) string { if incompleteReason != "" { fmt.Fprintf(&b, " incomplete_reason=%s", incompleteReason) } - // 附 body 截断片段（脱敏后），上限 1KB，避免日志膨胀。 + // The body itself, up to what Ops stores for an upstream detail. snippet := strings.TrimSpace(string(body)) - const maxSnippet = 1024 - if len(snippet) > maxSnippet { - snippet = snippet[:maxSnippet] + "...(truncated)" + if len(snippet) > opsMaxStoredErrorBodyBytes { + snippet = truncateString(snippet, opsMaxStoredErrorBodyBytes) + "...(truncated)" } if snippet != "" { fmt.Fprintf(&b, " body=%s", snippet) @@ -1364,7 +1397,7 @@ func (s *OpenAIGatewayService) handleOpenAIImagesOAuthNonStreamingResponse( return OpenAIUsage{}, 0, nil, upstreamErr } if textFallbackErr := openAIImagesTextFallbackError(body); textFallbackErr != nil { - setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.clientMessage(), summarizeOpenAIImagesNoOutputBody(body)) + setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.opsMessage(), summarizeOpenAIImagesNoOutputBody(body)) if !IsOpenAIImagesRetryableUpstreamError(textFallbackErr) { writeOpenAIImagesUpstreamErrorResponse(c, textFallbackErr) } @@ -1427,12 +1460,15 @@ func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse( pendingResults := make([]openAIResponsesImageResult, 0, 1) pendingSeen := make(map[string]struct{}) streamMeta := openAIResponsesImageResult{Model: strings.TrimSpace(fallbackModel)} + // fallbackText keeps the streamed model text for Ops. The client answer + // uses its first openAIImagesModelTextClientLimit bytes, the exact text the + // gateway accumulated before. var fallbackText strings.Builder appendFallbackText := func(text string) { - if text == "" || fallbackText.Len() >= 600 { + if text == "" || fallbackText.Len() >= openAIImagesModelTextOpsLimit { return } - remaining := 600 - fallbackText.Len() + remaining := openAIImagesModelTextOpsLimit - fallbackText.Len() if len(text) > remaining { text = text[:remaining] } @@ -1531,13 +1567,14 @@ func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse( } reconcileOpenAIResponsesImageResultSizes(finalResults, nil) if len(finalResults) == 0 { - textFallbackErr := openAIImagesTextFallbackErrorForText(fallbackText.String()) + streamedText := fallbackText.String() + textFallbackErr := openAIImagesTextFallbackErrorForWindow(openAIImagesModelTextClientWindow(streamedText), streamedText) if textFallbackErr == nil { textFallbackErr = openAIImagesTextFallbackError(dataBytes) } if textFallbackErr != nil { retryable := IsOpenAIImagesRetryableUpstreamError(textFallbackErr) - setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.clientMessage(), summarizeOpenAIImagesNoOutputBody(dataBytes)) + setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.opsMessage(), summarizeOpenAIImagesNoOutputBody(dataBytes)) if !retryable && !clientDisconnected { s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBodyFromUpstream(textFallbackErr)) } @@ -1868,7 +1905,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth( resp, err := s.doOpenAICodexUpstream(upstreamReq, account, proxyURL) SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds()) if err != nil { - return nil, s.handleOpenAIUpstreamTransportError(upstreamCtx, c, account, err, false) + return nil, s.handleOpenAIImagesUpstreamTransportError(upstreamCtx, c, account, err, upstreamReq.URL.String()) } if resp.StatusCode >= 400 { respBody := s.readUpstreamErrorBody(resp) diff --git a/backend/internal/service/openai_images_tool_cooldown_test.go b/backend/internal/service/openai_images_tool_cooldown_test.go index 0f38c200d782..9c6f03d2c7dd 100644 --- a/backend/internal/service/openai_images_tool_cooldown_test.go +++ b/backend/internal/service/openai_images_tool_cooldown_test.go @@ -175,3 +175,9 @@ func TestOpenAIImagesTextFallback_RemainsRetryableAndThusCascades(t *testing.T) require.True(t, IsOpenAIImagesRetryableUpstreamError(err), "文字兜底判据是可重试的——正因如此，写账号冷却会沿号池级联") } + +// openAIImagesTextFallbackErrorForText classifies text that is both the client +// window and the complete model text. +func openAIImagesTextFallbackErrorForText(text string) *OpenAIImagesUpstreamError { + return openAIImagesTextFallbackErrorForWindow(text, text) +} diff --git a/backend/internal/service/openai_model_not_supported_test.go b/backend/internal/service/openai_model_not_supported_test.go index bc1a36fa56e7..dd8c5dd2d3d8 100644 --- a/backend/internal/service/openai_model_not_supported_test.go +++ b/backend/internal/service/openai_model_not_supported_test.go @@ -166,7 +166,8 @@ func TestRateLimitService_HandleUpstreamError_ModelNotSupportedUsesModelCooldown require.True(t, handled) require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "openai/gpt-5.6-luna", repo.modelRateLimitCalls[0].scope) - require.Equal(t, string(openAIModelNotSupportedReason), repo.modelRateLimitCalls[0].reason) + require.Equal(t, string(openAIModelNotSupportedReason), repo.modelRateLimitCalls[0].reason, "the reason code stays the exact value availability matching reads") + require.Equal(t, "Model 'gpt-5.6-luna' is temporarily not supported. Please choose another model from GET /v1/models.", repo.modelRateLimitCalls[0].upstreamMessage) require.WithinDuration(t, time.Now().Add(upstreamModelNotSupportedCooldown), repo.modelRateLimitCalls[0].resetAt, 5*time.Second) } diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index 1bf98f2cff60..52f326aeb0ae 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -1554,17 +1554,21 @@ func TestOpenAIGatewayService_APIKeyPassthrough_RebuildsUpstreamErrors(t *testin require.Equal(t, "request_rejected", opsEvents[0].Kind) require.True(t, opsEvents[0].Passthrough) require.Equal(t, tt.statusCode, opsEvents[0].UpstreamStatusCode) - require.Equal(t, OpenAIRequestRejectedClientMessage, opsEvents[0].Message) + // The client never sees the upstream text; the administrator's + // Ops record keeps it. + require.Equal(t, sanitizeUpstreamErrorMessage(extractUpstreamErrorMessage([]byte(tt.responseBody))), opsEvents[0].Message) encoded, diagnostic := continuationDiagnosticTestEncoded(t, opsEvents[0].ContinuationDiagnostic) if tt.contentType == "application/json" { require.Equal(t, "request_validation", diagnostic.Get("classification").String()) + require.Equal(t, "secret-upstream.example invalid parameter", diagnostic.Get("upstream_error.message.value").String()) + require.Equal(t, "upstream_secret_code", diagnostic.Get("upstream_error.error_code.value").String()) + require.Equal(t, "private_field", diagnostic.Get("upstream_error.error_param.value").String()) } else { require.Equal(t, "unclassified_bad_request", diagnostic.Get("classification").String()) + require.Equal(t, tt.responseBody, diagnostic.Get("upstream_error.message.value").String()) } - require.NotContains(t, encoded, "secret-upstream.example") + // Only the error fields are projected into the diagnostic. require.NotContains(t, encoded, "sk-upstream-secret") - require.NotContains(t, encoded, "upstream_secret_code") - require.NotContains(t, encoded, "private_field") return } require.Equal(t, tt.wantStatus, rec.Code) @@ -2206,7 +2210,8 @@ func TestOpenAIGatewayService_OpenAIPassthrough_ContinuationErrorsStopBeforeAcco require.NotEmpty(t, events) require.Equal(t, "continuation_state", events[len(events)-1].Kind) require.True(t, events[len(events)-1].Passthrough) - require.Equal(t, OpenAIContinuationStateUnavailableClientMessage, events[len(events)-1].Message) + require.Equal(t, openAIOpsUpstreamErrorMessage([]byte(tt.responseBody)), events[len(events)-1].Message) + require.NotEmpty(t, events[len(events)-1].Message, "Ops keeps the upstream text, not the client sentence") }) } } diff --git a/backend/internal/service/openai_reasoning_recovery.go b/backend/internal/service/openai_reasoning_recovery.go index 12fc2ad2177e..9fdfc303a4fc 100644 --- a/backend/internal/service/openai_reasoning_recovery.go +++ b/backend/internal/service/openai_reasoning_recovery.go @@ -25,6 +25,17 @@ import ( const openAIReasoningRecoveryContextKey = "openai_reasoning_recovery_state" const openAIReasoningRecoveryHeader = "X-Sub2API-Reasoning-Recovery" +// The reasoning_recovery Ops detail is bounded by its encoded size: the Ops +// queue keeps at most OpsErrorLogQueueBodyMaxBytes per detail and shrinks a +// larger JSON detail to fields it does not know, which would lose everything. +// The upstream message and payload start at these caps and are halved until +// the encoded detail fits; action, items, usage and payload_sha256 always stay. +const ( + openAIReasoningRecoveryDetailEncodedLimit = 6 << 10 + openAIReasoningRecoveryMessageDetailLimit = 1 << 10 + openAIReasoningRecoveryPayloadDetailLimit = 4 << 10 +) + // openAIReasoningRecoveryState owns the single, explicitly lossy retry for an // HTTP Responses request. It never re-enters a protocol converter or scheduler. // original is immutable; wire is the exact payload of the latest sent attempt. @@ -61,6 +72,9 @@ type openAIReasoningRecoveryState struct { diagnosticStopReason string failureObservedBytes int failureTerminalForwarded bool + // redactPayload applies the upstream Agent Identity credential redaction + // before an upstream payload is recorded for administrators. + redactPayload func([]byte) []byte } func (s *OpenAIGatewayService) newOpenAIReasoningRecoveryState(ctx context.Context, c *gin.Context, account *Account, token string) *openAIReasoningRecoveryState { @@ -75,6 +89,7 @@ func (s *OpenAIGatewayService) newOpenAIReasoningRecoveryState(ctx context.Conte policyUnavailable: policyErr != nil, store: s.openAIReasoningStateStore(), budget: openAIReasoningCacheBudgetForRequest(c), } + r.redactPayload = func(payload []byte) []byte { return s.redactAgentIdentitySensitiveBody(ctx, account, payload) } if c != nil { c.Set(openAIReasoningRecoveryContextKey, r) } @@ -792,8 +807,14 @@ func (r *openAIReasoningRecoveryState) record(action string, status int, code st } detail := map[string]any{"action": action, "items": count, "usage_status": "unavailable"} usage := openAIReasoningUsageEvidence(payload) + recordedPayload := payload + if r.redactPayload != nil && len(payload) > 0 { + recordedPayload = r.redactPayload(payload) + } + upstreamMessage := "" if len(payload) > 0 { detail["payload_sha256"] = openAIReasoningDigest(payload) + upstreamMessage = extractOpenAISSEErrorMessage(recordedPayload) } if len(usage) == 0 && action == "retry_without_encrypted_content" { usage = r.attemptUsage @@ -805,7 +826,7 @@ func (r *openAIReasoningRecoveryState) record(action string, status int, code st detail["usage_status"] = "available" detail["usage"] = usage } - encoded, _ := json.Marshal(detail) + encoded := encodeOpenAIReasoningRecoveryDetail(detail, upstreamMessage, recordedPayload) logger.FromContext(r.ctx).Info("openai.reasoning_recovery", zap.Int64("account_id", r.account.ID), zap.String("action", action), zap.String("code", code), zap.Int("upstream_status", status), @@ -816,10 +837,57 @@ func (r *openAIReasoningRecoveryState) record(action string, status int, code st Kind: "reasoning_recovery", Stage: "inference", Scope: "request", Reason: code, UpstreamStatusCode: status, Message: action, Detail: string(encoded), UpstreamRequestID: r.responseHeaders.Get("x-request-id"), - ContinuationDiagnostic: r.diagnosticForRecordedAttempt(action, payload), + ContinuationDiagnostic: r.diagnosticForRecordedAttempt(action, recordedPayload), }) } +// encodeOpenAIReasoningRecoveryDetail adds the upstream message and the rejected +// payload to the essential detail, halving their bounds until the encoded +// detail fits openAIReasoningRecoveryDetailEncodedLimit. +func encodeOpenAIReasoningRecoveryDetail(detail map[string]any, upstreamMessage string, payload []byte) []byte { + essential, _ := json.Marshal(detail) + if upstreamMessage == "" && len(payload) == 0 { + return essential + } + messageLimit, payloadLimit := openAIReasoningRecoveryMessageDetailLimit, openAIReasoningRecoveryPayloadDetailLimit + for messageLimit > 0 || payloadLimit > 0 { + candidate := make(map[string]any, len(detail)+5) + for key, value := range detail { + candidate[key] = value + } + if upstreamMessage != "" && messageLimit > 0 { + text, truncated := continuationDiagnosticBoundValue(upstreamMessage, messageLimit) + candidate["upstream_message"] = text + if truncated { + candidate["upstream_message_truncated"] = true + } + } + if len(payload) > 0 { + if payloadLimit > 0 { + text, truncated := continuationDiagnosticBoundValue(string(payload), payloadLimit) + candidate["payload"] = text + if truncated { + candidate["payload_truncated"] = true + } + } else { + candidate["payload_truncated"] = true + } + candidate["payload_bytes"] = len(payload) + } + if encoded, err := json.Marshal(candidate); err == nil && len(encoded) <= openAIReasoningRecoveryDetailEncodedLimit { + return encoded + } + messageLimit, payloadLimit = messageLimit/2, payloadLimit/2 + if messageLimit < 64 { + messageLimit = 0 + } + if payloadLimit < 64 { + payloadLimit = 0 + } + } + return essential +} + func (r *openAIReasoningRecoveryState) diagnosticForRecordedAttempt(action string, payload []byte) *OpenAIContinuationDiagnostic { if r.diagnosticIncoming == nil || action == "rejected_history_skipped" { // Called before PrepareRequest freezes the stripped wire. A later failure diff --git a/backend/internal/service/openai_reasoning_recovery_terminal_test.go b/backend/internal/service/openai_reasoning_recovery_terminal_test.go index 817c2db52449..ce558282937a 100644 --- a/backend/internal/service/openai_reasoning_recovery_terminal_test.go +++ b/backend/internal/service/openai_reasoning_recovery_terminal_test.go @@ -166,8 +166,14 @@ func TestOpenAIReasoningRecoveryTerminalsPreserveClassificationAndDiagnostic(t * } encoded, encodeErr := json.Marshal(diagnostic) require.NoError(t, encodeErr) - for _, secret := range []string{"opaque-old", "visible summary", "private-signature-detail-8162", "private-validation-detail-7514", "source-token"} { + // The administrator's diagnostic keeps the upstream message of + // the final rejection; request history stays summarized, and the + // returned error never carries upstream text. + require.Equal(t, gjson.Get(payload, "message").String(), diagnostic.UpstreamError.Message.Value) + for _, secret := range []string{"opaque-old", "visible summary", "source-token"} { require.NotContains(t, string(encoded), secret) + } + for _, secret := range []string{"opaque-old", "visible summary", "private-signature-detail-8162", "private-validation-detail-7514", "source-token"} { require.NotContains(t, err.Error(), secret) } }) diff --git a/backend/internal/service/openai_reasoning_recovery_test.go b/backend/internal/service/openai_reasoning_recovery_test.go index ae238fffd199..714ca6552bca 100644 --- a/backend/internal/service/openai_reasoning_recovery_test.go +++ b/backend/internal/service/openai_reasoning_recovery_test.go @@ -356,6 +356,9 @@ func TestOpenAIReasoningRecoveryForwardSSEAndBufferedPreserveFailedAttemptUsage( recorded = true require.Contains(t, entry.Detail, `"input_tokens":3`) require.Contains(t, entry.Detail, `"output_tokens":2`) + // The rejected upstream payload itself, not only its hash. + require.Contains(t, gjson.Get(entry.Detail, "payload").String(), `"invalid_encrypted_content"`) + require.NotEmpty(t, gjson.Get(entry.Detail, "payload_sha256").String()) } } require.True(t, recorded) diff --git a/backend/internal/service/openai_refusal_recovery.go b/backend/internal/service/openai_refusal_recovery.go index 4efa4d7ebf7f..99e233db9485 100644 --- a/backend/internal/service/openai_refusal_recovery.go +++ b/backend/internal/service/openai_refusal_recovery.go @@ -10,7 +10,9 @@ import ( "unicode" "unicode/utf8" + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" "github.com/gin-gonic/gin" + "go.uber.org/zap" "golang.org/x/text/unicode/norm" ) @@ -211,6 +213,68 @@ func leadingOpenAIRefusalParagraphs(value string, limit int) string { return string(runes) } +// openAIRefusalEvidence is the configured keyword that matched and the leading +// original upstream text the matcher inspected, bounded by the matcher's own +// maxOpenAIRefusalParagraphRunes window. +type openAIRefusalEvidence struct { + Keyword string + Text string +} + +func newOpenAIRefusalEvidence(keyword, visibleText string) openAIRefusalEvidence { + return openAIRefusalEvidence{Keyword: keyword, Text: leadingOpenAIRefusalParagraphs(visibleText, maxOpenAIRefusalScanParagraphs)} +} + +const ( + openAIRefusalActionRewritten = "rewritten" + openAIRefusalActionPromptRetry = "prompt_retry" +) + +// recordOpenAIRefusalRecovery logs the refusal behind a rewrite or a prompt +// retry: the matched keyword and the original text. A handled refusal is not +// an upstream error, so it never enters the upstream error context; HTTP +// requests keep it as evidence that Ops attaches only when the request finally +// fails. The client keeps receiving the configured replacement or the retry +// terminal. +func recordOpenAIRefusalRecovery(ctx context.Context, c *gin.Context, account *Account, transport string, webSocket bool, early bool, action string, evidence openAIRefusalEvidence) { + if ctx == nil { + ctx = context.Background() + } + fields := []zap.Field{ + zap.String("transport", transport), + zap.Bool("early", early), + zap.String("keyword", evidence.Keyword), + zap.String("refusal_text", evidence.Text), + } + if account != nil { + fields = append(fields, zap.Int64("account_id", account.ID)) + } + logEvent := "openai.refusal_recovery_rewritten" + if action == openAIRefusalActionPromptRetry { + logEvent = "openai.refusal_recovery_prompt_retry" + } + logger.FromContext(ctx).Info(logEvent, fields...) + // A WebSocket connection carries many turns; its refusals stay in the log. + if c == nil || webSocket { + return + } + proxyID, proxyName := opsUpstreamProxyAttribution(account) + event := OpsUpstreamErrorEvent{ + ProxyID: proxyID, + ProxyName: proxyName, + Kind: "refusal_recovery", + Stage: string(GatewayFailureStageInference), + Scope: string(GatewayFailureScopeRequest), + Reason: action, + Message: fmt.Sprintf("upstream refusal matched keyword %q (%s, transport=%s)", evidence.Keyword, action, transport), + Detail: evidence.Text, + } + if account != nil { + event.Platform, event.AccountID, event.AccountName = account.Platform, account.ID, account.Name + } + appendOpsRefusalRecoveryEvent(c, event) +} + func NewOpenAIRefusalRecoveryFailoverError(upstreamHeaders http.Header) *UpstreamFailoverError { headers := make(http.Header) if upstreamHeaders != nil { diff --git a/backend/internal/service/openai_refusal_recovery_json.go b/backend/internal/service/openai_refusal_recovery_json.go index 05c9176e5f67..e38646fccc31 100644 --- a/backend/internal/service/openai_refusal_recovery_json.go +++ b/backend/internal/service/openai_refusal_recovery_json.go @@ -25,12 +25,20 @@ type openAIResponsesRewriteContentItem struct { } func RewriteOpenAIResponsesJSON(body []byte, matcher *OpenAIRefusalMatcher) ([]byte, bool, string, error) { + rewritten, matched, evidence, err := rewriteOpenAIResponsesJSONWithEvidence(body, matcher) + return rewritten, matched, evidence.Keyword, err +} + +// rewriteOpenAIResponsesJSONWithEvidence also returns the leading original text +// the matcher inspected, so the rewrite can be audited by administrators. +func rewriteOpenAIResponsesJSONWithEvidence(body []byte, matcher *OpenAIRefusalMatcher) ([]byte, bool, openAIRefusalEvidence, error) { + var none openAIRefusalEvidence var envelope openAIResponsesRewriteEnvelope if err := json.Unmarshal(body, &envelope); err != nil { - return nil, false, "", err + return nil, false, none, err } if envelope.Status != "completed" || len(envelope.Output) == 0 { - return body, false, "", nil + return body, false, none, nil } var visibleText strings.Builder @@ -40,7 +48,7 @@ func RewriteOpenAIResponsesJSON(body []byte, matcher *OpenAIRefusalMatcher) ([]b continue } if output.Type != "message" || output.Role != "assistant" || len(output.Content) == 0 { - return body, false, "", nil + return body, false, none, nil } if firstMessageID == "" { firstMessageID = output.ID @@ -52,22 +60,23 @@ func RewriteOpenAIResponsesJSON(body []byte, matcher *OpenAIRefusalMatcher) ([]b case "refusal": _, _ = visibleText.WriteString(content.Refusal) default: - return body, false, "", nil + return body, false, none, nil } } } if firstMessageID == "" { - return body, false, "", nil + return body, false, none, nil } matched, keyword := matcher.MatchLeadingParagraphs(visibleText.String()) if !matched { - return body, false, "", nil + return body, false, none, nil } + evidence := newOpenAIRefusalEvidence(keyword, visibleText.String()) var raw map[string]json.RawMessage if err := json.Unmarshal(body, &raw); err != nil { - return nil, false, "", err + return nil, false, none, err } replacementOutput := []openAIResponsesRewriteOutputItem{{ ID: firstMessageID, @@ -81,13 +90,13 @@ func RewriteOpenAIResponsesJSON(body []byte, matcher *OpenAIRefusalMatcher) ([]b }} encodedOutput, err := json.Marshal(replacementOutput) if err != nil { - return nil, false, "", err + return nil, false, none, err } raw["output"] = encodedOutput raw["status"] = json.RawMessage(`"completed"`) rewritten, err := json.Marshal(raw) if err != nil { - return nil, false, "", err + return nil, false, none, err } - return rewritten, true, keyword, nil + return rewritten, true, evidence, nil } diff --git a/backend/internal/service/openai_refusal_recovery_stream.go b/backend/internal/service/openai_refusal_recovery_stream.go index ba783a6ad3fc..77cc49a1d45d 100644 --- a/backend/internal/service/openai_refusal_recovery_stream.go +++ b/backend/internal/service/openai_refusal_recovery_stream.go @@ -34,6 +34,8 @@ type openAIRefusalStreamState struct { completedMessage []byte responseID string messageID string + // matchedEvidence is what the matcher decided on, kept for Ops and logs. + matchedEvidence openAIRefusalEvidence } func newOpenAIRefusalStreamState(matcher *OpenAIRefusalMatcher) *openAIRefusalStreamState { @@ -44,6 +46,15 @@ func newOpenAIRefusalStreamStateWithEarlyEmission(matcher *OpenAIRefusalMatcher, return &openAIRefusalStreamState{matcher: matcher, earlyEligible: eligible} } +// evidence returns the keyword and original leading text behind the latest +// refusal match on this stream. +func (s *openAIRefusalStreamState) evidence() openAIRefusalEvidence { + if s == nil { + return openAIRefusalEvidence{} + } + return s.matchedEvidence +} + func (s *openAIRefusalStreamState) reserveLine(line string) bool { if s == nil || s.passthrough || s.earlyEmitted { return true @@ -75,8 +86,9 @@ func (s *openAIRefusalStreamState) observe(eventType string, payload []byte) (op switch eventType { case "response.output_text.delta", "response.refusal.delta": _, _ = s.visibleText.WriteString(gjson.GetBytes(payload, "delta").String()) - if matched, _ := s.matcher.MatchLeadingParagraphs(s.visibleText.String()); matched { + if matched, keyword := s.matcher.MatchLeadingParagraphs(s.visibleText.String()); matched { s.matched = true + s.matchedEvidence = newOpenAIRefusalEvidence(keyword, s.visibleText.String()) return s.startEarlyReplacement() } if openAIRefusalScanWindowComplete(s.visibleText.String()) || utf8.RuneCountInString(s.visibleText.String()) >= maxOpenAIRefusalParagraphRunes { @@ -93,8 +105,9 @@ func (s *openAIRefusalStreamState) observe(eventType string, payload []byte) (op s.visibleText.Reset() _, _ = s.visibleText.WriteString(text) } - if matched, _ := s.matcher.MatchLeadingParagraphs(s.visibleText.String()); matched { + if matched, keyword := s.matcher.MatchLeadingParagraphs(s.visibleText.String()); matched { s.matched = true + s.matchedEvidence = newOpenAIRefusalEvidence(keyword, s.visibleText.String()) return s.startEarlyReplacement() } s.passthrough = true @@ -109,7 +122,7 @@ func (s *openAIRefusalStreamState) observe(eventType string, payload []byte) (op if err != nil { return openAIRefusalStreamHold, nil, err } - rewritten, matched, _, err := RewriteOpenAIResponsesJSON(responseJSON, s.matcher) + rewritten, matched, evidence, err := rewriteOpenAIResponsesJSONWithEvidence(responseJSON, s.matcher) if err != nil { return openAIRefusalStreamHold, nil, err } @@ -117,6 +130,7 @@ func (s *openAIRefusalStreamState) observe(eventType string, payload []byte) (op s.passthrough = true return openAIRefusalStreamPass, nil, nil } + s.matchedEvidence = evidence stream, err := buildOpenAIRefusalReplacementSSE(payload, rewritten, s.matcher.Replacement()) if err != nil { return openAIRefusalStreamHold, nil, err @@ -223,13 +237,16 @@ func (s *openAIRefusalStreamState) completeEarlyReplacement(payload []byte) (ope if err != nil { return openAIRefusalStreamHold, nil, err } - rewritten, matched, _, err := RewriteOpenAIResponsesJSON(responseJSON, s.matcher) + rewritten, matched, evidence, err := rewriteOpenAIResponsesJSONWithEvidence(responseJSON, s.matcher) if err != nil { return openAIRefusalStreamHold, nil, err } if !matched { return openAIRefusalStreamHold, nil, errors.New("early replacement terminal response is not text-only refusal output") } + if s.matchedEvidence.Keyword == "" { + s.matchedEvidence = evidence + } tail, err := buildOpenAIRefusalReplacementTerminalSSE( payload, rewritten, diff --git a/backend/internal/service/openai_refusal_recovery_ws.go b/backend/internal/service/openai_refusal_recovery_ws.go index b81a213c44ff..04e5c372ef25 100644 --- a/backend/internal/service/openai_refusal_recovery_ws.go +++ b/backend/internal/service/openai_refusal_recovery_ws.go @@ -28,6 +28,11 @@ type openAIRefusalRecoveryWSOutput struct { buffered []openAIRefusalRecoveryWSFrame write openAIRefusalRecoveryWSWriteFunc onBufferLimit func() + // onRewritten receives the matched keyword and original text of each + // replaced refusal; onDelivered receives every text frame the client + // actually received, including held frames when they are flushed. + onRewritten func(openAIRefusalEvidence) + onDelivered func([]byte) } type openAIRefusalRecoveryWSFrameConn struct { @@ -109,6 +114,7 @@ func (o *openAIRefusalRecoveryWSOutput) Write(ctx context.Context, messageType c } return err case openAIRefusalStreamReplace: + evidence := o.state.evidence() o.dropTurnBuffer() var replacementFrames [][]byte forEachOpenAISSEDataPayload(string(replacementSSE), func(data []byte) { @@ -120,6 +126,9 @@ func (o *openAIRefusalRecoveryWSOutput) Write(ctx context.Context, messageType c } } o.resetTurn() + if o.onRewritten != nil { + o.onRewritten(evidence) + } return nil } } @@ -138,6 +147,21 @@ func (o *openAIRefusalRecoveryWSOutput) Write(ctx context.Context, messageType c return err } +// SetRewriteObserver receives the matched keyword and original text whenever a +// refusal is replaced on this connection. +func (o *openAIRefusalRecoveryWSOutput) SetRewriteObserver(observer func(openAIRefusalEvidence)) { + if o != nil { + o.onRewritten = observer + } +} + +// SetDeliveryObserver receives each text frame after the client received it. +func (o *openAIRefusalRecoveryWSOutput) SetDeliveryObserver(observer func([]byte)) { + if o != nil { + o.onDelivered = observer + } +} + func (o *openAIRefusalRecoveryWSOutput) writeDirect(ctx context.Context, messageType coderws.MessageType, payload []byte) error { if err := o.write(ctx, messageType, payload); err != nil { return err @@ -148,6 +172,9 @@ func (o *openAIRefusalRecoveryWSOutput) writeDirect(ctx context.Context, message if openAIRefusalRecoveryWSSemanticEvent(eventType, payload) { o.semanticWrite = true } + if o.onDelivered != nil { + o.onDelivered(payload) + } } return nil } diff --git a/backend/internal/service/openai_refusal_recovery_ws_test.go b/backend/internal/service/openai_refusal_recovery_ws_test.go index d1c8fc9cf40d..469c59e84783 100644 --- a/backend/internal/service/openai_refusal_recovery_ws_test.go +++ b/backend/internal/service/openai_refusal_recovery_ws_test.go @@ -20,6 +20,10 @@ func TestOpenAIRefusalRecoveryWSOutputRewritesAcrossFramesAndResetsNextTurn(t *t written = append(written, append([]byte(nil), payload...)) return nil }, nil) + var rewrites []openAIRefusalEvidence + output.SetRewriteObserver(func(evidence openAIRefusalEvidence) { rewrites = append(rewrites, evidence) }) + var delivered [][]byte + output.SetDeliveryObserver(func(payload []byte) { delivered = append(delivered, append([]byte(nil), payload...)) }) firstTurn := [][]byte{ []byte(`{"type":"response.created","response":{"id":"resp_ws_1","model":"gpt-5.4","status":"in_progress","output":[]}}`), @@ -34,8 +38,11 @@ func TestOpenAIRefusalRecoveryWSOutputRewritesAcrossFramesAndResetsNextTurn(t *t require.Empty(t, written) } } + require.Equal(t, []openAIRefusalEvidence{{Keyword: "I'm unable", Text: "I'm unable to help."}}, rewrites) require.Len(t, written, 8) + // Only frames the client received reach the delivery observer. + require.Equal(t, written, delivered) require.Equal(t, "response.created", gjson.GetBytes(written[0], "type").String()) require.Equal(t, "继续当前任务", gjson.GetBytes(written[3], "delta").String()) require.Equal(t, "resp_ws_1", gjson.GetBytes(written[7], "response.id").String()) @@ -44,7 +51,7 @@ func TestOpenAIRefusalRecoveryWSOutputRewritesAcrossFramesAndResetsNextTurn(t *t require.NotContains(t, string(payload), "I'm unable") } - written = nil + written, delivered = nil, nil secondTurn := [][]byte{ []byte(`{"type":"response.created","response":{"id":"resp_ws_2","model":"gpt-5.4","status":"in_progress","output":[]}}`), []byte(`{"type":"response.output_text.delta","response_id":"resp_ws_2","item_id":"msg_2","delta":"Normal answer."}`), @@ -53,6 +60,7 @@ func TestOpenAIRefusalRecoveryWSOutputRewritesAcrossFramesAndResetsNextTurn(t *t for _, payload := range secondTurn { require.NoError(t, output.Write(context.Background(), coderws.MessageText, payload)) } + require.Equal(t, written, delivered) require.Len(t, written, 3) require.Equal(t, "resp_ws_2", gjson.GetBytes(written[0], "response.id").String()) diff --git a/backend/internal/service/openai_upstream_transport_error.go b/backend/internal/service/openai_upstream_transport_error.go index 02045e7e7978..eacfa8ccd08b 100644 --- a/backend/internal/service/openai_upstream_transport_error.go +++ b/backend/internal/service/openai_upstream_transport_error.go @@ -71,6 +71,12 @@ func classifyUpstreamTransportError(err error) upstreamTransportErrorClass { if err == nil { return upstreamTransportErrorClass{} } + // A Codex route refusal is never a proxy or network fault. Its text names + // the concrete reason, which can quote an earlier network error, so it must + // not reach the markers below; it stays transient as it always was. + if errors.Is(err, errCodexRoutingUnavailable) || errors.Is(err, ErrOpenAICodexTicketUnavailable) { + return upstreamTransportErrorClass{} + } // — Typed checks (preferred) —————————————————————————————————————————————— if errors.Is(err, syscall.ECONNREFUSED) || diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index e5c9e7d7fa6d..cbf225833a19 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -557,6 +557,13 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( logOpenAIWSModeInfo("refusal_recovery_buffer_limit account_id=%d transport=websocket", account.ID) }, ) + refusalOutput.SetRewriteObserver(func(evidence openAIRefusalEvidence) { + recordOpenAIRefusalRecovery(ctx, c, account, "websocket", true, false, openAIRefusalActionRewritten, evidence) + }) + // Bridge turns write through this output, which may hold or replace a + // frame; failures are marked when the client actually receives them. + refusalOutput.SetDeliveryObserver(func(payload []byte) { markOpenAIWSDeliveredFailure(c, payload) }) + c.Set(openAIWSDeliveryMarksFailuresKey, true) writeClientMessage = func(message []byte) error { return refusalOutput.Write(ctx, coderws.MessageText, message) } @@ -854,6 +861,20 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( logOpenAIWSModeInfo("ingress_replay_buffer_limit account_id=%d transport=websocket", account.ID) }, ) + refusalOutput.SetRewriteObserver(func(evidence openAIRefusalEvidence) { + recordOpenAIRefusalRecovery(ctx, c, account, "websocket", true, false, openAIRefusalActionRewritten, evidence) + }) + // Upstream v0.2.8 marks every error/response.failed event the client + // receives. Frames may be held for the replay decision, so they are marked + // on delivery, with the upstream payload behind a capacity-shed rewrite. + capacityShedOriginals := make(map[string][]byte) + refusalOutput.SetDeliveryObserver(func(payload []byte) { + if original, ok := capacityShedOriginals[string(payload)]; ok { + delete(capacityShedOriginals, string(payload)) + payload = original + } + markOpenAIWSDeliveredFailure(c, payload) + }) writeClientMessage = func(message []byte) error { return refusalOutput.Write(ctx, coderws.MessageText, message) } @@ -1185,7 +1206,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( rawEventType, _, _ := parseOpenAIWSEventEnvelope(rawUpstreamMessage) if rawEventType == "error" || rawEventType == "response.failed" { if failoverErr, ok := s.openAIBudgetExceededTerminalFailover( - ctx, account, lease.HandshakeHeaders(), rawUpstreamMessage, + ctx, c, account, lease.HandshakeHeaders(), rawUpstreamMessage, true, turn == 1 && !downstreamOutputStarted(), ); ok { lease.MarkBroken() replaySafe := turn == 1 && !downstreamOutputStarted() @@ -1344,6 +1365,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( _ = s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusBadRequest, nil, upstreamMessage, canonicalModel) if !downstreamOutputStarted() { lease.MarkBroken() + s.recordOpenAIWSModelNotSupportedAttempt(c, account, lease.HandshakeHeaders(), upstreamMessage) return nil, newOpenAIModelNotSupportedFailoverError(lease.HandshakeHeaders(), upstreamMessage) } } @@ -1392,6 +1414,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( _ = s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusBadRequest, nil, upstreamMessage, canonicalModel) if !downstreamOutputStarted() { lease.MarkBroken() + s.recordOpenAIWSModelNotSupportedAttempt(c, account, lease.HandshakeHeaders(), upstreamMessage) return nil, newOpenAIModelNotSupportedFailoverError(lease.HandshakeHeaders(), upstreamMessage) } } @@ -1432,6 +1455,10 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( if eventType == "error" || eventType == "response.failed" { if rewritten, changed := sanitizeOpenAICapacityShedErrorCodeForClient(clientMessage); changed { clientMessage = rewritten + if len(capacityShedOriginals) >= 16 { + clear(capacityShedOriginals) + } + capacityShedOriginals[string(clientMessage)] = append([]byte(nil), upstreamMessage...) } } if err := writeClientMessage(clientMessage); err != nil { diff --git a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go index 6c5e24cdd186..1caeeed317e5 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go +++ b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go @@ -261,7 +261,7 @@ func TestOpenAIWSIngressBudgetExceededTerminalWriteOrdering(t *testing.T) { require.NotContains(t, string(payload), "sensitive upstream detail") } require.Equal(t, 1, repo.setErrorCalls, "a budget terminal puts the account into the error state") - require.Equal(t, "sensitive upstream detail", repo.lastErrorMsg, "the account keeps the upstream message") + require.Equal(t, "sensitive upstream detail (type=budget_exceeded, code=429)", repo.lastErrorMsg, "the account keeps the upstream message with its type and code") }) } } diff --git a/backend/internal/service/openai_ws_forwarder_logutil.go b/backend/internal/service/openai_ws_forwarder_logutil.go index b2b2622baecd..236a1346cb88 100644 --- a/backend/internal/service/openai_ws_forwarder_logutil.go +++ b/backend/internal/service/openai_ws_forwarder_logutil.go @@ -514,6 +514,7 @@ func logOpenAIWSBindResponseAccountWarn(groupID, accountID int64, responseID str "openai.ws_bind_response_account_failed", zap.Int64("group_id", groupID), zap.Int64("account_id", accountID), + zap.String("response_id", truncateOpenAIWSLogValue(responseID, openAIWSIDValueMaxLen)), zap.Bool("response_id_present", strings.TrimSpace(responseID) != ""), zap.String("response_id_kind", normalizeOpenAIWSLogValue(ClassifyOpenAIPreviousResponseIDKind(responseID))), zap.Error(err), diff --git a/backend/internal/service/openai_ws_forwarder_support.go b/backend/internal/service/openai_ws_forwarder_support.go index ce6fa73ef033..beb9ccd0ce3f 100644 --- a/backend/internal/service/openai_ws_forwarder_support.go +++ b/backend/internal/service/openai_ws_forwarder_support.go @@ -166,9 +166,9 @@ func (s *OpenAIGatewayService) performOpenAIWSGeneratePrewarm( return newOpenAIModelNotSupportedFailoverError(lease.HandshakeHeaders(), message) } if canFallback { - return wrapOpenAIWSFallback("prewarm_"+fallbackReason, errors.New(errMsg)) + return wrapOpenAIWSFallback("prewarm_"+fallbackReason, newOpenAIWSUpstreamEventError(errMsg, message)) } - return wrapOpenAIWSFallback("prewarm_error_event", errors.New(errMsg)) + return wrapOpenAIWSFallback("prewarm_error_event", newOpenAIWSUpstreamEventError(errMsg, message)) } if eventType == "response.failed" && account.IsOpenAICompatible() && isOpenAIModelNotSupportedPayload(message) { prewarmModel, _ := reqBody["model"].(string) diff --git a/backend/internal/service/openai_ws_forwarder_v2.go b/backend/internal/service/openai_ws_forwarder_v2.go index 5b07c084b583..0634a9c0b916 100644 --- a/backend/internal/service/openai_ws_forwarder_v2.go +++ b/backend/internal/service/openai_ws_forwarder_v2.go @@ -615,6 +615,9 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2WithScope( logOpenAIWSModeInfo("refusal_recovery_buffer_limit account_id=%d transport=ws_to_sse", account.ID) }, ) + refusalOutput.SetRewriteObserver(func(evidence openAIRefusalEvidence) { + recordOpenAIRefusalRecovery(ctx, c, account, "ws_to_sse", true, false, openAIRefusalActionRewritten, evidence) + }) } // Keep per-read timeouts unchanged for connected clients. Once a client @@ -712,7 +715,7 @@ readLoop: rawEventType, _, _ := parseOpenAIWSEventEnvelope(rawUpstreamMessage) if rawEventType == "error" || rawEventType == "response.failed" { if failoverErr, ok := s.openAIBudgetExceededTerminalFailover( - ctx, account, lease.HandshakeHeaders(), rawUpstreamMessage, + ctx, c, account, lease.HandshakeHeaders(), rawUpstreamMessage, true, !wroteDownstream, ); ok { lease.MarkBroken() bufferedStreamEvents = bufferedStreamEvents[:0] @@ -803,7 +806,13 @@ readLoop: if refusalOutput != nil { refusalOutput.DropTurn() } - return nil, wrapOpenAIWSFallback(string(continuationKind), errors.New("upstream continuation state rejected")) + // Keep the rejected event for Ops; the caller still returns the + // fixed continuation terminal to the client. + failedMessage := extractOpenAISSEErrorMessage(message) + if failedMessage == "" { + failedMessage = "upstream continuation state rejected" + } + return nil, wrapOpenAIWSFallback(string(continuationKind), newOpenAIWSUpstreamEventError(failedMessage, message)) } if !wroteDownstream && account.IsOpenAICompatible() && isOpenAIModelNotSupportedPayload(message) { // This is an account/model capability response carried over an @@ -919,7 +928,7 @@ readLoop: return nil, newOpenAIModelNotSupportedFailoverError(lease.HandshakeHeaders(), message) } if !wroteDownstream && canFallback { - return nil, wrapOpenAIWSFallback(fallbackReason, errors.New(errMsg)) + return nil, wrapOpenAIWSFallback(fallbackReason, newOpenAIWSUpstreamEventError(errMsg, message)) } statusCode := openAIWSErrorHTTPStatusFromRaw(errCodeRaw, errTypeRaw) setOpsUpstreamError(c, statusCode, errMsg, "") @@ -1021,12 +1030,12 @@ readLoop: } finalResponse = s.correctToolCallsInResponseBody(finalResponse) if refusalRuntime.RewriteEnabled() { - rewritten, matched, _, rewriteErr := RewriteOpenAIResponsesJSON(finalResponse, refusalRuntime.Matcher) + rewritten, matched, evidence, rewriteErr := rewriteOpenAIResponsesJSONWithEvidence(finalResponse, refusalRuntime.Matcher) if rewriteErr != nil { logger.FromContext(ctx).Warn("openai.refusal_recovery_rewrite_failed", zap.String("transport", "upstream_websocket"), zap.Error(rewriteErr)) } else if matched { finalResponse = rewritten - logger.FromContext(ctx).Info("openai.refusal_recovery_rewritten", zap.String("transport", "upstream_websocket")) + recordOpenAIRefusalRecovery(ctx, c, account, "upstream_websocket", true, false, openAIRefusalActionRewritten, evidence) } } populateOpenAIUsageFromResponseJSON(finalResponse, usage) diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 066709166e43..5c27edfc9cd8 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -614,7 +614,11 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, true) } safeErr := sanitizeUpstreamErrorMessage(err.Error()) - _ = writeClientMessage(buildOpenAIWSHTTPBridgeErrorEvent(http.StatusBadGateway, "Upstream request failed")) + recordOpenAIWSBridgeTransportError(c, account, safeErr) + clientError := buildOpenAIWSHTTPBridgeErrorEvent(http.StatusBadGateway, "Upstream request failed") + if writeErr := writeClientMessage(clientError); writeErr == nil && !openAIWSDeliveryMarksFailures(c) { + markOpenAIWSDeliveredFailure(c, clientError) + } return nil, fmt.Errorf("upstream http bridge request failed: %s", safeErr) } rejectedFieldRetryState := newOpenAIResponsesRejectedFieldRetryState(body) @@ -677,7 +681,10 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( canonicalModel := canonicalOpenAIAccountSchedulingModel(account, originalModel) s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody, canonicalModel) } - _ = writeClientMessage(buildOpenAIWSHTTPBridgeErrorEvent(resp.StatusCode, upstreamMsg)) + clientError := buildOpenAIWSHTTPBridgeErrorEvent(resp.StatusCode, upstreamMsg) + if writeErr := writeClientMessage(clientError); writeErr == nil && !openAIWSDeliveryMarksFailures(c) { + markOpenAIWSDeliveredFailure(c, clientError) + } return nil, fmt.Errorf("upstream http bridge error: status=%d message=%s", resp.StatusCode, upstreamMsg) } defer func() { _ = resp.Body.Close() }() @@ -787,7 +794,8 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } wroteDownstream = true } - markOpenAIWSClientVisibleFailure(c, "response.failed", clientMessage) + // A cyber-policy block keeps only its dedicated row. + markOpenAIWSDeliveredFailure(c, clientMessage) return nil } @@ -832,7 +840,8 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( budgetExceeded := false var budgetFailoverErr *UpstreamFailoverError if rawEventType == "error" || rawEventType == "response.failed" { - budgetFailoverErr, budgetExceeded = s.openAIBudgetExceededTerminalFailover(ctx, account, resp.Header, rawUpstreamMessage) + budgetFailoverErr, budgetExceeded = s.openAIBudgetExceededTerminalFailover(ctx, c, account, resp.Header, rawUpstreamMessage, false, + turn == 1 && !wroteDownstream && !semanticOutputStarted) } if budgetExceeded { if turn == 1 && !wroteDownstream && !semanticOutputStarted { @@ -1012,7 +1021,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( pendingClientMessageBytes = 0 } messages = append(messages, clientMessage) - for _, message := range messages { + for index, message := range messages { if err := writeClientMessage(message); err != nil { if isOpenAIWSClientDisconnectError(err) { clientDisconnected = true @@ -1035,6 +1044,17 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( if !isKeepalive { wroteDownstream = true } + // Mark failures the client received, staged frames included + // when they are flushed; the current frame is marked with the + // upstream payload, as upstream v0.2.8 does. A refusal output + // marks on its own delivery instead. + if !openAIWSDeliveryMarksFailures(c) { + delivered := message + if index == len(messages)-1 { + delivered = upstreamMessage + } + markOpenAIWSDeliveredFailure(c, delivered) + } } } if startsSemanticOutput && !clientDisconnected { diff --git a/backend/internal/service/openai_ws_http_bridge_test.go b/backend/internal/service/openai_ws_http_bridge_test.go index ea7372115fa5..80cbd5c381d8 100644 --- a/backend/internal/service/openai_ws_http_bridge_test.go +++ b/backend/internal/service/openai_ws_http_bridge_test.go @@ -816,9 +816,15 @@ func TestProxyOpenAIWSHTTPBridgeTurnTransportErrorFailoverSafety(t *testing.T) { require.False(t, errors.As(err, &failoverErr)) } require.Len(t, writes, tt.wantWrites) + // A delivered error event is a client-visible failure (upstream + // v0.2.8); a hidden failover is not. + streamErr, marked := GetOpsStreamError(c) + require.Equal(t, tt.wantWrites > 0, marked) if tt.wantWrites > 0 { require.Equal(t, "error", gjson.GetBytes(writes[0], "type").String()) require.Equal(t, int64(http.StatusBadGateway), gjson.GetBytes(writes[0], "status").Int()) + require.True(t, streamErr.CountTowardsSLA) + require.Equal(t, http.StatusBadGateway, streamErr.IntendedStatus) } }) } @@ -873,6 +879,12 @@ func TestProxyOpenAIWSHTTPBridgeTurnHTTPStatusFailoverSafety(t *testing.T) { require.False(t, errors.As(err, &failoverErr)) } require.Len(t, writes, tt.wantWrites) + streamErr, marked := GetOpsStreamError(c) + require.Equal(t, tt.wantWrites > 0, marked) + if tt.wantWrites > 0 { + require.Equal(t, "temporary upstream failure", streamErr.Message) + require.Equal(t, tt.status, streamErr.IntendedStatus) + } }) } } @@ -1052,7 +1064,7 @@ func TestProxyOpenAIWSHTTPBridgeTurnBudgetExceededWriteOrdering(t *testing.T) { require.NotContains(t, string(written), "sensitive upstream detail") } require.Equal(t, 1, repo.setErrorCalls, "a budget terminal puts the account into the error state") - require.Equal(t, "sensitive upstream detail", repo.lastErrorMsg, "the account keeps the upstream message") + require.Equal(t, "sensitive upstream detail (type=budget_exceeded, code=429)", repo.lastErrorMsg, "the account keeps the upstream message with its type and code") }) } } diff --git a/backend/internal/service/openai_ws_v2_passthrough_adapter.go b/backend/internal/service/openai_ws_v2_passthrough_adapter.go index 558dee0aeb76..15fb1d46cde6 100644 --- a/backend/internal/service/openai_ws_v2_passthrough_adapter.go +++ b/backend/internal/service/openai_ws_v2_passthrough_adapter.go @@ -1267,6 +1267,11 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( logOpenAIWSV2Passthrough("refusal_recovery_buffer_limit account_id=%d transport=websocket", account.ID) }, ) + refusalOutput.SetRewriteObserver(func(evidence openAIRefusalEvidence) { + recordOpenAIRefusalRecovery(ctx, c, account, "websocket", true, false, openAIRefusalActionRewritten, evidence) + }) + // The output may hold or replace frames; mark failures on delivery. + refusalOutput.SetDeliveryObserver(func(payload []byte) { markOpenAIWSDeliveredFailure(c, payload) }) relayClientConn = &openAIRefusalRecoveryWSFrameConn{inner: policyClientConn, output: refusalOutput} } upstreamFirstMessageSent := false @@ -1389,6 +1394,11 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( } }, AfterClientWrite: func(msgType coderws.MessageType, payload []byte, writeErr error) { + // Upstream v0.2.8 marks delivered error/response.failed frames. + // With refusal recovery the output marks them on delivery. + if msgType == coderws.MessageText && writeErr == nil && refusalOutput == nil { + markOpenAIWSDeliveredFailure(c, payload) + } if msgType == coderws.MessageText && openAIWSPassthroughIsTerminalOutput(payload) { turnLifecycle.finishTerminalWrite(writeErr == nil, clientFrameConn.markTurnCompleted) } @@ -1418,7 +1428,8 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( } if eventType == "error" || eventType == "response.failed" { if failoverErr, ok := s.openAIBudgetExceededTerminalFailover( - ctx, account, handshakeHeaders, payload, + ctx, c, account, handshakeHeaders, payload, true, + completedTurns.Load() == 0 && (refusalOutput == nil || !refusalOutput.SemanticOutputStarted()), ); ok { replaySafe := completedTurns.Load() == 0 && (refusalOutput == nil || !refusalOutput.SemanticOutputStarted()) if refusalOutput != nil { @@ -1456,6 +1467,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( model = canonicalOpenAIAccountSchedulingModel(account, requestModel) } _ = s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusBadRequest, nil, payload, model) + s.recordOpenAIWSModelNotSupportedAttempt(c, account, handshakeHeaders, payload) return failoverErr } if (eventType == "error" || eventType == "response.failed") && markOpenAIWSV2PassthroughCyberPolicy(c, payload) { diff --git a/backend/internal/service/ops_models.go b/backend/internal/service/ops_models.go index a6830cadc0ce..053c3b05e16f 100644 --- a/backend/internal/service/ops_models.go +++ b/backend/internal/service/ops_models.go @@ -121,6 +121,9 @@ type OpsErrorLogFilter struct { // Optional correlation keys for exact matching. RequestID string ClientRequestID string + // ExcludeClientRequestIDPrefix drops rows whose client request ID starts + // with it (the user-facing list hides admin-only Image Studio rows). + ExcludeClientRequestIDPrefix string // User-scoped filters (used by the user-facing error requests endpoint and // by admin drill-down from the usage page). diff --git a/backend/internal/service/ops_service.go b/backend/internal/service/ops_service.go index 546181c2d327..afed559416a5 100644 --- a/backend/internal/service/ops_service.go +++ b/backend/internal/service/ops_service.go @@ -590,11 +590,9 @@ func sanitizeOpsUpstreamErrors(entry *OpsInsertErrorLogInput) error { // Only boundOpsUpstreamErrors may stamp this; never trust caller input. out.DroppedEarlierAttempts = 0 keepBody := i >= firstEventWithBody - if keepBody { - out.ContinuationDiagnostic = sanitizeOpenAIContinuationDiagnostic(ev.ContinuationDiagnostic) - } else { - out.ContinuationDiagnostic = nil - } + // Continuation diagnostics are self-bounded and stay on every retained + // attempt; boundOpsUpstreamErrors still enforces the entry byte budget. + out.ContinuationDiagnostic = sanitizeOpenAIContinuationDiagnostic(ev.ContinuationDiagnostic) urlMaxLen, messageMaxLen := 2048, 2048 if !keepBody { urlMaxLen, messageMaxLen = opsUpstreamErrorsOlderURLMaxLen, opsUpstreamErrorsOlderMessageMaxLen @@ -730,6 +728,9 @@ func (s *OpsService) ListUserErrorRequests(ctx context.Context, userID int64, fi //（error_phase='upstream' 但 status<400,最终成功返回）记录对用户不可见——符合预期。 filter.Phase = "" filter.IncludeRecoveredUpstream = false + // Image Studio gateway rows keep the upstream response for administrators; + // the Image Studio job itself shows the user its message. + filter.ExcludeClientRequestIDPrefix = ImageStudioClientRequestIDPrefix list, err := s.opsRepo.ListErrorLogs(ctx, filter) if err != nil { @@ -789,7 +790,7 @@ func (s *OpsService) GetUserErrorRequestDetail(ctx context.Context, userID, id i } // 归属只能由通过鉴权时写入的 user_id 确定。 ownedDirectly := detail.UserID != nil && *detail.UserID == userID - if !ownedDirectly { + if !ownedDirectly || strings.HasPrefix(detail.ClientRequestID, ImageStudioClientRequestIDPrefix) { return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found") } return ToUserErrorRequestDetail(detail), nil diff --git a/backend/internal/service/ops_service_user_error_test.go b/backend/internal/service/ops_service_user_error_test.go index e92f149a5261..7119024fa36f 100644 --- a/backend/internal/service/ops_service_user_error_test.go +++ b/backend/internal/service/ops_service_user_error_test.go @@ -66,6 +66,10 @@ func TestListUserErrorRequests_ForcesScopeAndRedacts(t *testing.T) { if stub.gotFilter.APIKeyID == nil || *stub.gotFilter.APIKeyID != kid { t.Fatalf("APIKeyID should be preserved, got %v", stub.gotFilter.APIKeyID) } + // Image Studio gateway rows stay admin-only. + if stub.gotFilter.ExcludeClientRequestIDPrefix != ImageStudioClientRequestIDPrefix { + t.Fatalf("Image Studio rows not excluded: %q", stub.gotFilter.ExcludeClientRequestIDPrefix) + } // 调用方传入的 filter 不应被原地篡改（验证 shallow copy 隔离生效） if in.View != "errors" || in.UserID != nil || in.Phase != "upstream" { t.Fatalf("caller filter was mutated: View=%q UserID=%v Phase=%q", in.View, in.UserID, in.Phase) @@ -163,3 +167,17 @@ func TestGetUserErrorRequestDetail_InvalidID(t *testing.T) { t.Fatal("expected error for id=-5") } } + +// An Image Studio gateway row keeps the upstream response for administrators; +// its owner does not get it from the error-request detail endpoint. +func TestGetUserErrorRequestDetail_HidesImageStudioRows(t *testing.T) { + owner := int64(5) + stub := &stubOpsRepoForUserErr{detailToReturn: &OpsErrorLogDetail{OpsErrorLog: OpsErrorLog{ + ID: 9, UserID: &owner, StatusCode: 502, ClientRequestID: ImageStudioClientRequestIDPrefix + "job-1-item-2-0a1b2c3d4e5f", + }}} + svc := &OpsService{opsRepo: stub} + got, err := svc.GetUserErrorRequestDetail(context.Background(), owner, 9) + if got != nil || !infraerrors.IsNotFound(err) { + t.Fatalf("expected NotFound for an Image Studio row, got %+v, %v", got, err) + } +} diff --git a/backend/internal/service/ops_upstream_context.go b/backend/internal/service/ops_upstream_context.go index 3435d984b4ee..c920b7f08dda 100644 --- a/backend/internal/service/ops_upstream_context.go +++ b/backend/internal/service/ops_upstream_context.go @@ -353,6 +353,27 @@ func GetOpsStreamErrors(c *gin.Context) []OpsStreamError { return nil } +// IsGatewayFixedClientMessage reports whether message is a sentence the gateway +// writes to clients in place of an upstream error text. Ops never records one +// as the upstream's own message. +func IsGatewayFixedClientMessage(message string) bool { + switch strings.TrimSpace(message) { + case OpenAIRequestRejectedClientMessage, + OpenAIContinuationStateUnavailableClientMessage, + OpenAIModelNotSupportedClientMessage, + OpenAIRequestBodyTooLargeClientMessage, + GrokCredentialUnavailableClientMessage, + AntigravityCredentialRejectedClientMessage, + openAISilentRefusalClientMessage, + openAIUpstreamAccessUnavailableClientMessage, + geminiCustomCodeSkippedClientMessage, + "Temporary upstream failure", + "Upstream failed to process the request": + return true + } + return false +} + // SetOpsUpstreamError is the exported wrapper for setOpsUpstreamError, used by // handler-layer code (e.g. failover-exhausted paths) that needs to record the // original upstream status code before mapping it to a client-facing code. @@ -422,8 +443,10 @@ type OpsUpstreamErrorEvent struct { Message string `json:"message,omitempty"` Detail string `json:"detail,omitempty"` - // Bounded, content-free metadata. Never use this for passthrough-rule - // matching or to decide whether a request can be retried. + // Administrator diagnostics: complete-value fingerprints plus bounded + // original values (IDs, instructions, the upstream error text). Never use + // this for passthrough-rule matching or to decide whether a request can be + // retried. ContinuationDiagnostic *OpenAIContinuationDiagnostic `json:"continuation_diagnostic,omitempty"` // SkipMonitoring is request-local rule state. It is intentionally excluded @@ -441,6 +464,46 @@ const ( opsProxyNameUnnamed = "proxy" ) +// OpsRefusalRecoveryEventsKey holds the refusal evidence behind HTTP refusal +// rewrites and prompt retries. It is not upstream error context: nothing +// classifies on it, a request that succeeds leaves no Ops row for it, and the +// Ops logger attaches it only to a request that finally fails. +const OpsRefusalRecoveryEventsKey = "ops_refusal_recovery_events" + +// opsRefusalRecoveryEventsMax bounds the evidence kept for one request. +const opsRefusalRecoveryEventsMax = 8 + +func appendOpsRefusalRecoveryEvent(c *gin.Context, ev OpsUpstreamErrorEvent) { + if c == nil { + return + } + events := OpsRefusalRecoveryEvents(c) + if len(events) >= opsRefusalRecoveryEventsMax { + return + } + if ev.AtUnixMs <= 0 { + ev.AtUnixMs = time.Now().UnixMilli() + } + normalizeOpsUpstreamProxyAttribution(&ev) + ev.Message = sanitizeUpstreamErrorMessage(strings.TrimSpace(ev.Message)) + ev.Detail = strings.TrimSpace(ev.Detail) + evCopy := ev + c.Set(OpsRefusalRecoveryEventsKey, append(events, &evCopy)) +} + +// OpsRefusalRecoveryEvents returns the refusal evidence recorded for the request. +func OpsRefusalRecoveryEvents(c *gin.Context) []*OpsUpstreamErrorEvent { + if c == nil { + return nil + } + if v, ok := c.Get(OpsRefusalRecoveryEventsKey); ok { + if events, ok := v.([]*OpsUpstreamErrorEvent); ok { + return events + } + } + return nil +} + func appendOpsUpstreamError(c *gin.Context, ev OpsUpstreamErrorEvent) { if c == nil { return diff --git a/backend/internal/service/ratelimit_service.go b/backend/internal/service/ratelimit_service.go index 8110f7a9f264..cd5c0e3fa693 100644 --- a/backend/internal/service/ratelimit_service.go +++ b/backend/internal/service/ratelimit_service.go @@ -2329,7 +2329,7 @@ func (s *RateLimitService) HandleOpenAIImageRateLimit(ctx context.Context, accou } resetAt := openAIImageRateLimitResetAt(headers, responseBody) - if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageRateLimitReason); err != nil { + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageRateLimitReason, modelRateLimitUpstreamMessage(responseBody)); err != nil { slog.Warn("openai_image_rate_limit_set_model_rate_limit_failed", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "error", err) return true } @@ -2367,7 +2367,7 @@ func (s *RateLimitService) HandleOpenAICodexSparkRateLimit(ctx context.Context, reset := now.Add(cooldown) resetAt = &reset } - if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *resetAt, openAICodexSparkRateLimitReason); err != nil { + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *resetAt, openAICodexSparkRateLimitReason, modelRateLimitUpstreamMessage(responseBody)); err != nil { slog.Warn("openai_codex_spark_model_rate_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err) } slog.Info("openai_codex_spark_model_rate_limited", "account_id", account.ID, "model", modelKey, "reset_at", *resetAt) @@ -2390,7 +2390,7 @@ func (s *RateLimitService) HandleOpenAIImageCapabilityLoss(ctx context.Context, } resetAt := time.Now().Add(openAIImageCapabilityLossCooldown) - if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageCapabilityLossReason); err != nil { + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageCapabilityLossReason, modelRateLimitUpstreamMessage(responseBody)); err != nil { slog.Warn("openai_image_capability_loss_set_model_rate_limit_failed", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "error", err) return true } @@ -2499,6 +2499,21 @@ const upstreamCodexPlanGatedModelReason = "upstream_400_codex_plan_gated_model" const tempUnschedBodyMaxBytes = 64 << 10 const tempUnschedMessageMaxBytes = 2048 +// ModelRateLimitUpstreamMessageKey stores the upstream's own error text in a +// model_rate_limits entry next to its reason code. SetModelRateLimit takes it +// as the second reason value. Administrators read it; reason matching and +// client classification only ever read "reason". +const ModelRateLimitUpstreamMessageKey = "upstream_message" + +// modelRateLimitUpstreamMessage is the upstream error text kept with a model +// cooldown: the extracted message, or the bounded body when it carries none. +func modelRateLimitUpstreamMessage(body []byte) string { + if message := openAIOpsUpstreamErrorMessage(body); message != "" { + return truncateString(message, tempUnschedMessageMaxBytes) + } + return strings.TrimSpace(sanitizeUpstreamErrorMessage(truncateString(strings.TrimSpace(string(body)), tempUnschedMessageMaxBytes))) +} + // HandleUpstreamModelNotFound marks the requested model as temporarily // unavailable on the account when the upstream deterministically reports it // cannot serve that model: a 404 model-not-found, or the Codex 400 rejecting a @@ -2537,11 +2552,17 @@ func (s *RateLimitService) HandleUpstreamModelNotFound(ctx context.Context, acco return true } resetAt := time.Now().Add(cooldown) - if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, resetAt, reason); err != nil { + // The persisted reason stays the exact code the availability diagnosis + // matches on; the upstream's own words are stored (upstream_message) and + // logged next to it. + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, resetAt, reason, modelRateLimitUpstreamMessage(responseBody)); err != nil { slog.Warn("upstream_model_not_found_set_model_rate_limit_failed", "account_id", account.ID, "model", modelKey, "reason", reason, "error", err) return true } - slog.Info("upstream_model_not_found_model_rate_limited", "account_id", account.ID, "model", modelKey, "reason", reason, "reset_at", resetAt) + slog.Info("upstream_model_not_found_model_rate_limited", "account_id", account.ID, "model", modelKey, "reason", reason, "reset_at", resetAt, + "upstream_status", statusCode, + "upstream_message", extractOpenAISSEErrorMessage(responseBody), + "upstream_body", truncateString(string(responseBody), tempUnschedMessageMaxBytes)) return true } diff --git a/backend/internal/service/ratelimit_service_model_not_found_test.go b/backend/internal/service/ratelimit_service_model_not_found_test.go index 9e2baa3b09dc..acaf29587a6f 100644 --- a/backend/internal/service/ratelimit_service_model_not_found_test.go +++ b/backend/internal/service/ratelimit_service_model_not_found_test.go @@ -14,10 +14,11 @@ import ( ) type modelNotFoundRateLimitCall struct { - accountID int64 - scope string - resetAt time.Time - reason string + accountID int64 + scope string + resetAt time.Time + reason string + upstreamMessage string } type modelNotFoundAccountRepoStub struct { @@ -41,6 +42,9 @@ func (r *modelNotFoundAccountRepoStub) SetModelRateLimit(ctx context.Context, id if len(reason) > 0 { call.reason = reason[0] } + if len(reason) > 1 { + call.upstreamMessage = reason[1] + } r.modelRateLimitCalls = append(r.modelRateLimitCalls, call) return r.modelRateLimitErr } @@ -66,6 +70,7 @@ func TestRateLimitService_HandleUpstreamError_ModelNotFoundUsesModelRateLimit(t require.Equal(t, account.ID, call.accountID) require.Equal(t, "gpt-5.4", call.scope) require.Equal(t, upstreamModelNotFoundReason, call.reason) + require.Equal(t, "model not found", call.upstreamMessage) require.WithinDuration(t, time.Now().Add(upstreamModelNotFoundCooldown), call.resetAt, 5*time.Second) } diff --git a/backend/migrations/262_account_job_error_text.sql b/backend/migrations/262_account_job_error_text.sql new file mode 100644 index 000000000000..e63a605b2ae1 --- /dev/null +++ b/backend/migrations/262_account_job_error_text.sql @@ -0,0 +1,13 @@ +-- Account jobs keep the original error text of a failed job or item. Upstream +-- responses and connection errors are often longer than 512 characters, and a +-- rejected write left the job or item without its result. VARCHAR(512) to TEXT +-- only changes the catalog: no table rewrite and no index on these columns. +-- Running it again is a no-op. +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '120s'; + +ALTER TABLE admin_account_jobs + ALTER COLUMN error_message TYPE TEXT; + +ALTER TABLE admin_account_job_items + ALTER COLUMN error_message TYPE TEXT; diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 6479b609bc66..8cc022c45cf5 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -193,7 +193,7 @@ security: # 默认 CSP 策略（如果静态资源托管在其他域名，请自行覆盖） # Note: __CSP_NONCE__ will be replaced with 'nonce-xxx' at request time for inline script security # 注意：__CSP_NONCE__ 会在请求时被替换为 'nonce-xxx'，用于内联脚本安全 - policy: "default-src 'self'; worker-src 'self' blob:; script-src 'self' __CSP_NONCE__ https://challenges.cloudflare.com https://*.alicdn.com https://static.cloudflareinsights.com https://turing.captcha.qcloud.com https://turing.captcha.gtimg.com https://ca.turing.captcha.qcloud.com https://global.turing.captcha.gtimg.com https://www.tycaptcha.com https://cloudcache.tencentcs.com https://*.stripe.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; style-src 'self' 'unsafe-inline' https://*.captcha.gtimg.com https://fonts.googleapis.com https://*.alicdn.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; img-src 'self' data: blob: https:; font-src 'self' data: https://fonts.gstatic.com; connect-src 'self' https://turing.captcha.qcloud.com https://www.tycaptcha.com https://rce.tencentrio.com https:; frame-src 'self' https://challenges.cloudflare.com https://turing.captcha.qcloud.com https://ca.turing.captcha.qcloud.com https://www.tycaptcha.com https://*.stripe.com https://checkout.airwallex.com https://checkout-demo.airwallex.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" + policy: "default-src 'self'; worker-src 'self' blob:; script-src 'self' __CSP_NONCE__ https://challenges.cloudflare.com https://*.alicdn.com https://static.cloudflareinsights.com https://turing.captcha.qcloud.com https://turing.captcha.gtimg.com https://ca.turing.captcha.qcloud.com https://global.turing.captcha.gtimg.com https://www.tycaptcha.com https://cloudcache.tencentcs.com https://*.stripe.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; style-src 'self' 'unsafe-inline' https://*.captcha.gtimg.com https://fonts.googleapis.com https://font.sec.miui.com https://*.alicdn.com https://static.airwallex.com https://checkout.airwallex.com https://static-demo.airwallex.com https://checkout-demo.airwallex.com; img-src 'self' data: blob: https:; font-src 'self' data: https://fonts.gstatic.com https://cdn-file.hyperos.mi.com; connect-src 'self' https://turing.captcha.qcloud.com https://www.tycaptcha.com https://rce.tencentrio.com https:; frame-src 'self' https://challenges.cloudflare.com https://turing.captcha.qcloud.com https://ca.turing.captcha.qcloud.com https://www.tycaptcha.com https://*.stripe.com https://checkout.airwallex.com https://checkout-demo.airwallex.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" proxy_probe: # Allow skipping TLS verification for proxy probe (debug only) # 允许代理探测时跳过 TLS 证书验证（仅用于调试） diff --git a/frontend/src/__tests__/integration/data-import.spec.ts b/frontend/src/__tests__/integration/data-import.spec.ts index b746f8405085..cd7a480c8812 100644 --- a/frontend/src/__tests__/integration/data-import.spec.ts +++ b/frontend/src/__tests__/integration/data-import.spec.ts @@ -217,7 +217,7 @@ describe('account data import job', () => { await selectFiles(wrapper, [jsonFile('broken.json', '{')]) expect(showError).toHaveBeenCalledTimes(1) - expect(showError).toHaveBeenCalledWith('admin.accounts.dataImportParseFailedFile') + expect(showError).toHaveBeenCalledWith(expect.stringMatching(/^admin\.accounts\.dataImportParseFailedFile: \S/)) expect(wrapper.find('[data-test="import-preview"]').exists()).toBe(false) expect(wrapper.get('[data-test="submit-import-job"]').attributes('disabled')).toBeDefined() await wrapper.get('#account-import-job-form').trigger('submit') @@ -267,7 +267,8 @@ describe('account data import job', () => { await selectFiles(wrapper, [jsonFile('broken.json', '{')]) - expect(showError).toHaveBeenCalledWith('admin.accounts.dataImportParseFailedFile') + // The JSON.parse error text follows the label. + expect(showError).toHaveBeenCalledWith(expect.stringMatching(/^admin\.accounts\.dataImportParseFailedFile: \S/)) expect(importData).not.toHaveBeenCalled() expect(wrapper.get('[data-test="submit-import-job"]').attributes('disabled')).toBeDefined() }) @@ -277,7 +278,7 @@ describe('account data import job', () => { await selectFiles(wrapper, [jsonFile('random.json', { name: 'not-an-export' })]) - expect(showError).toHaveBeenCalledWith('admin.accounts.dataImportInvalidFile') + expect(showError).toHaveBeenCalledWith('admin.accounts.dataImportInvalidFile: accounts must be an array') expect(importData).not.toHaveBeenCalled() }) diff --git a/frontend/src/api/admin/codexRuntime.ts b/frontend/src/api/admin/codexRuntime.ts index ed0f78f9d44c..2af1327a14b3 100644 --- a/frontend/src/api/admin/codexRuntime.ts +++ b/frontend/src/api/admin/codexRuntime.ts @@ -2,6 +2,9 @@ import { apiClient } from '../client' export interface CodexRuntimeConfig extends Record { enabled?: boolean + fail_closed?: boolean + models?: string[] + routing_schema?: number request_zstd?: boolean proxy_url?: string proxy_protocol?: string diff --git a/frontend/src/api/admin/codexTickets.ts b/frontend/src/api/admin/codexTickets.ts index 11889dee02c6..294970bf9d5d 100644 --- a/frontend/src/api/admin/codexTickets.ts +++ b/frontend/src/api/admin/codexTickets.ts @@ -1,22 +1,37 @@ import { apiClient } from '../client' import { accountJobIdempotencyHeaders, type AccountJob } from './accountJobs' +/** A recorded Codex routing observation, including original error text. */ +export interface RoutingObservation { + stage: string; code: string; http_status?: number; requested_model?: string; reasoning_effort?: string; response_model?: string + completed: boolean; model_matched: boolean; state_length?: number; cookie_names?: string[]; observed_at: string + duration_ms?: number; transport?: string; cookie_sent: boolean; error?: string + upstream_error_type?: string; upstream_error_code?: string; upstream_error_message?: string; upstream_error_param?: string + upstream_body?: string; request_id?: string; cf_ray?: string + state?: string; response_headers?: Array<{ name: string; value: string }>; response_headers_omitted?: string[] +} export interface TicketResult { stage?: string - code: string; success: boolean; message?: string; http_status?: number; observed_length?: number - duration_ms?: number; expires_at?: string; fingerprint?: string + code: string; success: boolean; message?: string; error?: string; http_status?: number; observed_length?: number + response_model?: string; duration_ms?: number; expires_at?: string; fingerprint?: string; observation?: RoutingObservation +} +export interface ProxyCertificate { + server_name?: string; subject: string; issuer: string; serial_number?: string; not_before: string; not_after: string + dns_names?: string[]; sha256: string; spki_sha256: string } export interface ProxyTestResult { failure_detail?: string success: boolean; network_reachable: boolean; protocol: string; http_status?: number; code: string; message: string stages: Array<{ name: string; success: boolean; duration_ms: number; message?: string }> certificate_trust?: string; certificate_fingerprint?: string; protocol_suggestion?: string + certificates?: ProxyCertificate[]; certificate_error?: string } export interface ProxyCandidate { selection_id: string protocol: string host: string port: number + username?: string username_masked?: string source_line: number source_column: number diff --git a/frontend/src/api/admin/plugins.ts b/frontend/src/api/admin/plugins.ts index 1bdd7a0e83a3..361a9673f080 100644 --- a/frontend/src/api/admin/plugins.ts +++ b/frontend/src/api/admin/plugins.ts @@ -75,6 +75,43 @@ export interface PluginInstallation { runtime_message: string } +// Read-only view of a first-party installation the host retired when the +// feature became built in. It is never part of the managed plugin list. +export interface RetiredPluginInstallation { + id: number + plugin_key: string + name: string + version: string + description: string + author: string + manifest: PluginManifest + artifact_path: string + install_path: string + binary_path: string + binary_sha256: string + signature_status: string + state: string + last_error: string + installed_by?: number | null + installed_at: string + enabled_at?: string | null + updated_at: string + bindings: PluginBinding[] + config?: unknown + config_text?: string + config_error?: string +} + +export interface RetiredPluginsView { + receipt: { + version: number + completed: boolean + retired_at: string + plugins: Record + } | null + installations: RetiredPluginInstallation[] +} + export interface PluginTestResult { success: boolean message: string @@ -100,6 +137,11 @@ export async function list(): Promise { return data } +export async function listRetired(): Promise { + const { data } = await apiClient.get('/admin/plugins/retired') + return data +} + export async function upload(file: File): Promise { const form = new FormData() form.append('plugin', file) @@ -161,6 +203,7 @@ export async function createUISession(id: number): Promise { export default { list, + listRetired, upload, enable, disable, diff --git a/frontend/src/api/admin/settings.ts b/frontend/src/api/admin/settings.ts index 812c1cef6917..bfff070925ba 100644 --- a/frontend/src/api/admin/settings.ts +++ b/frontend/src/api/admin/settings.ts @@ -788,6 +788,9 @@ export interface SystemSettings { // Allow user view error requests allow_user_view_error_requests: boolean; + + // Retired switches, echoed read-only by GET (never written): setting key -> stored value + deprecated_settings?: Record; } export interface UpdateSettingsRequest { diff --git a/frontend/src/components/account/AccountStatusIndicator.vue b/frontend/src/components/account/AccountStatusIndicator.vue index ea3803786576..7a6c6e1882b8 100644 --- a/frontend/src/components/account/AccountStatusIndicator.vue +++ b/frontend/src/components/account/AccountStatusIndicator.vue @@ -9,7 +9,7 @@ -
+
{{ t('admin.accounts.status.rateLimited') }} {{ rateLimitResumeText }}
@@ -99,49 +99,61 @@ : 'columns-3 gap-x-2' ]" > -
- - - - {{ t('admin.accounts.status.creditsExhausted') }} - {{ formatCountdown(item.reset_at) }} - - - - ⚡ - {{ formatScopeName(item.model) }} - {{ formatCountdown(item.reset_at) }} - - - - - {{ formatScopeName(item.model) }} - {{ formatCountdown(item.reset_at) }} - - -
- {{ - item.kind === 'credits_exhausted' - ? t('admin.accounts.status.creditsExhaustedUntil', { time: formatDateTimeToMinute(item.reset_at) }) - : item.kind === 'credits_active' - ? t('admin.accounts.status.modelCreditOveragesUntil', { model: formatScopeName(item.model), time: formatDateTimeToMinute(item.reset_at) }) - : t('admin.accounts.status.modelRateLimitedUntil', { model: formatScopeName(item.model), time: formatDateTimeToMinute(item.reset_at) }) - }} -
-
+
+ + + +
+
+ {{ + item.kind === 'credits_exhausted' + ? t('admin.accounts.status.creditsExhaustedUntil', { time: formatDateTimeToMinute(item.reset_at) }) + : item.kind === 'credits_active' + ? t('admin.accounts.status.modelCreditOveragesUntil', { model: formatScopeName(item.model), time: formatDateTimeToMinute(item.reset_at) }) + : t('admin.accounts.status.modelRateLimitedUntil', { model: formatScopeName(item.model), time: formatDateTimeToMinute(item.reset_at) }) + }} +
+
+ {{ t('admin.accounts.status.modelCooldownReason', { reason: item.reason }) }} +
+ +
+
@@ -170,6 +182,7 @@ import { computed, ref } from 'vue' import { useIntervalFn } from '@vueuse/core' import { useI18n } from 'vue-i18n' +import InfoPopover from '@/components/common/InfoPopover.vue' import Icon from '@/components/icons/Icon.vue' import type { Account } from '@/types' import { formatCountdown, formatDateTime, formatDateTimeToMinute, formatCountdownWithSuffix, formatTime } from '@/utils/format' @@ -211,13 +224,16 @@ type AccountModelStatusItem = { kind: 'rate_limit' | 'credits_exhausted' | 'credits_active' model: string reset_at: string + // the stored reason code and the upstream's own error text (model_rate_limits.) + reason?: string + upstream_message?: string } // Computed: active model statuses (普通模型限流 + 积分耗尽 + 走积分中) const activeModelStatuses = computed(() => { const extra = props.account.extra as Record | undefined const modelLimits = extra?.model_rate_limits as - | Record + | Record | undefined const now = new Date() const items: AccountModelStatusItem[] = [] @@ -231,16 +247,17 @@ const activeModelStatuses = computed(() => { for (const [model, info] of Object.entries(modelLimits)) { if (new Date(info.rate_limit_reset_at) <= now) continue + const recorded = { reason: info.reason, upstream_message: info.upstream_message } if (model === 'AICredits') { // AICredits key → 积分已用尽 - items.push({ kind: 'credits_exhausted', model, reset_at: info.rate_limit_reset_at }) + items.push({ kind: 'credits_exhausted', model, reset_at: info.rate_limit_reset_at, ...recorded }) } else if (allowOverages && !hasActiveAICredits) { // 普通模型限流 + overages 启用 + 积分可用 → 正在走积分 - items.push({ kind: 'credits_active', model, reset_at: info.rate_limit_reset_at }) + items.push({ kind: 'credits_active', model, reset_at: info.rate_limit_reset_at, ...recorded }) } else { // 普通模型限流 - items.push({ kind: 'rate_limit', model, reset_at: info.rate_limit_reset_at }) + items.push({ kind: 'rate_limit', model, reset_at: info.rate_limit_reset_at, ...recorded }) } } diff --git a/frontend/src/components/account/AccountTestModal.vue b/frontend/src/components/account/AccountTestModal.vue index f4de70e54e86..e6602f1e2579 100644 --- a/frontend/src/components/account/AccountTestModal.vue +++ b/frontend/src/components/account/AccountTestModal.vue @@ -81,7 +81,7 @@
@@ -211,9 +211,9 @@ status === 'connecting' || !selectedModelId ? 'cursor-not-allowed bg-primary-400 text-white' : status === 'success' - ? 'bg-green-500 text-white hover:bg-green-600' + ? 'bg-green-700 text-white hover:bg-green-800' : status === 'error' - ? 'bg-orange-500 text-white hover:bg-orange-600' + ? 'bg-orange-700 text-white hover:bg-orange-800' : 'bg-primary-500 text-white hover:bg-primary-600' ]" > diff --git a/frontend/src/components/account/AccountUsageCell.vue b/frontend/src/components/account/AccountUsageCell.vue index 784cd950a719..f1c6a2e096e2 100644 --- a/frontend/src/components/account/AccountUsageCell.vue +++ b/frontend/src/components/account/AccountUsageCell.vue @@ -181,18 +181,18 @@