diff --git a/.downstream/upstream-base b/.downstream/upstream-base index ec13377f3e51..6c8370e93b95 100644 --- a/.downstream/upstream-base +++ b/.downstream/upstream-base @@ -1 +1 @@ -v0.2.14 +v0.2.15 diff --git a/.downstream/upstream-review-v0.2.15.md b/.downstream/upstream-review-v0.2.15.md new file mode 100644 index 000000000000..d40f17504377 --- /dev/null +++ b/.downstream/upstream-review-v0.2.15.md @@ -0,0 +1,27 @@ +# Official v0.2.15 integration + +- Official tag: `v0.2.15`, commit `f2669c8cf62555cd92389b3f55920e9e6e7c6ff2`. +- Downstream starting commit: `d469422e94b0655b22dd9d0a826decda61ffb91e`. +- The 40 initial conflicts cover dependency/CI compatibility, platform catalogs, + tests, account interfaces and OpenAI request/usage handling. Existing Go + 1.27.2 and dependency security pins and narrow HTTP2 exceptions remain. +- Cline, Command Code, catalog-driven platforms, protocol conversion, model + discovery, TPS and async UI fixes follow the official release. Each WS turn + uses its refreshed group pricing while retaining immutable usage snapshots. +- OpenAI encrypted-content recovery and account retry rules follow official; + downstream recovery state, its admin API/page and special failovers are retired. + Historical settings and diagnostic data are retained. Claude recovery, Codex + identity/borrowing, Pelican observation, prompt injection, capacity evidence, + full administrator visibility and mapped test-model contracts remain. +- Named standalone API-key tool outputs retain their namespace; official lazy + request reconstruction is used. Compact filters retain existing facets and + state. New platforms take prompt/effort extensions only on supported paths. +- Official 242 and additive downstream 277 end both upgrade and fresh-install + paths without the two database platform CHECK constraints. 251/260/265 stay + byte-for-byte unchanged; 277 changes no rows. VERSION is 0.2.15. + +Local validation uses fake upstreams, UI fixtures and an isolated PostgreSQL +18.1 database. It covers official recovery, namespace preservation, new-platform +prompt/effort support, mapped models, filter disclosure and WS turn pricing. +Required PR CI remains authoritative. Release and production status is recorded +only by the operator workspace's production pointer. diff --git a/.downstream/upstream-risk.json b/.downstream/upstream-risk.json index bdf36c2123d7..7348834e6ce7 100644 --- a/.downstream/upstream-risk.json +++ b/.downstream/upstream-risk.json @@ -1,26 +1,80 @@ { - "critical_file_count": 10, + "critical_file_count": 62, "critical_files": [ - "deploy/.env.example", - "deploy/APPLE_CONTAINER.md", - "deploy/README.md", - "deploy/docker-compose.dev.yml", - "deploy/docker-compose.local.yml", - "deploy/docker-compose.standalone.yml", - "deploy/docker-compose.yml", - "deploy/docker-deploy.sh", - "frontend/package.json", - "frontend/pnpm-lock.yaml" + ".github/workflows/backend-ci.yml", + ".github/workflows/release.yml", + ".github/workflows/security-scan.yml", + "Dockerfile", + "backend/ent/runtime/runtime.go", + "backend/ent/schema/composite_model_route.go", + "backend/ent/schema/user_platform_quota.go", + "backend/go.mod", + "backend/go.sum", + "backend/internal/repository/ops_repo_dashboard.go", + "backend/internal/repository/ops_repo_dashboard_integration_test.go", + "backend/internal/repository/ops_repo_dashboard_timeout_test.go", + "backend/internal/repository/platform_list_db_integration_test.go", + "backend/internal/repository/user_platform_quota_repo.go", + "backend/internal/service/gateway_cache_ttl_order_test.go", + "backend/internal/service/gateway_claude_oauth_body.go", + "backend/internal/service/gateway_forward_as_chat_completions.go", + "backend/internal/service/gateway_forward_as_chat_completions_ping_test.go", + "backend/internal/service/gateway_forward_as_responses.go", + "backend/internal/service/gateway_service.go", + "backend/internal/service/gateway_tool_changes_beta_test.go", + "backend/internal/service/gateway_upstream_request.go", + "backend/internal/service/openai_apikey_responses_probe.go", + "backend/internal/service/openai_codex_model_metadata.go", + "backend/internal/service/openai_codex_model_metadata_test.go", + "backend/internal/service/openai_codex_models_pinned.go", + "backend/internal/service/openai_codex_models_pinned_test.go", + "backend/internal/service/openai_codex_transform.go", + "backend/internal/service/openai_gateway_chat_completions.go", + "backend/internal/service/openai_gateway_chat_completions_anthropic_native.go", + "backend/internal/service/openai_gateway_count_tokens.go", + "backend/internal/service/openai_gateway_forward.go", + "backend/internal/service/openai_gateway_messages.go", + "backend/internal/service/openai_gateway_messages_anthropic_native.go", + "backend/internal/service/openai_gateway_passthrough.go", + "backend/internal/service/openai_gateway_provider_claude_billing_test.go", + "backend/internal/service/openai_gateway_request_body.go", + "backend/internal/service/openai_gateway_response_handling.go", + "backend/internal/service/openai_gateway_responses_anthropic_native.go", + "backend/internal/service/openai_gateway_responses_empty_completed_test.go", + "backend/internal/service/openai_gateway_scheduling.go", + "backend/internal/service/openai_gateway_service_hotpath_test.go", + "backend/internal/service/openai_models_list.go", + "backend/internal/service/openai_models_list_test.go", + "backend/internal/service/openai_oauth_web_search_history.go", + "backend/internal/service/openai_oauth_web_search_history_forward_test.go", + "backend/internal/service/openai_oauth_web_search_history_test.go", + "backend/internal/service/openai_responses_namespace.go", + "backend/internal/service/openai_responses_namespace_benchmark_test.go", + "backend/internal/service/openai_responses_namespace_test.go", + "backend/internal/service/openai_ws_v2/passthrough_relay.go", + "backend/internal/service/openai_ws_v2/passthrough_relay_image_usage_test.go", + "backend/internal/service/openai_ws_v2_passthrough_adapter.go", + "backend/internal/service/openai_ws_v2_passthrough_image_usage_test.go", + "backend/internal/service/ratelimit_cline.go", + "backend/internal/service/ratelimit_cline_test.go", + "backend/internal/service/ratelimit_cn_providers.go", + "backend/internal/service/ratelimit_command_code.go", + "backend/internal/service/ratelimit_service.go", + "backend/migrations/242_drop_platform_check_constraints.sql", + "backend/migrations/platform_check_constraints_drop_migration_test.go", + "deploy/Dockerfile" ], - "downstream_commit": "7132011d7a4275e7d6407c92db4ec205a1c14ad1", - "downstream_file_count": 1232, + "downstream_commit": "d469422e94b0655b22dd9d0a826decda61ffb91e", + "downstream_file_count": 1340, "downstream_files": [ ".dockerignore", ".downstream/README.md", + ".downstream/gateway-borrow-notices.md", ".downstream/gpt6-models-and-prompts.md", ".downstream/native-domains.md", ".downstream/upstream-base", ".downstream/upstream-review-v0.2.13.md", + ".downstream/upstream-review-v0.2.14.md", ".downstream/upstream-risk.json", ".gitattributes", ".github/audit-exceptions.yml", @@ -45,12 +99,17 @@ ".github/workflows/upstream-risk-gate.yml", ".github/workflows/upstream-sync.yml", ".gitignore", + "DEV_GUIDE.md", "DOWNSTREAM.md", "Dockerfile", "Dockerfile.goreleaser", "Makefile", + "README.md", + "README_CN.md", + "README_JA.md", "THIRD_PARTY_NOTICES.md", "backend/.golangci.yml", + "backend/Dockerfile", "backend/cmd/cleanup-ingress-reject-logs/main.go", "backend/cmd/server/VERSION", "backend/cmd/server/main.go", @@ -137,8 +196,14 @@ "backend/internal/handler/admin/account_upstream_billing_rates_test.go", "backend/internal/handler/admin/batch_update_credentials_test.go", "backend/internal/handler/admin/codex_error_diagnostics.go", + "backend/internal/handler/admin/codex_fingerprint_handler.go", + "backend/internal/handler/admin/codex_fingerprint_handler_test.go", + "backend/internal/handler/admin/codex_gateway_borrow_handler.go", + "backend/internal/handler/admin/codex_gateway_borrow_handler_test.go", "backend/internal/handler/admin/group_handler_simple_mode_test.go", "backend/internal/handler/admin/payment_handler_test.go", + "backend/internal/handler/admin/pelican_test_handler.go", + "backend/internal/handler/admin/pelican_test_handler_test.go", "backend/internal/handler/admin/reasoning_recovery_handler.go", "backend/internal/handler/admin/reasoning_recovery_handler_test.go", "backend/internal/handler/admin/scheduled_test_handler.go", @@ -152,12 +217,15 @@ "backend/internal/handler/admin/snapshot_cache_test.go", "backend/internal/handler/admin/system_prompt_handler.go", "backend/internal/handler/admin/user_platform_quota_admin_test.go", + "backend/internal/handler/codex_gateway_borrow_failover_test.go", + "backend/internal/handler/codex_gateway_borrow_ws_failure_test.go", "backend/internal/handler/dto/credentials_redact.go", "backend/internal/handler/dto/credentials_redact_test.go", "backend/internal/handler/dto/mappers.go", "backend/internal/handler/dto/settings.go", "backend/internal/handler/dto/types.go", "backend/internal/handler/failover_loop_test.go", + "backend/internal/handler/gateway_claude_stream_error_test.go", "backend/internal/handler/gateway_handler.go", "backend/internal/handler/gateway_handler_chat_completions.go", "backend/internal/handler/gateway_handler_responses.go", @@ -303,7 +371,14 @@ "backend/internal/repository/channel_repo_pricing_time_test.go", "backend/internal/repository/channel_repo_test.go", "backend/internal/repository/codex_fingerprint_seed_migration_integration_test.go", + "backend/internal/repository/codex_gateway_borrow_test_repo.go", + "backend/internal/repository/codex_gateway_borrow_test_repo_integration_test.go", + "backend/internal/repository/codex_gateway_borrow_test_repo_test.go", + "backend/internal/repository/codex_gateway_borrow_upstream.go", + "backend/internal/repository/codex_gateway_borrow_upstream_test.go", "backend/internal/repository/codex_runtime_plugin_purge_migration_integration_test.go", + "backend/internal/repository/concurrency_account_lease_test.go", + "backend/internal/repository/concurrency_cache.go", "backend/internal/repository/dashboard_aggregation_repo.go", "backend/internal/repository/db_pool.go", "backend/internal/repository/decompress_response_test.go", @@ -320,6 +395,7 @@ "backend/internal/repository/group_usage_rollup_trigger_integration_test.go", "backend/internal/repository/http_upstream.go", "backend/internal/repository/http_upstream_compression_test.go", + "backend/internal/repository/http_upstream_http2_keepalive_test.go", "backend/internal/repository/image_studio_drop_migration_integration_test.go", "backend/internal/repository/integration_harness_test.go", "backend/internal/repository/migrations_historical_preflight.go", @@ -356,6 +432,7 @@ "backend/internal/securityaudit/prompt_event_repository.go", "backend/internal/securityaudit/prompt_repository_integration_test.go", "backend/internal/server/api_contract_test.go", + "backend/internal/server/http.go", "backend/internal/server/middleware/audit_log.go", "backend/internal/server/middleware/audit_log_test.go", "backend/internal/server/middleware/group_model_allowlist.go", @@ -373,6 +450,7 @@ "backend/internal/server/routes/gateway_model_allowlist_test.go", "backend/internal/server/routes/gateway_models_pinned_test.go", "backend/internal/server/routes/gateway_test.go", + "backend/internal/server/routes/pelican_tests.go", "backend/internal/server/routes/reasoning_recovery_routes_test.go", "backend/internal/server/routes/system_prompt_routes_test.go", "backend/internal/service/account.go", @@ -427,11 +505,14 @@ "backend/internal/service/agent_request_capabilities_test.go", "backend/internal/service/agent_selection_diagnostics_test.go", "backend/internal/service/agent_upstream_error_scope_test.go", + "backend/internal/service/antigravity_credits_overages.go", "backend/internal/service/antigravity_gateway_compat_stream.go", "backend/internal/service/antigravity_gateway_gemini.go", "backend/internal/service/antigravity_gateway_retry.go", "backend/internal/service/antigravity_gateway_service.go", "backend/internal/service/antigravity_gateway_upstream.go", + "backend/internal/service/antigravity_internal500_penalty.go", + "backend/internal/service/antigravity_token_provider.go", "backend/internal/service/api_key_auth_cache_impl.go", "backend/internal/service/api_key_auth_cache_version_test.go", "backend/internal/service/audit_log_service.go", @@ -449,8 +530,28 @@ "backend/internal/service/cn_provider_balance_check_service_test.go", "backend/internal/service/codex_error_diagnostics.go", "backend/internal/service/codex_error_diagnostics_test.go", + "backend/internal/service/codex_fingerprint_account_view.go", + "backend/internal/service/codex_fingerprint_account_view_test.go", + "backend/internal/service/codex_fingerprint_policy_test.go", + "backend/internal/service/codex_gateway_borrow.go", + "backend/internal/service/codex_gateway_borrow_core_test.go", + "backend/internal/service/codex_gateway_borrow_http.go", + "backend/internal/service/codex_gateway_borrow_http_test.go", + "backend/internal/service/codex_gateway_borrow_pelican.go", + "backend/internal/service/codex_gateway_borrow_preview.go", + "backend/internal/service/codex_gateway_borrow_preview_test.go", + "backend/internal/service/codex_gateway_borrow_probe.go", + "backend/internal/service/codex_gateway_borrow_probe_stream_test.go", + "backend/internal/service/codex_gateway_borrow_test_runner.go", + "backend/internal/service/codex_gateway_borrow_test_runner_test.go", + "backend/internal/service/codex_gateway_borrow_token_test.go", + "backend/internal/service/codex_gateway_borrow_ws_anchor.go", + "backend/internal/service/codex_gateway_borrow_ws_passthrough_test.go", + "backend/internal/service/codex_gateway_borrow_ws_phase_test.go", + "backend/internal/service/codex_gateway_borrow_ws_test.go", "backend/internal/service/codex_outbound_contract.go", "backend/internal/service/composite_platform_test.go", + "backend/internal/service/concurrency_service.go", "backend/internal/service/content_moderation_runtime_cache_test.go", "backend/internal/service/conversation_standard_pricing.go", "backend/internal/service/conversation_standard_pricing_test.go", @@ -459,20 +560,30 @@ "backend/internal/service/domain_constants.go", "backend/internal/service/email_service_smtp_test.go", "backend/internal/service/gateway_anthropic_passthrough.go", + "backend/internal/service/gateway_claude_prelude_test.go", + "backend/internal/service/gateway_claude_signature_recovery.go", + "backend/internal/service/gateway_claude_signature_recovery_test.go", + "backend/internal/service/gateway_claude_stream_error.go", + "backend/internal/service/gateway_claude_stream_error_test.go", "backend/internal/service/gateway_compat_reasoning_pricing_test.go", "backend/internal/service/gateway_forward.go", "backend/internal/service/gateway_forward_as_chat_completions.go", "backend/internal/service/gateway_forward_as_responses.go", + "backend/internal/service/gateway_forward_partial_usage_test.go", "backend/internal/service/gateway_hotpath_optimization_test.go", "backend/internal/service/gateway_model_availability.go", "backend/internal/service/gateway_model_availability_test.go", "backend/internal/service/gateway_multiplatform_test.go", + "backend/internal/service/gateway_observation.go", + "backend/internal/service/gateway_observation_test.go", "backend/internal/service/gateway_scheduling.go", "backend/internal/service/gateway_service.go", "backend/internal/service/gateway_service_bedrock_beta_test.go", "backend/internal/service/gateway_service_subscription_billing_test.go", "backend/internal/service/gateway_systemone.go", "backend/internal/service/gateway_systemone_test.go", + "backend/internal/service/gateway_upstream_response.go", + "backend/internal/service/gateway_upstream_transport_error.go", "backend/internal/service/gateway_usage_billing.go", "backend/internal/service/gemini_chat_completions_compat_service.go", "backend/internal/service/gemini_messages_compat_service.go", @@ -482,12 +593,16 @@ "backend/internal/service/grok_credential_failure.go", "backend/internal/service/grok_free_quota_gate_test.go", "backend/internal/service/grok_quota_service_test.go", + "backend/internal/service/grok_token_provider.go", + "backend/internal/service/grok_upstream_errors.go", "backend/internal/service/grok_upstream_errors_test.go", + "backend/internal/service/grok_upstream_failure.go", "backend/internal/service/group_model_allowlist.go", "backend/internal/service/group_model_allowlist_test.go", "backend/internal/service/group_model_capacity.go", "backend/internal/service/group_model_capacity_test.go", "backend/internal/service/group_peak_rate_test.go", + "backend/internal/service/http_upstream_profile.go", "backend/internal/service/kimi_anthropic_usage_test.go", "backend/internal/service/model_context_capacity.go", "backend/internal/service/model_context_capacity_catalog_test.go", @@ -503,6 +618,7 @@ "backend/internal/service/model_rate_limit.go", "backend/internal/service/oauth_service.go", "backend/internal/service/oauth_service_test.go", + "backend/internal/service/observed_account_lease_test.go", "backend/internal/service/openai_account_model_transient.go", "backend/internal/service/openai_account_runtime_block_fastpath.go", "backend/internal/service/openai_account_runtime_block_fastpath_test.go", @@ -514,6 +630,7 @@ "backend/internal/service/openai_alpha_search_test.go", "backend/internal/service/openai_apikey_compat.go", "backend/internal/service/openai_apikey_compat_test.go", + "backend/internal/service/openai_apikey_health_breaker.go", "backend/internal/service/openai_apikey_protocol_modes.go", "backend/internal/service/openai_apikey_protocol_modes_test.go", "backend/internal/service/openai_astra_pro_mode_test.go", @@ -529,6 +646,7 @@ "backend/internal/service/openai_codex_fingerprint_test.go", "backend/internal/service/openai_codex_identity.go", "backend/internal/service/openai_codex_identity_snapshot.go", + "backend/internal/service/openai_codex_identity_test.go", "backend/internal/service/openai_codex_model_metadata.go", "backend/internal/service/openai_codex_model_metadata_test.go", "backend/internal/service/openai_codex_models_service.go", @@ -540,6 +658,7 @@ "backend/internal/service/openai_codex_transform_test.go", "backend/internal/service/openai_codex_turn_state.go", "backend/internal/service/openai_codex_turn_state_test.go", + "backend/internal/service/openai_codex_version_sync_service.go", "backend/internal/service/openai_compact_fallback_test.go", "backend/internal/service/openai_compact_model_mapping_test.go", "backend/internal/service/openai_compact_stream_bridge.go", @@ -642,6 +761,7 @@ "backend/internal/service/openai_previous_response_id.go", "backend/internal/service/openai_previous_response_id_test.go", "backend/internal/service/openai_privacy_service.go", + "backend/internal/service/openai_proxy_stream_circuit.go", "backend/internal/service/openai_quota_auto_reset.go", "backend/internal/service/openai_quota_service.go", "backend/internal/service/openai_raw_stream_truncation.go", @@ -687,6 +807,8 @@ "backend/internal/service/openai_sticky_compat.go", "backend/internal/service/openai_sticky_compat_test.go", "backend/internal/service/openai_subscription_test.go", + "backend/internal/service/openai_team_linked_error.go", + "backend/internal/service/openai_token_provider.go", "backend/internal/service/openai_upstream_client_error_test.go", "backend/internal/service/openai_upstream_transport_error.go", "backend/internal/service/openai_upstream_transport_error_handle_test.go", @@ -711,6 +833,7 @@ "backend/internal/service/openai_ws_http_bridge_isolation_test.go", "backend/internal/service/openai_ws_http_bridge_resume_test.go", "backend/internal/service/openai_ws_http_bridge_test.go", + "backend/internal/service/openai_ws_pool.go", "backend/internal/service/openai_ws_protocol_forward_test.go", "backend/internal/service/openai_ws_ratelimit_signal_test.go", "backend/internal/service/openai_ws_reasoning_mode_model_test.go", @@ -720,6 +843,7 @@ "backend/internal/service/openai_ws_v2_passthrough_adapter.go", "backend/internal/service/openai_ws_v2_passthrough_adapter_effort_test.go", "backend/internal/service/openai_ws_v2_passthrough_lifecycle_test.go", + "backend/internal/service/ops_claude_signature_evidence_test.go", "backend/internal/service/ops_concurrency.go", "backend/internal/service/ops_log_runtime_test.go", "backend/internal/service/ops_metrics_collector.go", @@ -732,6 +856,13 @@ "backend/internal/service/ops_upstream_context.go", "backend/internal/service/ops_user_error_test.go", "backend/internal/service/payment_order_result_test.go", + "backend/internal/service/pelican_account_generator.go", + "backend/internal/service/pelican_account_generator_test.go", + "backend/internal/service/pelican_borrow_ws_test.go", + "backend/internal/service/pelican_execution.go", + "backend/internal/service/pelican_test_catalog.go", + "backend/internal/service/pelican_test_catalog_test.go", + "backend/internal/service/pelican_wire.go", "backend/internal/service/pricing_named_family_test.go", "backend/internal/service/pricing_service.go", "backend/internal/service/quota_activity.go", @@ -751,6 +882,7 @@ "backend/internal/service/responses_chat_protocol_callers_test.go", "backend/internal/service/scheduler_snapshot_service.go", "backend/internal/service/setting_admin_observability.go", + "backend/internal/service/setting_codex_fingerprint.go", "backend/internal/service/setting_codex_runtime.go", "backend/internal/service/setting_gateway_runtime.go", "backend/internal/service/setting_gateway_runtime_refusal_test.go", @@ -858,6 +990,8 @@ "backend/migrations/272_purge_first_party_plugins.sql", "backend/migrations/273_purge_account_traffic_telemetry_switch.sql", "backend/migrations/274_purge_release_acceptance_api_keys.sql", + "backend/migrations/275_codex_gateway_borrow_tests.sql", + "backend/migrations/276_pelican_test_execution.sql", "backend/migrations/account_jobs_slim_migration_test.go", "backend/migrations/historical_cindy_null_total_migration_integration_test.go", "backend/migrations/minimax_platform_migration_test.go", @@ -892,10 +1026,16 @@ "frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts", "frontend/src/api/accountOperationIdempotency.ts", "frontend/src/api/admin/__tests__/accountJobs.spec.ts", + "frontend/src/api/admin/__tests__/codexGatewayBorrow.spec.ts", + "frontend/src/api/admin/__tests__/pelicanTests.spec.ts", "frontend/src/api/admin/accountJobs.ts", "frontend/src/api/admin/accounts.ts", + "frontend/src/api/admin/codexFingerprint.ts", + "frontend/src/api/admin/codexGatewayBorrow.ts", "frontend/src/api/admin/codexRuntime.ts", "frontend/src/api/admin/index.ts", + "frontend/src/api/admin/ops.ts", + "frontend/src/api/admin/pelicanTests.ts", "frontend/src/api/admin/reasoningRecovery.ts", "frontend/src/api/admin/settings.ts", "frontend/src/api/admin/system.ts", @@ -941,8 +1081,11 @@ "frontend/src/components/account/__tests__/UsageProgressBar.spec.ts", "frontend/src/components/account/__tests__/credentialsBuilder.spec.ts", "frontend/src/components/account/credentialsBuilder.ts", + "frontend/src/components/admin/CodexBorrowNav.vue", + "frontend/src/components/admin/CodexFingerprintIdentity.vue", "frontend/src/components/admin/ObservabilitySettingsPanel.vue", "frontend/src/components/admin/OfficialModelCatalogPanel.vue", + "frontend/src/components/admin/__tests__/CodexBorrowNav.spec.ts", "frontend/src/components/admin/account-jobs/AccountOperationConfirmDialog.vue", "frontend/src/components/admin/account-jobs/AccountOperationDialog.vue", "frontend/src/components/admin/account-jobs/AccountOperationProgress.vue", @@ -985,14 +1128,17 @@ "frontend/src/components/admin/account/__tests__/AccountTestModal.qualityCheck.spec.ts", "frontend/src/components/admin/account/__tests__/AccountTestModal.spec.ts", "frontend/src/components/admin/account/__tests__/AccountTestModal.testPlan.spec.ts", + "frontend/src/components/admin/account/__tests__/AccountTestReasoningSelect.spec.ts", "frontend/src/components/admin/account/__tests__/AccountUsageViews.spec.ts", "frontend/src/components/admin/account/__tests__/ImportDataModal.spec.ts", "frontend/src/components/admin/account/__tests__/ReAuthAccountModal.spec.ts", "frontend/src/components/admin/account/__tests__/ScheduledTestsPanel.spec.ts", "frontend/src/components/admin/announcements/AnnouncementReadStatusDialog.vue", "frontend/src/components/admin/announcements/AnnouncementTargetingEditor.vue", + "frontend/src/components/admin/codex/BorrowPelicanPreview.vue", "frontend/src/components/admin/codex/CodexContinuationDiagnostics.vue", "frontend/src/components/admin/codex/CodexRuntimeSettings.vue", + "frontend/src/components/admin/codex/__tests__/BorrowPelicanPreview.spec.ts", "frontend/src/components/admin/codex/__tests__/CodexRuntime.spec.ts", "frontend/src/components/admin/group/CodexManifestAccountsField.vue", "frontend/src/components/admin/group/GroupRPMOverridesModal.vue", @@ -1088,6 +1234,7 @@ "frontend/src/composables/__tests__/useModelContextCapacities.spec.ts", "frontend/src/composables/__tests__/useModelWhitelist.spec.ts", "frontend/src/composables/useAccountTestPrompt.ts", + "frontend/src/composables/useCodexBorrowUI.ts", "frontend/src/composables/useModelContextCapacities.ts", "frontend/src/composables/useModelWhitelist.ts", "frontend/src/composables/usePresentationColors.ts", @@ -1101,7 +1248,9 @@ "frontend/src/i18n/__tests__/accountConsoleLocales.spec.ts", "frontend/src/i18n/locales/en/admin/accountTasks.ts", "frontend/src/i18n/locales/en/admin/accounts.ts", + "frontend/src/i18n/locales/en/admin/codexGatewayBorrow.ts", "frontend/src/i18n/locales/en/admin/index.ts", + "frontend/src/i18n/locales/en/admin/ops.ts", "frontend/src/i18n/locales/en/admin/reasoningRecovery.ts", "frontend/src/i18n/locales/en/admin/resources.ts", "frontend/src/i18n/locales/en/admin/settings.ts", @@ -1111,7 +1260,9 @@ "frontend/src/i18n/locales/en/misc.ts", "frontend/src/i18n/locales/zh/admin/accountTasks.ts", "frontend/src/i18n/locales/zh/admin/accounts.ts", + "frontend/src/i18n/locales/zh/admin/codexGatewayBorrow.ts", "frontend/src/i18n/locales/zh/admin/index.ts", + "frontend/src/i18n/locales/zh/admin/ops.ts", "frontend/src/i18n/locales/zh/admin/reasoningRecovery.ts", "frontend/src/i18n/locales/zh/admin/resources.ts", "frontend/src/i18n/locales/zh/admin/settings.ts", @@ -1120,6 +1271,7 @@ "frontend/src/i18n/locales/zh/dashboard.ts", "frontend/src/i18n/locales/zh/misc.ts", "frontend/src/main.ts", + "frontend/src/router/__tests__/codex-borrow-routes.spec.ts", "frontend/src/router/index.ts", "frontend/src/stores/__tests__/accountJobs.spec.ts", "frontend/src/stores/accountJobs.ts", @@ -1173,6 +1325,10 @@ "frontend/src/views/admin/BackupView.vue", "frontend/src/views/admin/ChannelMonitorView.vue", "frontend/src/views/admin/ChannelsView.vue", + "frontend/src/views/admin/CodexFingerprintView.vue", + "frontend/src/views/admin/CodexGatewayBorrowStatusView.vue", + "frontend/src/views/admin/CodexGatewayBorrowView.vue", + "frontend/src/views/admin/CodexPelicanComparisonView.vue", "frontend/src/views/admin/CodexRuntimeView.vue", "frontend/src/views/admin/DashboardView.vue", "frontend/src/views/admin/GroupsView.vue", @@ -1196,6 +1352,10 @@ "frontend/src/views/admin/__tests__/AccountsView.selectionScope.spec.ts", "frontend/src/views/admin/__tests__/AccountsView.sparkShadow.spec.ts", "frontend/src/views/admin/__tests__/AccountsView.usageWindowsHint.spec.ts", + "frontend/src/views/admin/__tests__/CodexFingerprintView.spec.ts", + "frontend/src/views/admin/__tests__/CodexGatewayBorrowStatusView.spec.ts", + "frontend/src/views/admin/__tests__/CodexGatewayBorrowView.spec.ts", + "frontend/src/views/admin/__tests__/CodexPelicanComparisonView.spec.ts", "frontend/src/views/admin/__tests__/GroupsView.codexManifest.spec.ts", "frontend/src/views/admin/__tests__/GroupsView.duplicate.spec.ts", "frontend/src/views/admin/__tests__/ProxiesView.copyFormats.spec.ts", @@ -1249,55 +1409,456 @@ "tools/check_pnpm_audit_exceptions.py" ], "initial_conflict_files": [ - ".github/audit-exceptions.yml", - "frontend/package.json", - "frontend/pnpm-lock.yaml" + ".github/workflows/backend-ci.yml", + "backend/.golangci.yml", + "backend/go.mod", + "backend/internal/handler/admin/account_handler.go", + "backend/internal/handler/admin/account_handler_available_models_test.go", + "backend/internal/handler/gateway_models_test.go", + "backend/internal/handler/openai_gateway_handler.go", + "backend/internal/service/account_test_service.go", + "backend/internal/service/model_rate_limit.go", + "backend/internal/service/openai_gateway_chat_completions.go", + "backend/internal/service/openai_gateway_forward.go", + "backend/internal/service/openai_gateway_request_body.go", + "backend/internal/service/openai_gateway_scheduling.go", + "backend/internal/service/openai_responses_namespace.go", + "frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts", + "frontend/src/api/admin/settings.ts", + "frontend/src/api/admin/users.ts", + "frontend/src/components/account/AccountUsageCell.vue", + "frontend/src/components/account/CreateAccountModal.vue", + "frontend/src/components/account/EditAccountModal.vue", + "frontend/src/components/account/ModelWhitelistSelector.vue", + "frontend/src/components/account/OpenCodeGoProtocolRulesEditor.vue", + "frontend/src/components/account/__tests__/EditAccountModal.spec.ts", + "frontend/src/components/account/__tests__/ModelWhitelistSelector.spec.ts", + "frontend/src/components/admin/account/ScheduledTestsPanel.vue", + "frontend/src/components/admin/group/GroupRPMOverridesModal.vue", + "frontend/src/components/admin/group/GroupRateMultipliersModal.vue", + "frontend/src/components/admin/user/__tests__/UserPlatformQuotaModal.spec.ts", + "frontend/src/components/common/BaseDialog.vue", + "frontend/src/components/keys/__tests__/UseKeyModal.spec.ts", + "frontend/src/components/user/UserPlatformQuotaCell.vue", + "frontend/src/constants/__tests__/platforms.spec.ts", + "frontend/src/i18n/locales/en/admin/accounts.ts", + "frontend/src/i18n/locales/zh/admin/accounts.ts", + "frontend/src/utils/platformColors.ts", + "frontend/src/views/admin/AccountsView.vue", + "frontend/src/views/admin/ChannelsView.vue", + "frontend/src/views/admin/SettingsView.vue", + "frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts", + "frontend/src/views/user/__tests__/StripePaymentView.spec.ts" ], "merge_conflicts": [], - "overlap_file_count": 11, + "overlap_file_count": 122, "overlap_files": [ - ".github/audit-exceptions.yml", + ".github/workflows/backend-ci.yml", + ".github/workflows/release.yml", + ".github/workflows/security-scan.yml", + "Dockerfile", + "Makefile", + "backend/.golangci.yml", + "backend/Dockerfile", "backend/cmd/server/VERSION", - "deploy/.env.example", - "deploy/README.md", - "deploy/docker-compose.dev.yml", - "deploy/docker-compose.local.yml", - "deploy/docker-compose.standalone.yml", - "deploy/docker-compose.yml", - "frontend/package.json", - "frontend/pnpm-lock.yaml", - "frontend/src/components/keys/UseKeyModal.vue" + "backend/ent/runtime/runtime.go", + "backend/go.mod", + "backend/go.sum", + "backend/internal/handler/admin/account_handler.go", + "backend/internal/handler/admin/account_handler_available_models_test.go", + "backend/internal/handler/gateway_handler.go", + "backend/internal/handler/gateway_models_test.go", + "backend/internal/handler/openai_codex_models_handler_test.go", + "backend/internal/handler/openai_gateway_handler.go", + "backend/internal/handler/openai_gateway_handler_test.go", + "backend/internal/pkg/apicompat/anthropic_responses_test.go", + "backend/internal/pkg/apicompat/chatcompletions_to_responses.go", + "backend/internal/pkg/apicompat/responses_to_chatcompletions.go", + "backend/internal/pkg/apicompat/types.go", + "backend/internal/pkg/claude/constants.go", + "backend/internal/repository/user_platform_quota_repo.go", + "backend/internal/server/api_contract_test.go", + "backend/internal/server/routes/gateway.go", + "backend/internal/service/account.go", + "backend/internal/service/account_scheduling_threshold_eval.go", + "backend/internal/service/account_service.go", + "backend/internal/service/account_test_service.go", + "backend/internal/service/account_test_service_cn_adaptive.go", + "backend/internal/service/admin_account.go", + "backend/internal/service/cn_provider_balance_check_service.go", + "backend/internal/service/composite_platform_test.go", + "backend/internal/service/domain_constants.go", + "backend/internal/service/gateway_forward_as_chat_completions.go", + "backend/internal/service/gateway_forward_as_responses.go", + "backend/internal/service/gateway_service.go", + "backend/internal/service/model_rate_limit.go", + "backend/internal/service/openai_codex_model_metadata.go", + "backend/internal/service/openai_codex_model_metadata_test.go", + "backend/internal/service/openai_codex_transform.go", + "backend/internal/service/openai_gateway_chat_completions.go", + "backend/internal/service/openai_gateway_chat_completions_anthropic_native.go", + "backend/internal/service/openai_gateway_count_tokens.go", + "backend/internal/service/openai_gateway_forward.go", + "backend/internal/service/openai_gateway_messages.go", + "backend/internal/service/openai_gateway_messages_anthropic_native.go", + "backend/internal/service/openai_gateway_passthrough.go", + "backend/internal/service/openai_gateway_request_body.go", + "backend/internal/service/openai_gateway_response_handling.go", + "backend/internal/service/openai_gateway_responses_anthropic_native.go", + "backend/internal/service/openai_gateway_scheduling.go", + "backend/internal/service/openai_gateway_service_hotpath_test.go", + "backend/internal/service/openai_models_list.go", + "backend/internal/service/openai_responses_namespace.go", + "backend/internal/service/openai_ws_v2_passthrough_adapter.go", + "backend/internal/service/ratelimit_service.go", + "backend/internal/service/scheduler_snapshot_service.go", + "backend/internal/service/upstream_models.go", + "backend/internal/web/embed_on.go", + "backend/internal/web/embed_test.go", + "deploy/Dockerfile", + "frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts", + "frontend/src/api/admin/ops.ts", + "frontend/src/api/admin/settings.ts", + "frontend/src/api/admin/users.ts", + "frontend/src/components/account/AccountUsageCell.vue", + "frontend/src/components/account/CreateAccountModal.vue", + "frontend/src/components/account/EditAccountModal.vue", + "frontend/src/components/account/ModelWhitelistSelector.vue", + "frontend/src/components/account/OpenCodeGoProtocolRulesEditor.vue", + "frontend/src/components/account/__tests__/AccountUsageCell.spec.ts", + "frontend/src/components/account/__tests__/CreateAccountModal.spec.ts", + "frontend/src/components/account/__tests__/EditAccountModal.spec.ts", + "frontend/src/components/account/__tests__/ModelWhitelistSelector.spec.ts", + "frontend/src/components/account/__tests__/credentialsBuilder.spec.ts", + "frontend/src/components/account/credentialsBuilder.ts", + "frontend/src/components/admin/account/ScheduledTestsPanel.vue", + "frontend/src/components/admin/group/GroupRPMOverridesModal.vue", + "frontend/src/components/admin/group/GroupRateMultipliersModal.vue", + "frontend/src/components/admin/monitor/MonitorTemplateManagerDialog.vue", + "frontend/src/components/admin/usage/UsageTable.vue", + "frontend/src/components/admin/user/UserPlatformQuotaModal.vue", + "frontend/src/components/admin/user/__tests__/UserPlatformQuotaModal.spec.ts", + "frontend/src/components/common/AnnouncementBell.vue", + "frontend/src/components/common/BaseDialog.vue", + "frontend/src/components/common/Select.vue", + "frontend/src/components/keys/UseKeyModal.vue", + "frontend/src/components/keys/__tests__/UseKeyModal.spec.ts", + "frontend/src/components/user/UserPlatformQuotaCell.vue", + "frontend/src/components/user/profile/ProfileBalanceNotifyCard.vue", + "frontend/src/components/user/profile/ProfileIdentityBindingsSection.vue", + "frontend/src/composables/__tests__/useModelWhitelist.spec.ts", + "frontend/src/composables/useModelWhitelist.ts", + "frontend/src/constants/__tests__/platforms.spec.ts", + "frontend/src/constants/platforms.ts", + "frontend/src/i18n/locales/en/admin/accounts.ts", + "frontend/src/i18n/locales/en/admin/ops.ts", + "frontend/src/i18n/locales/en/admin/resources.ts", + "frontend/src/i18n/locales/en/dashboard.ts", + "frontend/src/i18n/locales/zh/admin/accounts.ts", + "frontend/src/i18n/locales/zh/admin/ops.ts", + "frontend/src/i18n/locales/zh/admin/resources.ts", + "frontend/src/i18n/locales/zh/dashboard.ts", + "frontend/src/types/index.ts", + "frontend/src/utils/keyGroupProviders.ts", + "frontend/src/utils/platformColors.ts", + "frontend/src/views/admin/AccountsView.vue", + "frontend/src/views/admin/BackupView.vue", + "frontend/src/views/admin/ChannelsView.vue", + "frontend/src/views/admin/GroupsView.vue", + "frontend/src/views/admin/PluginsView.vue", + "frontend/src/views/admin/SettingsView.vue", + "frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts", + "frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue", + "frontend/src/views/admin/ops/components/OpsAlertRulesCard.vue", + "frontend/src/views/admin/ops/components/OpsDashboardHeader.vue", + "frontend/src/views/admin/ops/components/OpsSettingsDialog.vue", + "frontend/src/views/user/CustomPageView.vue", + "frontend/src/views/user/StripePaymentView.vue", + "frontend/src/views/user/__tests__/StripePaymentView.spec.ts" ], "risk_class": "review_required", "schema": 1, - "upstream_base": "v0.2.13", - "upstream_file_count": 25, + "upstream_base": "v0.2.14", + "upstream_file_count": 278, "upstream_files": [ - ".github/audit-exceptions.yml", - "README.md", - "README_CN.md", - "README_JA.md", + ".github/workflows/backend-ci.yml", + ".github/workflows/release.yml", + ".github/workflows/security-scan.yml", + "Dockerfile", + "Makefile", + "backend/.golangci.yml", + "backend/Dockerfile", "backend/cmd/server/VERSION", - "backend/internal/integration/e2e_user_flow_test.go", - "backend/internal/payment/provider/easypay.go", - "backend/internal/payment/provider/easypay_notify_security_test.go", - "backend/internal/service/payment_resume_service.go", - "backend/internal/service/payment_resume_service_test.go", - "backend/internal/setup/cli.go", - "backend/internal/setup/handler.go", - "backend/internal/setup/setup.go", - "backend/internal/setup/setup_test.go", - "deploy/.env.example", - "deploy/APPLE_CONTAINER.md", - "deploy/README.md", - "deploy/docker-compose.dev.yml", - "deploy/docker-compose.local.yml", - "deploy/docker-compose.standalone.yml", - "deploy/docker-compose.yml", - "deploy/docker-deploy.sh", - "frontend/package.json", - "frontend/pnpm-lock.yaml", - "frontend/src/components/keys/UseKeyModal.vue" + "backend/ent/runtime/runtime.go", + "backend/ent/schema/composite_model_route.go", + "backend/ent/schema/user_platform_quota.go", + "backend/go.mod", + "backend/go.sum", + "backend/internal/domain/constants.go", + "backend/internal/domain/platforms.go", + "backend/internal/domain/platforms_test.go", + "backend/internal/handler/admin/account_handler.go", + "backend/internal/handler/admin/account_handler_available_models_test.go", + "backend/internal/handler/admin/channel_handler.go", + "backend/internal/handler/admin/group_handler.go", + "backend/internal/handler/admin/group_handler_platform_test.go", + "backend/internal/handler/admin/platform_validation.go", + "backend/internal/handler/gateway_handler.go", + "backend/internal/handler/gateway_models_test.go", + "backend/internal/handler/openai_codex_models_handler_test.go", + "backend/internal/handler/openai_gateway_handler.go", + "backend/internal/handler/openai_gateway_handler_test.go", + "backend/internal/handler/openai_gateway_ws_group_pricing_test.go", + "backend/internal/model/error_passthrough_rule.go", + "backend/internal/model/error_passthrough_rule_test.go", + "backend/internal/pkg/antigravity/claude_types.go", + "backend/internal/pkg/antigravity/claude_types_test.go", + "backend/internal/pkg/antigravity/stream_transformer.go", + "backend/internal/pkg/apicompat/anthropic_responses_test.go", + "backend/internal/pkg/apicompat/chatcompletions_to_responses.go", + "backend/internal/pkg/apicompat/chatcompletions_to_responses_developer_role_test.go", + "backend/internal/pkg/apicompat/chatcompletions_to_responses_legacy_function_call_test.go", + "backend/internal/pkg/apicompat/chatcompletions_to_responses_tool_choice_test.go", + "backend/internal/pkg/apicompat/responses_to_chatcompletions.go", + "backend/internal/pkg/apicompat/responses_to_chatcompletions_refusal_test.go", + "backend/internal/pkg/apicompat/thinking_signature_wire_test.go", + "backend/internal/pkg/apicompat/types.go", + "backend/internal/pkg/claude/constants.go", + "backend/internal/pkg/xai/sso_device.go", + "backend/internal/pkg/xai/sso_device_test.go", + "backend/internal/repository/ops_repo_dashboard.go", + "backend/internal/repository/ops_repo_dashboard_integration_test.go", + "backend/internal/repository/ops_repo_dashboard_timeout_test.go", + "backend/internal/repository/platform_list_db_integration_test.go", + "backend/internal/repository/user_platform_quota_repo.go", + "backend/internal/server/api_contract_test.go", + "backend/internal/server/routes/gateway.go", + "backend/internal/service/account.go", + "backend/internal/service/account_header_override.go", + "backend/internal/service/account_scheduling_threshold_eval.go", + "backend/internal/service/account_service.go", + "backend/internal/service/account_test_logging.go", + "backend/internal/service/account_test_logging_test.go", + "backend/internal/service/account_test_service.go", + "backend/internal/service/account_test_service_cn_adaptive.go", + "backend/internal/service/admin_account.go", + "backend/internal/service/admin_group.go", + "backend/internal/service/anthropic_apikey_auth.go", + "backend/internal/service/anthropic_buffered_negative_index_test.go", + "backend/internal/service/auth_service.go", + "backend/internal/service/channel_monitor_checker.go", + "backend/internal/service/channel_monitor_endpoint_test.go", + "backend/internal/service/channel_monitor_v2.go", + "backend/internal/service/channel_monitor_v2_test.go", + "backend/internal/service/channel_service.go", + "backend/internal/service/channel_service_test.go", + "backend/internal/service/cline.go", + "backend/internal/service/cline_test.go", + "backend/internal/service/cline_usage.go", + "backend/internal/service/cn_provider_balance_check_service.go", + "backend/internal/service/cn_provider_balance_service.go", + "backend/internal/service/cn_provider_quota_service.go", + "backend/internal/service/cn_providers_test.go", + "backend/internal/service/command_code.go", + "backend/internal/service/command_code_test.go", + "backend/internal/service/command_code_usage.go", + "backend/internal/service/command_code_usage_test.go", + "backend/internal/service/composite_platform.go", + "backend/internal/service/composite_platform_test.go", + "backend/internal/service/composite_route_resolver.go", + "backend/internal/service/composite_route_resolver_test.go", + "backend/internal/service/domain_constants.go", + "backend/internal/service/gateway_cache_ttl_order_test.go", + "backend/internal/service/gateway_claude_oauth_body.go", + "backend/internal/service/gateway_forward_as_chat_completions.go", + "backend/internal/service/gateway_forward_as_chat_completions_ping_test.go", + "backend/internal/service/gateway_forward_as_responses.go", + "backend/internal/service/gateway_service.go", + "backend/internal/service/gateway_tool_changes_beta_test.go", + "backend/internal/service/gateway_upstream_request.go", + "backend/internal/service/model_protocol_catalog.go", + "backend/internal/service/model_protocol_catalog_test.go", + "backend/internal/service/model_rate_limit.go", + "backend/internal/service/openai_apikey_responses_probe.go", + "backend/internal/service/openai_codex_model_metadata.go", + "backend/internal/service/openai_codex_model_metadata_test.go", + "backend/internal/service/openai_codex_models_pinned.go", + "backend/internal/service/openai_codex_models_pinned_test.go", + "backend/internal/service/openai_codex_transform.go", + "backend/internal/service/openai_gateway_chat_completions.go", + "backend/internal/service/openai_gateway_chat_completions_anthropic_native.go", + "backend/internal/service/openai_gateway_count_tokens.go", + "backend/internal/service/openai_gateway_forward.go", + "backend/internal/service/openai_gateway_messages.go", + "backend/internal/service/openai_gateway_messages_anthropic_native.go", + "backend/internal/service/openai_gateway_passthrough.go", + "backend/internal/service/openai_gateway_provider_claude_billing_test.go", + "backend/internal/service/openai_gateway_request_body.go", + "backend/internal/service/openai_gateway_response_handling.go", + "backend/internal/service/openai_gateway_responses_anthropic_native.go", + "backend/internal/service/openai_gateway_responses_empty_completed_test.go", + "backend/internal/service/openai_gateway_scheduling.go", + "backend/internal/service/openai_gateway_service_hotpath_test.go", + "backend/internal/service/openai_models_list.go", + "backend/internal/service/openai_models_list_test.go", + "backend/internal/service/openai_oauth_web_search_history.go", + "backend/internal/service/openai_oauth_web_search_history_forward_test.go", + "backend/internal/service/openai_oauth_web_search_history_test.go", + "backend/internal/service/openai_responses_namespace.go", + "backend/internal/service/openai_responses_namespace_benchmark_test.go", + "backend/internal/service/openai_responses_namespace_test.go", + "backend/internal/service/openai_ws_v2/passthrough_relay.go", + "backend/internal/service/openai_ws_v2/passthrough_relay_image_usage_test.go", + "backend/internal/service/openai_ws_v2_passthrough_adapter.go", + "backend/internal/service/openai_ws_v2_passthrough_image_usage_test.go", + "backend/internal/service/opencode_go.go", + "backend/internal/service/opencode_go_test.go", + "backend/internal/service/opencode_go_usage.go", + "backend/internal/service/opencode_go_usage_test.go", + "backend/internal/service/opencode_unsupported_models_routing_test.go", + "backend/internal/service/opencode_unsupported_models_test.go", + "backend/internal/service/ops_dashboard_models.go", + "backend/internal/service/platform_catalog.go", + "backend/internal/service/platform_catalog_test.go", + "backend/internal/service/platform_list_test.go", + "backend/internal/service/provider_profile.go", + "backend/internal/service/provider_profile_test.go", + "backend/internal/service/ratelimit_cline.go", + "backend/internal/service/ratelimit_cline_test.go", + "backend/internal/service/ratelimit_cn_providers.go", + "backend/internal/service/ratelimit_command_code.go", + "backend/internal/service/ratelimit_service.go", + "backend/internal/service/scheduler_snapshot_service.go", + "backend/internal/service/testdata/record_upstream_protocol_routing_main.py", + "backend/internal/service/testdata/upstream_protocol_routing_main_golden.json", + "backend/internal/service/upstream_billing_probe.go", + "backend/internal/service/upstream_billing_probe_test.go", + "backend/internal/service/upstream_models.go", + "backend/internal/service/upstream_protocol_routing.go", + "backend/internal/service/upstream_protocol_routing_matrix_test.go", + "backend/internal/service/upstream_protocol_routing_test.go", + "backend/internal/web/embed_on.go", + "backend/internal/web/embed_test.go", + "backend/migrations/242_drop_platform_check_constraints.sql", + "backend/migrations/platform_check_constraints_drop_migration_test.go", + "deploy/Dockerfile", + "frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts", + "frontend/src/api/admin/cnProviders.ts", + "frontend/src/api/admin/ops.ts", + "frontend/src/api/admin/settings.ts", + "frontend/src/api/admin/users.ts", + "frontend/src/components/account/AccountUsageCell.vue", + "frontend/src/components/account/CNProviderBalanceCell.vue", + "frontend/src/components/account/CNProviderQuotaCell.vue", + "frontend/src/components/account/CreateAccountModal.vue", + "frontend/src/components/account/EditAccountModal.vue", + "frontend/src/components/account/ModelWhitelistSelector.vue", + "frontend/src/components/account/OpenCodeGoProtocolRulesEditor.vue", + "frontend/src/components/account/__tests__/AccountUsageCell.spec.ts", + "frontend/src/components/account/__tests__/CreateAccountModal.spec.ts", + "frontend/src/components/account/__tests__/EditAccountModal.spec.ts", + "frontend/src/components/account/__tests__/ModelWhitelistSelector.spec.ts", + "frontend/src/components/account/__tests__/OpenCodeGoProtocolRulesEditor.spec.ts", + "frontend/src/components/account/__tests__/credentialsBuilder.platformCatalog.spec.ts", + "frontend/src/components/account/__tests__/credentialsBuilder.spec.ts", + "frontend/src/components/account/credentialsBuilder.ts", + "frontend/src/components/admin/ErrorPassthroughRulesModal.vue", + "frontend/src/components/admin/__tests__/ErrorPassthroughRulesModal.toggle.spec.ts", + "frontend/src/components/admin/account/AccountTableFilters.vue", + "frontend/src/components/admin/account/ScheduledTestsPanel.vue", + "frontend/src/components/admin/account/__tests__/AccountTableFilters.spec.ts", + "frontend/src/components/admin/account/__tests__/ScheduledTestsPanel.results.spec.ts", + "frontend/src/components/admin/channel/PricingEntryCard.vue", + "frontend/src/components/admin/channel/__tests__/PricingEntryCard.modelDefaultPrice.spec.ts", + "frontend/src/components/admin/group/GroupRPMOverridesModal.vue", + "frontend/src/components/admin/group/GroupRateMultipliersModal.vue", + "frontend/src/components/admin/group/__tests__/GroupRPMOverridesModal.edit.spec.ts", + "frontend/src/components/admin/group/__tests__/GroupRateMultipliersModal.requests.spec.ts", + "frontend/src/components/admin/monitor/MonitorTemplateManagerDialog.vue", + "frontend/src/components/admin/monitor/__tests__/MonitorTemplateManagerDialog.requests.spec.ts", + "frontend/src/components/admin/usage/UsageCleanupDialog.vue", + "frontend/src/components/admin/usage/UsageTable.vue", + "frontend/src/components/admin/usage/__tests__/UsageCleanupDialog.spec.ts", + "frontend/src/components/admin/usage/__tests__/UsageTable.spec.ts", + "frontend/src/components/admin/user/BulkEditUserModal.vue", + "frontend/src/components/admin/user/UserPlatformQuotaModal.vue", + "frontend/src/components/admin/user/__tests__/BulkEditUserModal.spec.ts", + "frontend/src/components/admin/user/__tests__/UserPlatformQuotaModal.spec.ts", + "frontend/src/components/auth/PendingOAuthCreateAccountForm.vue", + "frontend/src/components/auth/__tests__/PendingOAuthCreateAccountForm.spec.ts", + "frontend/src/components/common/AnnouncementBell.vue", + "frontend/src/components/common/BaseDialog.vue", + "frontend/src/components/common/PlatformIcon.vue", + "frontend/src/components/common/ProxySelector.vue", + "frontend/src/components/common/Select.vue", + "frontend/src/components/common/__tests__/AnnouncementBell.spec.ts", + "frontend/src/components/common/__tests__/BaseDialog.escape.spec.ts", + "frontend/src/components/common/__tests__/disabledSelectors.spec.ts", + "frontend/src/components/keys/UseKeyModal.vue", + "frontend/src/components/keys/__tests__/UseKeyModal.spec.ts", + "frontend/src/components/user/UserPlatformQuotaCell.vue", + "frontend/src/components/user/profile/ProfileBalanceNotifyCard.vue", + "frontend/src/components/user/profile/ProfileEditForm.vue", + "frontend/src/components/user/profile/ProfileIdentityBindingsSection.vue", + "frontend/src/components/user/profile/__tests__/ProfileBalanceNotifyCard.spec.ts", + "frontend/src/components/user/profile/__tests__/ProfileEditForm.draft.spec.ts", + "frontend/src/components/user/profile/__tests__/ProfileIdentityBindingsSection.spec.ts", + "frontend/src/composables/__tests__/useBatchImageAccess.retry.spec.ts", + "frontend/src/composables/__tests__/useModelWhitelist.spec.ts", + "frontend/src/composables/__tests__/useStepUp.spec.ts", + "frontend/src/composables/useBatchImageAccess.ts", + "frontend/src/composables/useModelWhitelist.ts", + "frontend/src/composables/useStepUp.ts", + "frontend/src/constants/__tests__/platforms.spec.ts", + "frontend/src/constants/platformCatalog.builtin.json", + "frontend/src/constants/platformCatalog.ts", + "frontend/src/constants/platforms.ts", + "frontend/src/i18n/locales/en/admin/accounts.ts", + "frontend/src/i18n/locales/en/admin/ops.ts", + "frontend/src/i18n/locales/en/admin/resources.ts", + "frontend/src/i18n/locales/en/dashboard.ts", + "frontend/src/i18n/locales/zh/admin/accounts.ts", + "frontend/src/i18n/locales/zh/admin/ops.ts", + "frontend/src/i18n/locales/zh/admin/resources.ts", + "frontend/src/i18n/locales/zh/dashboard.ts", + "frontend/src/stores/__tests__/adminCompliance.reset.spec.ts", + "frontend/src/stores/adminCompliance.ts", + "frontend/src/stores/announcements.ts", + "frontend/src/types/index.ts", + "frontend/src/utils/__tests__/formatBytes.spec.ts", + "frontend/src/utils/__tests__/latencyHealth.spec.ts", + "frontend/src/utils/__tests__/pricing.formatScaled.spec.ts", + "frontend/src/utils/format.ts", + "frontend/src/utils/keyGroupProviders.ts", + "frontend/src/utils/latencyHealth.ts", + "frontend/src/utils/platformColors.ts", + "frontend/src/utils/pricing.ts", + "frontend/src/views/admin/AccountsView.vue", + "frontend/src/views/admin/BackupView.vue", + "frontend/src/views/admin/ChannelsView.vue", + "frontend/src/views/admin/GroupsView.vue", + "frontend/src/views/admin/PluginsView.vue", + "frontend/src/views/admin/SettingsView.vue", + "frontend/src/views/admin/__tests__/BackupView.spec.ts", + "frontend/src/views/admin/__tests__/ChannelsView.modelSync.spec.ts", + "frontend/src/views/admin/__tests__/PluginsView.spec.ts", + "frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts", + "frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue", + "frontend/src/views/admin/ops/components/OpsAlertRulesCard.vue", + "frontend/src/views/admin/ops/components/OpsDashboardHeader.vue", + "frontend/src/views/admin/ops/components/OpsDashboardSkeleton.vue", + "frontend/src/views/admin/ops/components/OpsSettingsDialog.vue", + "frontend/src/views/admin/ops/components/__tests__/OpsAlertEventsCard.pagination.spec.ts", + "frontend/src/views/admin/ops/components/__tests__/OpsAlertRulesCard.duration.spec.ts", + "frontend/src/views/admin/ops/components/__tests__/OpsDashboardHeader.spec.ts", + "frontend/src/views/admin/ops/components/__tests__/OpsSettingsDialog.loading.spec.ts", + "frontend/src/views/user/AirwallexPaymentView.vue", + "frontend/src/views/user/CustomPageView.vue", + "frontend/src/views/user/StripePaymentView.vue", + "frontend/src/views/user/__tests__/AirwallexPaymentView.lifecycle.spec.ts", + "frontend/src/views/user/__tests__/CustomPageView.race.spec.ts", + "frontend/src/views/user/__tests__/StripePaymentView.spec.ts", + "frontend/src/views/user/__tests__/UsageView.spec.ts" ], - "upstream_tag": "v0.2.14" + "upstream_tag": "v0.2.15" } diff --git a/.github/scripts/test_reasoning_recovery.sh b/.github/scripts/test_reasoning_recovery.sh deleted file mode 100644 index b30e4de78ab8..000000000000 --- a/.github/scripts/test_reasoning_recovery.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -cd "$(dirname "$0")/../.." - -# Offline contracts only; no account credentials or model calls. -go -C backend test -p=1 -tags unit ./internal/pkg/apicompat ./internal/repository ./internal/service ./internal/handler/admin \ - -run 'Test(OpenAIReasoning|OpenAIHTTPTerminal|GatewayCacheReasoningState|ResponsesReasoningConfiguration|ReasoningRecovery)' -count=1 diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index 467d62c64e55..bf8a2ba4579f 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -40,12 +40,6 @@ jobs: - name: Verify Go version run: | go version | grep -q 'go1.27.2' - - name: Verify Go modules - working-directory: backend - run: go mod verify - - name: Go vet - working-directory: backend - run: go vet ./... - name: Unit tests working-directory: backend run: make test-unit diff --git a/.github/workflows/downstream-verify.yml b/.github/workflows/downstream-verify.yml index ed9099a05366..abc00084979e 100644 --- a/.github/workflows/downstream-verify.yml +++ b/.github/workflows/downstream-verify.yml @@ -34,8 +34,6 @@ jobs: python3 .github/scripts/test_release_preflight.py - name: Agent protocol contracts run: bash .github/scripts/test_agent_protocol.sh - - name: Reasoning recovery contracts - run: bash .github/scripts/test_reasoning_recovery.sh - name: Model context capacity contracts run: bash .github/scripts/test_model_context_capacity.sh - name: Service tests @@ -142,7 +140,6 @@ jobs: src/components/account/__tests__/CreateAccountModal.spec.ts \ src/components/payment/__tests__/StripePaymentInline.contrast.spec.ts \ src/views/user/__tests__/StripePaymentView.spec.ts \ - src/views/admin/__tests__/ReasoningRecoveryView.spec.ts \ src/views/admin/ops/components/__tests__/OpsErrorDetailModal.spec.ts \ src/api/admin/__tests__/codexGatewayBorrow.spec.ts \ src/components/admin/codex/__tests__/BorrowPelicanPreview.spec.ts \ diff --git a/DOWNSTREAM.md b/DOWNSTREAM.md index 2eba511d356a..47d15b7dae15 100644 --- a/DOWNSTREAM.md +++ b/DOWNSTREAM.md @@ -5,7 +5,7 @@ This fork maintains the `codexrip` patch set over official Sub2API releases. ## Source baseline - Integrated official baseline: the release recorded in `.downstream/upstream-base`. -- Retained compatibility decisions: [upstream review](.downstream/upstream-review-v0.2.14.md). +- Retained compatibility decisions: [upstream review](.downstream/upstream-review-v0.2.15.md). - The operator workspace `docs/sub2api.md` owns the production version and image digest. A source merge or Release does not establish deployment completion. ## Settings and account operations @@ -159,15 +159,36 @@ identity and verified-replay checks remain in force. and manual routing stay within their existing configuration. - Container/public health confirms availability, not actual model functionality. +## OpenAI encrypted-content recovery + +HTTP and WebSocket encrypted-content recovery follows official v0.2.15, +including the official session lineage memory. The former downstream targeted +stripping, persistent rejection memory, SSE repair, missing-item-id repair and +special account failovers are retired. The global recovery page and its +`GET/PUT /api/v1/admin/reasoning-recovery` interface are removed. Stored settings, +historical SQL and diagnostic records remain intact; ordinary Claude thinking +signature recovery is independent and remains available. + +Cline and Command Code share the existing system-prompt insertion points. +Connection-test reasoning options require declared model support and a wire +path that retains the chosen effort; unsupported and `ultra` options remain +unavailable. The compact account filters retain multi-selection, tags, proxy, +plan and count controls; hiding additional filters never resets their values. + +Official migration 242 removes platform CHECK constraints. Immutable downstream +251, 260 and 265 restore the quota constraint during a fresh installation, so +277 repeats the final removal after them. No historical checksum or existing +quota/route row is changed by 242 or 277. + ## Upstream updates Scheduled discovery may identify newer official releases. It cannot deploy production. Each integration records its selected official commit, conflicts and retained downstream contracts before a new immutable release is published. -`.downstream/upstream-risk.json` records the v0.2.13-to-v0.2.14 integration against -downstream commit `7132011d7a4275e7d6407c92db4ec205a1c14ad1`. The risk gate -recomputes those exact inputs. `initial_conflict_files` lists the three dependency +`.downstream/upstream-risk.json` records the v0.2.14-to-v0.2.15 integration against +downstream commit `d469422e94b0655b22dd9d0a826decda61ffb91e`. The risk gate +recomputes those exact inputs. `initial_conflict_files` lists the 40 conflicts resolved during this merge; `merge_conflicts: []` records that none remain. A `review_required` candidate needs a real completed review and the `upstream-reviewed` label before promotion. diff --git a/Makefile b/Makefile index bc978c70f1b6..361e7835e098 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,12 @@ FRONTEND_CRITICAL_VITEST := \ src/components/account/__tests__/OpenAIReferralCell.spec.ts \ src/components/account/__tests__/OpenAIReferralCell.transport.spec.ts \ src/components/account/__tests__/OpenAIQuotaResetCell.spark_shadow.spec.ts \ + src/constants/__tests__/platforms.spec.ts \ + src/components/account/__tests__/credentialsBuilder.platformCatalog.spec.ts \ + src/components/account/__tests__/CreateAccountModal.spec.ts \ + src/components/account/__tests__/EditAccountModal.spec.ts \ + src/components/account/__tests__/credentialsBuilder.spec.ts \ + src/components/account/__tests__/OpenCodeGoProtocolRulesEditor.spec.ts \ src/components/keys/__tests__/BulkEditKeysModal.spec.ts \ src/components/admin/user/__tests__/UserPlatformQuotaModal.spec.ts \ src/views/user/__tests__/KeysView.spec.ts \ @@ -18,6 +24,7 @@ FRONTEND_CRITICAL_VITEST := \ src/views/user/__tests__/PaymentResultView.spec.ts \ src/views/user/__tests__/ChannelStatusView.mode.spec.ts \ src/components/user/profile/__tests__/ProfileInfoCard.spec.ts \ + src/components/user/profile/__tests__/ProfileIdentityBindingsSection.spec.ts \ src/views/admin/__tests__/SettingsView.spec.ts \ src/components/account/__tests__/EditAccountModal.spec.ts \ src/components/account/__tests__/ModelWhitelistSelector.spec.ts \ @@ -25,7 +32,8 @@ FRONTEND_CRITICAL_VITEST := \ src/components/layout/__tests__/AppSidebar.simple-mode.spec.ts \ src/features/channel-monitor-v2/__tests__/designSystem.structure.spec.ts \ src/features/channel-monitor-v2/__tests__/monitorFormat.spec.ts \ - src/features/channel-monitor-v2/__tests__/monitorZoom.spec.ts + src/features/channel-monitor-v2/__tests__/monitorZoom.spec.ts \ + src/components/admin/channel/__tests__/PricingEntryCard.modelDefaultPrice.spec.ts # 一键编译前后端 build: build-backend build-frontend diff --git a/backend/cmd/server/VERSION b/backend/cmd/server/VERSION index 769ed6ae7908..b005e307c519 100644 --- a/backend/cmd/server/VERSION +++ b/backend/cmd/server/VERSION @@ -1 +1 @@ -0.2.14 +0.2.15 diff --git a/backend/cmd/server/wire_gen.go b/backend/cmd/server/wire_gen.go index abfb409a7db2..907c1f0804d2 100644 --- a/backend/cmd/server/wire_gen.go +++ b/backend/cmd/server/wire_gen.go @@ -160,11 +160,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { grokOAuthClient := repository.NewGrokOAuthClient() grokOAuthService := service.ProvideGrokOAuthService(proxyRepository, grokOAuthClient, configConfig, redisClient) grokTokenProvider := service.ProvideGrokTokenProvider(accountRepository, geminiTokenCache, grokOAuthService, oAuthRefreshAPI, tempUnschedCache) - reasoningRecoveryService, err := service.ProvideReasoningRecoveryService(settingRepository) - if err != nil { - return nil, err - } - openAIGatewayService := service.ProvideOpenAIGatewayService(accountRepository, usageLogRepository, usageBillingRepository, userRepository, userSubscriptionRepository, userGroupRateRepository, gatewayCache, configConfig, schedulerSnapshotService, concurrencyService, billingService, rateLimitService, billingCacheService, httpUpstream, deferredService, openAITokenProvider, grokTokenProvider, modelPricingResolver, channelService, balanceNotifyService, settingService, serviceUserPlatformQuotaRepository, systemPromptService, reasoningRecoveryService) + openAIGatewayService := service.ProvideOpenAIGatewayService(accountRepository, usageLogRepository, usageBillingRepository, userRepository, userSubscriptionRepository, userGroupRateRepository, gatewayCache, configConfig, schedulerSnapshotService, concurrencyService, billingService, rateLimitService, billingCacheService, httpUpstream, deferredService, openAITokenProvider, grokTokenProvider, modelPricingResolver, channelService, balanceNotifyService, settingService, serviceUserPlatformQuotaRepository, systemPromptService) geminiOAuthClient := repository.NewGeminiOAuthClient(configConfig) geminiCliCodeAssistClient := repository.NewGeminiCliCodeAssistClient() driveClient := repository.NewGeminiDriveClient() @@ -303,7 +299,6 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { auditLogService := service.ProvideAuditLogService(auditLogRepository, settingService) auditLogHandler := admin.NewAuditLogHandler(auditLogService, totpService) systemPromptHandler := admin.NewSystemPromptHandler(systemPromptService) - reasoningRecoveryHandler := admin.NewReasoningRecoveryHandler(reasoningRecoveryService) codexGatewayBorrowProbeUpstream := repository.NewCodexGatewayBorrowProbeUpstream(configConfig) codexGatewayBorrowService, err := service.ProvideCodexGatewayBorrowService(settingRepository, accountRepository, openAIGatewayService, codexGatewayBorrowProbeUpstream, tlsFingerprintProfileService) if err != nil { @@ -319,7 +314,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { openCodeGoUsageService := service.ProvideOpenCodeGoUsageService(accountRepository, httpUpstream, settingService, leaderLockCache, db) idempotencyCoordinator := service.ProvideIdempotencyCoordinator(idempotencyRepository, configConfig) claudeResetCreditService := service.ProvideClaudeResetCreditService(accountRepository, claudeTokenProvider, proxyRepository, settingService, idempotencyCoordinator, leaderLockCache) - adminHandlers := handler.ProvideAdminHandlers(dashboardHandler, adminUserHandler, groupHandler, accountHandler, accountJobHandler, adminAnnouncementHandler, dataManagementHandler, backupHandler, oAuthHandler, openAIOAuthHandler, geminiOAuthHandler, antigravityOAuthHandler, grokOAuthHandler, cnProviderHandler, proxyHandler, adminRedeemHandler, promoHandler, settingHandler, opsHandler, systemHandler, adminSubscriptionHandler, adminUsageHandler, userAttributeHandler, errorPassthroughHandler, tlsFingerprintProfileHandler, pluginHandler, adminAPIKeyHandler, scheduledTestHandler, channelHandler, channelMonitorHandler, channelMonitorRequestTemplateHandler, contentModerationHandler, promptAdminHandler, paymentHandler, affiliateHandler, complianceHandler, auditLogHandler, systemPromptHandler, reasoningRecoveryHandler, codexGatewayBorrowHandler, pelicanTestHandler, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, claudeResetCreditService) + adminHandlers := handler.ProvideAdminHandlers(dashboardHandler, adminUserHandler, groupHandler, accountHandler, accountJobHandler, adminAnnouncementHandler, dataManagementHandler, backupHandler, oAuthHandler, openAIOAuthHandler, geminiOAuthHandler, antigravityOAuthHandler, grokOAuthHandler, cnProviderHandler, proxyHandler, adminRedeemHandler, promoHandler, settingHandler, opsHandler, systemHandler, adminSubscriptionHandler, adminUsageHandler, userAttributeHandler, errorPassthroughHandler, tlsFingerprintProfileHandler, pluginHandler, adminAPIKeyHandler, scheduledTestHandler, channelHandler, channelMonitorHandler, channelMonitorRequestTemplateHandler, contentModerationHandler, promptAdminHandler, paymentHandler, affiliateHandler, complianceHandler, auditLogHandler, systemPromptHandler, codexGatewayBorrowHandler, pelicanTestHandler, upstreamBillingProbeService, ollamaCloudUsageService, openCodeGoUsageService, claudeResetCreditService) usageRecordWorkerPool := service.NewUsageRecordWorkerPool(configConfig) userMsgQueueCache := repository.NewUserMsgQueueCache(redisClient) userMessageQueueService := service.ProvideUserMessageQueueService(userMsgQueueCache, rpmCache, configConfig) diff --git a/backend/ent/runtime/runtime.go b/backend/ent/runtime/runtime.go index b28ef960eafd..1e12bb220466 100644 --- a/backend/ent/runtime/runtime.go +++ b/backend/ent/runtime/runtime.go @@ -1025,7 +1025,21 @@ func init() { // compositemodelroute.DefaultTargetPlatform holds the default value on creation for the target_platform field. compositemodelroute.DefaultTargetPlatform = compositemodelrouteDescTargetPlatform.Default.(string) // compositemodelroute.TargetPlatformValidator is a validator for the "target_platform" field. It is called by the builders before save. - compositemodelroute.TargetPlatformValidator = compositemodelrouteDescTargetPlatform.Validators[0].(func(string) error) + compositemodelroute.TargetPlatformValidator = func() func(string) error { + validators := compositemodelrouteDescTargetPlatform.Validators + fns := [...]func(string) error{ + validators[0].(func(string) error), + validators[1].(func(string) error), + } + return func(target_platform string) error { + for _, fn := range fns { + if err := fn(target_platform); err != nil { + return err + } + } + return nil + } + }() // compositemodelrouteDescUpstreamModel is the schema descriptor for upstream_model field. compositemodelrouteDescUpstreamModel := compositemodelrouteFields[4].Descriptor() // compositemodelroute.DefaultUpstreamModel holds the default value on creation for the upstream_model field. diff --git a/backend/ent/schema/composite_model_route.go b/backend/ent/schema/composite_model_route.go index 82b5bb274b53..3e44aa5101b0 100644 --- a/backend/ent/schema/composite_model_route.go +++ b/backend/ent/schema/composite_model_route.go @@ -1,6 +1,8 @@ package schema import ( + "fmt" + "github.com/Wei-Shaw/sub2api/ent/schema/mixins" "github.com/Wei-Shaw/sub2api/internal/domain" @@ -45,6 +47,13 @@ func (CompositeModelRoute) Fields() []ent.Field { field.String("target_platform"). MaxLen(50). Default(domain.PlatformOpenAI). + Validate(func(s string) error { + // 目标平台须为平台清单中的具体平台;数据库不再维护 CHECK 约束。 + if !domain.IsConcretePlatform(s) { + return fmt.Errorf("target_platform %q is not a concrete platform", s) + } + return nil + }). Comment("Concrete provider platform."), field.String("upstream_model"). MaxLen(200). diff --git a/backend/ent/schema/user_platform_quota.go b/backend/ent/schema/user_platform_quota.go index 663be0a6ecbc..3949a09721ec 100644 --- a/backend/ent/schema/user_platform_quota.go +++ b/backend/ent/schema/user_platform_quota.go @@ -11,6 +11,7 @@ import ( "entgo.io/ent/schema/field" "entgo.io/ent/schema/index" "github.com/Wei-Shaw/sub2api/ent/schema/mixins" + "github.com/Wei-Shaw/sub2api/internal/domain" ) // UserPlatformQuota holds the schema definition for per-user per-platform quota. @@ -38,15 +39,12 @@ func (UserPlatformQuota) Fields() []ent.Field { MaxLen(32). NotEmpty(). Validate(func(s string) error { - // 注意:平台列表的单一权威源为 service.AllowedQuotaPlatforms; - // 此处为 ent 构建期约束,需与 service.AllowedQuotaPlatforms 保持同步。 - switch s { - case "anthropic", "openai", "gemini", "antigravity", "grok", - "kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe": - return nil - default: + // 平台白名单为平台清单(domain/platforms.go)中的全部具体平台, + // 与 service.AllowedQuotaPlatforms 同源;数据库不再维护 CHECK 约束。 + if !domain.IsConcretePlatform(s) { return fmt.Errorf("platform %q is not allowed", s) } + return nil }), // 日 / 周 / 月 USD 上限: diff --git a/backend/go.mod b/backend/go.mod index 73c1ad07c73e..3e660d61df34 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -19,6 +19,7 @@ require ( github.com/coder/websocket v1.8.14 github.com/dgraph-io/ristretto v0.2.0 github.com/gin-gonic/gin v1.9.1 + github.com/go-playground/validator/v10 v10.14.0 github.com/go-webauthn/webauthn v0.17.4 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 @@ -122,7 +123,6 @@ require ( github.com/go-openapi/inflect v0.19.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/go-webauthn/x v0.2.6 // indirect github.com/goccy/go-json v0.10.2 // indirect diff --git a/backend/go.sum b/backend/go.sum index 363311a9e1ae..97b747c1945f 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -666,6 +666,7 @@ golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxD golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 h1:F5BWKvW126NXR74uxkxuc1jQHhm/rwm/J3rSiFyuRs4= +golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/backend/internal/domain/constants.go b/backend/internal/domain/constants.go index 37cb3a738f7b..f889281e874c 100644 --- a/backend/internal/domain/constants.go +++ b/backend/internal/domain/constants.go @@ -30,9 +30,15 @@ const ( PlatformMiniMax = "minimax" // MiniMax (M 系列) PlatformTypeSafe = "typesafe" // TypeSafe AI System One (Jev) // PlatformOpenCodeGo 是 OpenCode 平台(账号类型 Zen 按量 / Go 订阅)。 - // 值保持 opencode_go 以兼容已落库的分组、配额与 Composite 路由 CHECK。 + // 值保持 opencode_go 以兼容已落库的账号、分组、配额与 Composite 路由。 PlatformOpenCodeGo = "opencode_go" - PlatformComposite = "composite" + // PlatformCommandCode 是 Command Code Provider API(多模型聚合,积分计费, + // 订阅套餐另有 5 小时 / 每周滚动窗口)。 + PlatformCommandCode = "command_code" + // PlatformCline 是 Cline API(多模型聚合,OpenAI 兼容 Chat Completions;按量积分计费, + // 另有 ClinePass 订阅)。 + PlatformCline = "cline" + PlatformComposite = "composite" ) // Account mode constants 区分国产供应商的「按量付费(余额)」与「Coding Plan」两种接入方式。 diff --git a/backend/internal/domain/platforms.go b/backend/internal/domain/platforms.go new file mode 100644 index 000000000000..1bb41813a045 --- /dev/null +++ b/backend/internal/domain/platforms.go @@ -0,0 +1,137 @@ +package domain + +// 平台清单:平台的单一数据来源。"所有平台"类列表(分组平台、用户额度、组合 +// 路由目标、调度快照、定价匹配等)与"某网关族"类列表都从这里派生;新增平台 +// 登记一条即可在这些位置生效,无需逐处修改或新增数据库约束迁移。 +// +// 依赖具体实现的能力(OAuth、渠道监控、余额 / Coding Plan 探测、用量窗口、 +// Codex 模型目录等)仍由各自的代码显式列出,不在此处声明。 + +// PlatformGateway 标识平台请求由哪个网关服务处理。 +type PlatformGateway string + +const ( + // PlatformGatewayAnthropic 由 GatewayService 处理(Claude Messages 协议,以及 TypeSafe 的 /v1/systemone)。 + PlatformGatewayAnthropic PlatformGateway = "anthropic" + // PlatformGatewayOpenAI 由 OpenAIGatewayService 处理(OpenAI、Grok 与多协议 API Key 供应商)。 + PlatformGatewayOpenAI PlatformGateway = "openai" + // PlatformGatewayGemini 由 Gemini 兼容网关处理。 + PlatformGatewayGemini PlatformGateway = "gemini" + // PlatformGatewayAntigravity 由 Antigravity 网关处理。 + PlatformGatewayAntigravity PlatformGateway = "antigravity" +) + +// PlatformSpec 描述一个具体(非 composite)平台。 +type PlatformSpec struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` + Gateway PlatformGateway `json:"gateway"` + // CNProvider 标识国产单一厂商(Kimi / 智谱 / DeepSeek / MiniMax),共享国产 + // 供应商的余额 / Coding Plan 等能力。 + CNProvider bool `json:"cn_provider"` + // LiteLLMProvider 是定价目录中的 provider 名;空表示不支持从定价目录同步模型。 + LiteLLMProvider string `json:"litellm_provider,omitempty"` +} + +// platformList 的顺序即平台的展示顺序。 +var platformList = []PlatformSpec{ + {ID: PlatformAnthropic, DisplayName: "Anthropic", Gateway: PlatformGatewayAnthropic, LiteLLMProvider: "anthropic"}, + {ID: PlatformOpenAI, DisplayName: "OpenAI", Gateway: PlatformGatewayOpenAI, LiteLLMProvider: "openai"}, + {ID: PlatformGemini, DisplayName: "Gemini", Gateway: PlatformGatewayGemini, LiteLLMProvider: "gemini"}, + {ID: PlatformAntigravity, DisplayName: "Antigravity", Gateway: PlatformGatewayAntigravity, LiteLLMProvider: "anthropic"}, + {ID: PlatformGrok, DisplayName: "Grok", Gateway: PlatformGatewayOpenAI, LiteLLMProvider: "xai"}, + {ID: PlatformKimi, DisplayName: "Kimi", Gateway: PlatformGatewayOpenAI, CNProvider: true, LiteLLMProvider: "moonshot"}, + {ID: PlatformZhipu, DisplayName: "Zhipu GLM", Gateway: PlatformGatewayOpenAI, CNProvider: true, LiteLLMProvider: "zhipu"}, + {ID: PlatformDeepseek, DisplayName: "DeepSeek", Gateway: PlatformGatewayOpenAI, CNProvider: true, LiteLLMProvider: "deepseek"}, + {ID: PlatformMiniMax, DisplayName: "MiniMax", Gateway: PlatformGatewayOpenAI, CNProvider: true, LiteLLMProvider: "minimax"}, + {ID: PlatformOpenCodeGo, DisplayName: "OpenCode", Gateway: PlatformGatewayOpenAI, LiteLLMProvider: "opencode-go"}, + {ID: PlatformTypeSafe, DisplayName: "TypeSafe / Jev", Gateway: PlatformGatewayAnthropic, LiteLLMProvider: "typesafe"}, + {ID: PlatformCommandCode, DisplayName: "Command Code", Gateway: PlatformGatewayOpenAI}, + {ID: PlatformCline, DisplayName: "Cline", Gateway: PlatformGatewayOpenAI}, +} + +var platformIndex = func() map[string]int { + index := make(map[string]int, len(platformList)) + for i, spec := range platformList { + index[spec.ID] = i + } + return index +}() + +// Platforms 返回全部具体平台(副本),按展示顺序。 +func Platforms() []PlatformSpec { + out := make([]PlatformSpec, len(platformList)) + copy(out, platformList) + return out +} + +// LookupPlatform 返回具体平台的登记信息;composite 与未知值返回 false。 +func LookupPlatform(id string) (PlatformSpec, bool) { + i, ok := platformIndex[id] + if !ok { + return PlatformSpec{}, false + } + return platformList[i], true +} + +// IsConcretePlatform 报告 id 是否为已登记的具体平台(不含 composite)。 +func IsConcretePlatform(id string) bool { + _, ok := platformIndex[id] + return ok +} + +// IsGroupPlatform 报告 id 是否可作为分组平台(具体平台或 composite)。 +func IsGroupPlatform(id string) bool { + return id == PlatformComposite || IsConcretePlatform(id) +} + +// ConcretePlatformIDs 返回全部具体平台标识,按展示顺序。 +func ConcretePlatformIDs() []string { + return PlatformIDsWhere(func(PlatformSpec) bool { return true }) +} + +// PlatformIDsWhere 返回满足条件的具体平台标识,按展示顺序。 +func PlatformIDsWhere(match func(PlatformSpec) bool) []string { + out := make([]string, 0, len(platformList)) + for _, spec := range platformList { + if match(spec) { + out = append(out, spec.ID) + } + } + return out +} + +// compositePrecedenceHead 是组合分组查找顺序的固定前缀(历史顺序:Gemini 先于 OpenAI)。 +var compositePrecedenceHead = []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI} + +// CompositePrecedencePlatformIDs 返回按平台依次回退时的顺序:组合分组在请求目标 +// 解析前逐平台查找定价 / 模型映射(先命中者生效)、拼接默认模型列表,以及调度快照 +// 遍历。沿用历史顺序 anthropic、gemini、openai,其余按展示顺序,新登记的平台追加在末尾。 +func CompositePrecedencePlatformIDs() []string { + out := make([]string, 0, len(platformList)) + seen := make(map[string]struct{}, len(platformList)) + for _, id := range compositePrecedenceHead { + if IsConcretePlatform(id) { + out = append(out, id) + seen[id] = struct{}{} + } + } + for _, spec := range platformList { + if _, ok := seen[spec.ID]; !ok { + out = append(out, spec.ID) + } + } + return out +} + +// UsesOpenAIGateway 报告平台是否由 OpenAI 网关处理。 +func UsesOpenAIGateway(id string) bool { + spec, ok := LookupPlatform(id) + return ok && spec.Gateway == PlatformGatewayOpenAI +} + +// IsCNProviderPlatform 报告平台是否为国产单一厂商。 +func IsCNProviderPlatform(id string) bool { + spec, ok := LookupPlatform(id) + return ok && spec.CNProvider +} diff --git a/backend/internal/domain/platforms_test.go b/backend/internal/domain/platforms_test.go new file mode 100644 index 000000000000..248218a322b9 --- /dev/null +++ b/backend/internal/domain/platforms_test.go @@ -0,0 +1,74 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// 重构前各处手写的平台列表,作为平台清单派生结果的等价基准;重构后新登记的平台 +// (Command Code、Cline)按同类平台(OpenCode)的位置补入。 +var ( + legacyDisplayOrder = []string{ + PlatformAnthropic, PlatformOpenAI, PlatformGemini, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, + PlatformCommandCode, + PlatformCline, + } + legacyCompositePrecedence = []string{ + PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, + PlatformCommandCode, + PlatformCline, + } + legacyOpenAIGateway = []string{ + PlatformOpenAI, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, + PlatformCommandCode, + PlatformCline, + } + legacyCNProviders = []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax} + legacyLiteLLMByPlatf = map[string]string{ + PlatformAnthropic: "anthropic", PlatformOpenAI: "openai", PlatformGemini: "gemini", + PlatformAntigravity: "anthropic", PlatformGrok: "xai", PlatformKimi: "moonshot", + PlatformZhipu: "zhipu", PlatformDeepseek: "deepseek", PlatformMiniMax: "minimax", + PlatformOpenCodeGo: "opencode-go", PlatformTypeSafe: "typesafe", PlatformCommandCode: "", PlatformCline: "", + } +) + +func TestPlatformListMatchesLegacyLists(t *testing.T) { + require.Equal(t, legacyDisplayOrder, ConcretePlatformIDs()) + require.Equal(t, legacyCompositePrecedence, CompositePrecedencePlatformIDs()) + require.Equal(t, legacyOpenAIGateway, PlatformIDsWhere(func(spec PlatformSpec) bool { + return spec.Gateway == PlatformGatewayOpenAI + })) + require.Equal(t, legacyCNProviders, PlatformIDsWhere(func(spec PlatformSpec) bool { return spec.CNProvider })) + for platform, provider := range legacyLiteLLMByPlatf { + spec, ok := LookupPlatform(platform) + require.True(t, ok, platform) + require.Equal(t, provider, spec.LiteLLMProvider, platform) + } +} + +func TestPlatformListPredicates(t *testing.T) { + for _, platform := range legacyDisplayOrder { + require.True(t, IsConcretePlatform(platform), platform) + require.True(t, IsGroupPlatform(platform), platform) + spec, ok := LookupPlatform(platform) + require.True(t, ok, platform) + require.NotEmpty(t, spec.DisplayName, platform) + require.NotEmpty(t, spec.Gateway, platform) + } + require.False(t, IsConcretePlatform(PlatformComposite)) + require.True(t, IsGroupPlatform(PlatformComposite)) + for _, invalid := range []string{"", "moonshot", "Kimi", "openai ", "glm", "bogus"} { + require.False(t, IsConcretePlatform(invalid), invalid) + require.False(t, IsGroupPlatform(invalid), invalid) + require.False(t, UsesOpenAIGateway(invalid), invalid) + } +} + +func TestPlatformsReturnsCopy(t *testing.T) { + platforms := Platforms() + platforms[0].ID = "mutated" + require.Equal(t, PlatformAnthropic, Platforms()[0].ID) +} diff --git a/backend/internal/handler/admin/account_handler.go b/backend/internal/handler/admin/account_handler.go index 453f9e7b64b4..9178ed86a0c8 100644 --- a/backend/internal/handler/admin/account_handler.go +++ b/backend/internal/handler/admin/account_handler.go @@ -1276,7 +1276,7 @@ func (h *AccountHandler) Update(c *gin.Context) { // 网关会按"现状即证据"默认走 Responses。 func (h *AccountHandler) scheduleOpenAIResponsesProbe(account *service.Account) { if account == nil || account.Type != service.AccountTypeAPIKey || - (account.Platform != service.PlatformOpenAI && !service.IsCNProvider(account.Platform)) { + (account.Platform != service.PlatformOpenAI && !account.RoutesProtocolByInbound()) { return } if h.accountTestService == nil { @@ -2826,10 +2826,9 @@ func (h *AccountHandler) accountTestCatalog(ctx context.Context, account *servic return models, nil } - // Handle Antigravity accounts: return Claude + Gemini models + // Explicit account mappings expose their request-side names to connectivity tests. if account.Platform == service.PlatformAntigravity { - // 直接复用 antigravity.DefaultModels(),与 /v1/models 端点保持同步 - return antigravity.DefaultModels(), nil + return antigravityAccountTestModels(account.Credentials["model_mapping"]), nil } // Handle Grok accounts @@ -3109,6 +3108,48 @@ func (h *AccountHandler) PreviewModelContextCapacities(c *gin.Context) { response.Success(c, gin.H{"capacity_rows": service.BuildAccountModelContextCapacityRows(account, req.ModelIDs)}) } +// antigravityAccountTestModels uses the stored mapping rather than GetModelMapping, +// which supplies defaults and compatibility aliases that the administrator did not configure. +func antigravityAccountTestModels(rawMapping any) []antigravity.ClaudeModel { + var mappedIDs []string + switch mapping := rawMapping.(type) { + case map[string]any: + for id := range mapping { + if strings.TrimSpace(id) != "" { + mappedIDs = append(mappedIDs, id) + } + } + case map[string]string: + for id := range mapping { + if strings.TrimSpace(id) != "" { + mappedIDs = append(mappedIDs, id) + } + } + } + if len(mappedIDs) == 0 { + return antigravity.DefaultModels() + } + + sort.Strings(mappedIDs) + defaultByID := make(map[string]antigravity.ClaudeModel) + for _, model := range antigravity.DefaultModels() { + defaultByID[model.ID] = model + } + models := make([]antigravity.ClaudeModel, 0, len(mappedIDs)) + for _, id := range mappedIDs { + if model, ok := defaultByID[id]; ok { + models = append(models, model) + continue + } + models = append(models, antigravity.ClaudeModel{ + ID: id, + Type: "model", + DisplayName: id, + }) + } + return models +} + // SyncUpstreamModels handles syncing live supported models from an account's upstream. // POST /api/v1/admin/accounts/:id/models/sync-upstream func (h *AccountHandler) SyncUpstreamModels(c *gin.Context) { diff --git a/backend/internal/handler/admin/account_handler_available_models_test.go b/backend/internal/handler/admin/account_handler_available_models_test.go index 187e101084a7..4f46581f5a4f 100644 --- a/backend/internal/handler/admin/account_handler_available_models_test.go +++ b/backend/internal/handler/admin/account_handler_available_models_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/antigravity" "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" "github.com/Wei-Shaw/sub2api/internal/service" @@ -368,6 +369,49 @@ func TestAccountHandlerGetAvailableModels_GeminiGoogleOneUsesConservativeCatalog require.NotContains(t, ids, "gemini-2.5-flash-image") } +func TestAccountHandlerGetAvailableModels_Antigravity(t *testing.T) { + defaults := antigravity.DefaultModels() + known := defaults[0] + custom := antigravity.ClaudeModel{ID: "claude-custom-alias", Type: "model", DisplayName: "claude-custom-alias"} + cases := []struct { + name string + credentials map[string]any + want []antigravity.ClaudeModel + }{ + {name: "nil credentials", want: defaults}, + {name: "missing mapping", credentials: map[string]any{}, want: defaults}, + {name: "empty JSON mapping", credentials: map[string]any{"model_mapping": map[string]any{}}, want: defaults}, + {name: "empty string mapping", credentials: map[string]any{"model_mapping": map[string]string{}}, want: defaults}, + {name: "blank key", credentials: map[string]any{"model_mapping": map[string]any{" ": "upstream-model"}}, want: defaults}, + {name: "JSON mapping keys and metadata", credentials: map[string]any{"model_mapping": map[string]any{ + known.ID: known.ID, custom.ID: "upstream-model", + }}, want: []antigravity.ClaudeModel{custom, known}}, + {name: "string mapping keys and metadata", credentials: map[string]any{"model_mapping": map[string]string{ + known.ID: known.ID, custom.ID: "upstream-model", + }}, want: []antigravity.ClaudeModel{custom, known}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, accountType := range []string{service.AccountTypeOAuth, service.AccountTypeAPIKey} { + t.Run(accountType, func(t *testing.T) { + svc := &availableModelsAdminService{ + stubAdminService: newStubAdminService(), + account: service.Account{ID: 51, Platform: service.PlatformAntigravity, Type: accountType, Credentials: tc.credentials}, + } + rec := httptest.NewRecorder() + setupAvailableModelsRouter(svc).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/admin/accounts/51/models", nil)) + require.Equal(t, http.StatusOK, rec.Code) + var resp struct { + Data []antigravity.ClaudeModel `json:"data"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + require.Equal(t, tc.want, resp.Data) + }) + } + }) + } +} + func TestAccountHandlerSyncUpstreamModels_ConfigErrorReturnsBadRequest(t *testing.T) { svc := &availableModelsAdminService{ stubAdminService: newStubAdminService(), diff --git a/backend/internal/handler/admin/channel_handler.go b/backend/internal/handler/admin/channel_handler.go index 065aedb2f497..4d2aa03e156c 100644 --- a/backend/internal/handler/admin/channel_handler.go +++ b/backend/internal/handler/admin/channel_handler.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + "github.com/Wei-Shaw/sub2api/internal/domain" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/pagination" "github.com/Wei-Shaw/sub2api/internal/pkg/response" @@ -631,20 +632,14 @@ func (h *ChannelHandler) GetModelDefaultPricing(c *gin.Context) { }) } -// platformToLiteLLMProvider maps a channel platform name to the corresponding -// LiteLLM provider string used as the key in the pricing catalog. -var platformToLiteLLMProvider = map[string]string{ - service.PlatformAnthropic: "anthropic", - service.PlatformOpenAI: "openai", - service.PlatformGemini: "gemini", - service.PlatformAntigravity: "anthropic", - service.PlatformGrok: "xai", - service.PlatformKimi: "moonshot", - service.PlatformZhipu: "zhipu", - service.PlatformDeepseek: "deepseek", - service.PlatformMiniMax: "minimax", - service.PlatformOpenCodeGo: "opencode-go", - service.PlatformTypeSafe: "typesafe", +// platformLiteLLMProvider maps a channel platform name to the corresponding +// LiteLLM provider string used as the key in the pricing catalog (platform list). +func platformLiteLLMProvider(platform string) (string, bool) { + spec, ok := domain.LookupPlatform(platform) + if !ok || spec.LiteLLMProvider == "" { + return "", false + } + return spec.LiteLLMProvider, true } // SyncPricingModels 返回 LiteLLM 定价目录中指定平台的最新模型列表 @@ -657,7 +652,7 @@ func (h *ChannelHandler) SyncPricingModels(c *gin.Context) { return } - provider, ok := platformToLiteLLMProvider[platform] + provider, ok := platformLiteLLMProvider(platform) if !ok { response.ErrorFrom(c, infraerrors.BadRequest("UNSUPPORTED_PLATFORM", fmt.Sprintf("unsupported platform: %s", platform)). diff --git a/backend/internal/handler/admin/group_handler.go b/backend/internal/handler/admin/group_handler.go index 774a00f88273..e35ed66571d9 100644 --- a/backend/internal/handler/admin/group_handler.go +++ b/backend/internal/handler/admin/group_handler.go @@ -184,7 +184,7 @@ func sanitizeUpdateGroupRequestForSimpleMode(req *UpdateGroupRequest) { type CreateGroupRequest struct { Name string `json:"name" binding:"required"` Description string `json:"description"` - Platform string `json:"platform" binding:"omitempty,oneof=anthropic openai gemini antigravity grok kimi zhipu deepseek minimax opencode_go typesafe composite"` + Platform string `json:"platform" binding:"omitempty,group_platform"` RateMultiplier float64 `json:"rate_multiplier"` IsExclusive bool `json:"is_exclusive"` SubscriptionType string `json:"subscription_type" binding:"omitempty,oneof=standard subscription"` @@ -258,7 +258,7 @@ type CreateGroupRequest struct { type UpdateGroupRequest struct { Name string `json:"name"` Description *string `json:"description"` - Platform string `json:"platform" binding:"omitempty,oneof=anthropic openai gemini antigravity grok kimi zhipu deepseek minimax opencode_go typesafe composite"` + Platform string `json:"platform" binding:"omitempty,group_platform"` RateMultiplier *float64 `json:"rate_multiplier"` IsExclusive *bool `json:"is_exclusive"` Status string `json:"status" binding:"omitempty,oneof=active inactive"` @@ -332,7 +332,7 @@ type UpdateGroupRequest struct { type CompositeRouteRequest struct { PublicModel string `json:"public_model" binding:"required"` MatchType string `json:"match_type" binding:"omitempty,oneof=exact prefix"` - TargetPlatform string `json:"target_platform" binding:"required,oneof=anthropic openai gemini antigravity grok kimi zhipu deepseek minimax opencode_go typesafe"` + TargetPlatform string `json:"target_platform" binding:"required,concrete_platform"` UpstreamModel string `json:"upstream_model"` Endpoint string `json:"endpoint" binding:"omitempty,oneof=any messages count_tokens responses chat_completions embeddings images gemini"` Priority int `json:"priority"` diff --git a/backend/internal/handler/admin/group_handler_platform_test.go b/backend/internal/handler/admin/group_handler_platform_test.go index d5d48845e1d3..ba456059864d 100644 --- a/backend/internal/handler/admin/group_handler_platform_test.go +++ b/backend/internal/handler/admin/group_handler_platform_test.go @@ -29,6 +29,7 @@ func TestGroupPlatformBinding_AllowedPlatforms(t *testing.T) { "anthropic", "openai", "gemini", "antigravity", "grok", "kimi", "zhipu", "deepseek", "minimax", "opencode_go", "composite", "typesafe", + "command_code", "cline", } for _, platform := range allowed { t.Run("create_"+platform, func(t *testing.T) { @@ -73,7 +74,7 @@ func TestGroupPlatformBinding_RejectsInvalidPlatforms(t *testing.T) { } func TestCompositeRouteTargetPlatform_AllowsConcreteProviders(t *testing.T) { - for _, platform := range []string{"kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe"} { + for _, platform := range []string{"kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe", "command_code", "cline"} { var req CompositeRouteRequest body := fmt.Sprintf(`{"public_model":"m","target_platform":%q}`, platform) require.NoError(t, bindGroupPlatformJSON(t, &req, body)) diff --git a/backend/internal/handler/admin/platform_validation.go b/backend/internal/handler/admin/platform_validation.go new file mode 100644 index 000000000000..a534e57838a6 --- /dev/null +++ b/backend/internal/handler/admin/platform_validation.go @@ -0,0 +1,30 @@ +package admin + +import ( + "fmt" + + "github.com/Wei-Shaw/sub2api/internal/domain" + "github.com/gin-gonic/gin/binding" + "github.com/go-playground/validator/v10" +) + +// 平台相关的请求绑定校验标签,由平台清单(domain/platforms.go)驱动,替代在 +// binding 标签里手写 oneof 白名单(新增平台时易漏加,导致分组无法创建): +// - group_platform:已登记的具体平台或 composite; +// - concrete_platform:已登记的具体平台。 +func init() { + engine, ok := binding.Validator.Engine().(*validator.Validate) + if !ok { + panic("platform validation requires a go-playground validator engine") + } + if err := engine.RegisterValidation("group_platform", func(fl validator.FieldLevel) bool { + return domain.IsGroupPlatform(fl.Field().String()) + }); err != nil { + panic(fmt.Errorf("register group_platform validation: %w", err)) + } + if err := engine.RegisterValidation("concrete_platform", func(fl validator.FieldLevel) bool { + return domain.IsConcretePlatform(fl.Field().String()) + }); err != nil { + panic(fmt.Errorf("register concrete_platform validation: %w", err)) + } +} diff --git a/backend/internal/handler/admin/reasoning_recovery_handler.go b/backend/internal/handler/admin/reasoning_recovery_handler.go deleted file mode 100644 index cbc3a926bb14..000000000000 --- a/backend/internal/handler/admin/reasoning_recovery_handler.go +++ /dev/null @@ -1,50 +0,0 @@ -package admin - -import ( - "github.com/Wei-Shaw/sub2api/internal/pkg/response" - "github.com/Wei-Shaw/sub2api/internal/server/middleware" - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/gin-gonic/gin" -) - -// ReasoningRecoveryHandler manages the global reasoning recovery switch. -type ReasoningRecoveryHandler struct { - service *service.ReasoningRecoveryService -} - -func NewReasoningRecoveryHandler(recovery *service.ReasoningRecoveryService) *ReasoningRecoveryHandler { - return &ReasoningRecoveryHandler{service: recovery} -} - -// Get returns the stored switch. -// GET /api/v1/admin/reasoning-recovery -func (h *ReasoningRecoveryHandler) Get(c *gin.Context) { - config, err := h.service.Get(c.Request.Context()) - if err != nil { - response.ErrorFrom(c, err) - return - } - c.Header("Cache-Control", "no-store") - response.Success(c, config) -} - -// Save stores the switch and returns what was saved. -// PUT /api/v1/admin/reasoning-recovery -func (h *ReasoningRecoveryHandler) Save(c *gin.Context) { - raw, err := c.GetRawData() - var request service.ReasoningRecoveryConfig - if err == nil { - request, err = service.DecodeReasoningRecoveryConfig(raw) - } - if err != nil { - response.BadRequest(c, "Invalid reasoning recovery settings: "+err.Error()) - return - } - config, err := h.service.Save(c.Request.Context(), request) - if err != nil { - response.ErrorFrom(c, err) - return - } - middleware.SetAuditExtra(c, map[string]any{"enabled": config.Enabled, "result": "reasoning_recovery_saved"}) - response.Success(c, config) -} diff --git a/backend/internal/handler/admin/reasoning_recovery_handler_test.go b/backend/internal/handler/admin/reasoning_recovery_handler_test.go deleted file mode 100644 index a5d517556699..000000000000 --- a/backend/internal/handler/admin/reasoning_recovery_handler_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package admin - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" -) - -type reasoningRecoverySettingRepo struct { - service.SettingRepository - values map[string]string -} - -func (r *reasoningRecoverySettingRepo) GetValue(_ context.Context, key string) (string, error) { - value, ok := r.values[key] - if !ok { - return "", service.ErrSettingNotFound - } - return value, nil -} - -func (r *reasoningRecoverySettingRepo) Set(_ context.Context, key, value string) error { - r.values[key] = value - return nil -} - -// The reasoning recovery endpoints read and save one switch in the standard -// envelope. Without a stored value the switch is on; a body that is not exactly -// an enabled boolean object is refused and stores nothing. -func TestReasoningRecoveryEndpointsReadAndSaveTheSwitch(t *testing.T) { - gin.SetMode(gin.TestMode) - repo := &reasoningRecoverySettingRepo{values: map[string]string{}} - recovery := service.NewReasoningRecoveryService(repo) - h := NewReasoningRecoveryHandler(recovery) - call := func(method, body string) *httptest.ResponseRecorder { - recorder := httptest.NewRecorder() - c, _ := gin.CreateTestContext(recorder) - c.Request = httptest.NewRequest(method, "/api/v1/admin/reasoning-recovery", strings.NewReader(body)) - if method == http.MethodGet { - h.Get(c) - } else { - h.Save(c) - } - return recorder - } - - read := call(http.MethodGet, "") - require.Equal(t, http.StatusOK, read.Code) - require.JSONEq(t, `{"code":0,"message":"success","data":{"enabled":true}}`, read.Body.String()) - require.Empty(t, repo.values) - - saved := call(http.MethodPut, `{"enabled":false}`) - require.Equal(t, http.StatusOK, saved.Code) - require.JSONEq(t, `{"code":0,"message":"success","data":{"enabled":false}}`, saved.Body.String()) - require.Equal(t, map[string]string{"reasoning_recovery_config": `{"enabled":false}`}, repo.values) - require.False(t, recovery.Enabled(), "a save applies to this instance at once") - require.JSONEq(t, `{"code":0,"message":"success","data":{"enabled":false}}`, call(http.MethodGet, "").Body.String()) - - for _, body := range []string{`{"enabled":"true"}`, `{"enabled":null}`, `{"enabled":true,"other":true}`, `{"other":true}`, `{}`, `null`, ``} { - rejected := call(http.MethodPut, body) - require.Equal(t, http.StatusBadRequest, rejected.Code, body) - require.Equal(t, map[string]string{"reasoning_recovery_config": `{"enabled":false}`}, repo.values) - require.False(t, recovery.Enabled()) - } - - require.Equal(t, http.StatusOK, call(http.MethodPut, `{"enabled":true}`).Code) - require.Equal(t, map[string]string{"reasoning_recovery_config": `{"enabled":true}`}, repo.values) - require.True(t, recovery.Enabled()) - - // A read answers with the stored value, here one another instance saved, - // and applies it to this instance. - repo.values["reasoning_recovery_config"] = `{"enabled":false}` - require.JSONEq(t, `{"code":0,"message":"success","data":{"enabled":false}}`, call(http.MethodGet, "").Body.String()) - require.False(t, recovery.Enabled()) -} diff --git a/backend/internal/handler/gateway_handler.go b/backend/internal/handler/gateway_handler.go index 258c115d1a26..2f3ba2f6170b 100644 --- a/backend/internal/handler/gateway_handler.go +++ b/backend/internal/handler/gateway_handler.go @@ -1173,7 +1173,7 @@ func (h *GatewayHandler) Models(c *gin.Context) { } if platform == service.PlatformComposite { - availableModels := h.compositeAvailableModels(c.Request.Context(), groupID, true) + availableModels := h.compositeAvailableModels(c.Request.Context(), groupID, "", true) if apiKey != nil && apiKey.Group != nil && apiKey.Group.ModelAllowlistEnabled() { source := availableModels if len(source) == 0 { @@ -1274,19 +1274,19 @@ func (h *GatewayHandler) codexModelIDsForGroup(ctx context.Context, group *servi platform = group.Platform } if platform == service.PlatformComposite { - availableModels := h.compositeAvailableModels(ctx, groupID, false) + availableModels := h.compositeAvailableModels(ctx, groupID, service.CompositeRouteEndpointResponses, false) fallbackModels := defaultCodexModelIDsForPlatform(service.PlatformComposite) + models := availableModels + if len(models) == 0 { + models = fallbackModels + } if group.ModelAllowlistEnabled() { - source := availableModels - if len(source) == 0 { - source = fallbackModels - } - return group.ModelAllowlist.FilterForListing(source) + models = group.ModelAllowlist.FilterForListing(models) } - if len(availableModels) > 0 { - return availableModels + if filtered, err := h.gatewayService.FilterCompositeCodexModels(ctx, group.ID, models); err == nil { + return filtered } - return fallbackModels + return models } availableModels := h.gatewayService.GetAvailableModels(ctx, groupID, platform) @@ -1303,14 +1303,14 @@ func (h *GatewayHandler) codexModelIDsForGroup(ctx context.Context, group *servi // compositeAvailableModels lists the models the composite group can serve. // includeSystemOne adds TypeSafe models, which only work through /v1/systemone; // LLM client catalogs (Codex) must exclude them. -func (h *GatewayHandler) compositeAvailableModels(ctx context.Context, groupID *int64, includeSystemOne bool) []string { +func (h *GatewayHandler) compositeAvailableModels(ctx context.Context, groupID *int64, endpoint string, includeSystemOne bool) []string { if h == nil || h.gatewayService == nil { return nil } seen := make(map[string]struct{}) models := make([]string, 0) schedulablePlatforms := h.gatewayService.GetSchedulablePlatforms(ctx, groupID) - for _, platform := range []string{service.PlatformAnthropic, service.PlatformGemini, service.PlatformOpenAI, service.PlatformAntigravity, service.PlatformGrok, service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, service.PlatformMiniMax, service.PlatformOpenCodeGo, service.PlatformTypeSafe} { + for _, platform := range domain.CompositePrecedencePlatformIDs() { if platform == service.PlatformTypeSafe && !includeSystemOne { continue } @@ -1334,6 +1334,16 @@ func (h *GatewayHandler) compositeAvailableModels(ctx context.Context, groupID * models = append(models, model) } } + // A route can expose a public ID that no account model mapping contains. + // On lookup failure, retain the existing account-derived catalog only. + if routeModels, err := h.gatewayService.GetCompositeRouteModels(ctx, groupID, endpoint, includeSystemOne); err == nil { + for _, model := range routeModels { + if _, ok := seen[model]; !ok { + seen[model] = struct{}{} + models = append(models, model) + } + } + } return models } @@ -1507,10 +1517,13 @@ func defaultModelIDsForPlatform(platform string) []string { case service.PlatformComposite: ids := make([]string, 0) seen := make(map[string]struct{}) - // TypeSafe is deliberately absent: jev-latest only works through - // /v1/systemone, so the static fallback never advertises it to LLM - // clients. compositeAvailableModels lists it when the group can serve it. - for _, concretePlatform := range []string{service.PlatformAnthropic, service.PlatformGemini, service.PlatformOpenAI, service.PlatformAntigravity, service.PlatformGrok, service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, service.PlatformMiniMax, service.PlatformOpenCodeGo} { + for _, concretePlatform := range domain.CompositePrecedencePlatformIDs() { + // TypeSafe is deliberately skipped: jev-latest only works through + // /v1/systemone, so the static fallback never advertises it to LLM + // clients. compositeAvailableModels lists it when the group can serve it. + if concretePlatform == service.PlatformTypeSafe { + continue + } for _, id := range defaultModelIDsForPlatform(concretePlatform) { if _, ok := seen[id]; ok { continue diff --git a/backend/internal/handler/gateway_models_test.go b/backend/internal/handler/gateway_models_test.go index c847e8ef2804..a5bc989311e9 100644 --- a/backend/internal/handler/gateway_models_test.go +++ b/backend/internal/handler/gateway_models_test.go @@ -3,6 +3,7 @@ package handler import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "strings" @@ -21,6 +22,16 @@ type gatewayModelsAccountRepoStub struct { byGroup map[int64][]service.Account } +type gatewayModelsRouteRepoStub struct { + service.CompositeModelRouteRepository + routes []service.CompositeModelRoute + err error +} + +func (s *gatewayModelsRouteRepoStub) ListByGroup(_ context.Context, _ int64, _ bool) ([]service.CompositeModelRoute, error) { + return s.routes, s.err +} + type gatewayModelsResponseForTest struct { Object string `json:"object"` Data []gatewayModelItemForTest `json:"data"` @@ -80,12 +91,16 @@ func (s *gatewayModelsAccountRepoStub) ListModelAvailabilityCandidates(ctx conte return s.ListSchedulableByGroupID(ctx, *groupID) } -func newGatewayModelsHandlerForTest(repo service.AccountRepository) *GatewayHandler { +func newGatewayModelsHandlerForTest(repo service.AccountRepository, routeRepo ...service.CompositeModelRouteRepository) *GatewayHandler { + var resolver *service.CompositeRouteResolver + if len(routeRepo) > 0 { + resolver = service.NewCompositeRouteResolver(routeRepo[0]) + } return &GatewayHandler{ gatewayService: service.NewGatewayService( repo, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, resolver, nil, nil, ), } } @@ -103,6 +118,66 @@ func TestWriteOpenAIModelsListOmitsContextMetadataForOrdinaryProviders(t *testin require.NotContains(t, rec.Body.String(), "max_output_tokens") } +func TestGatewayCompositeRouteModelsAppearInBothCatalogs(t *testing.T) { + gin.SetMode(gin.TestMode) + const groupID int64 = 7784 + accounts := &gatewayModelsAccountRepoStub{byGroup: map[int64][]service.Account{ + groupID: {{ID: 1, Platform: service.PlatformOpenAI, Credentials: map[string]any{ + "model_mapping": map[string]any{"mapped-model": "upstream"}, + }}}, + }} + routes := &gatewayModelsRouteRepoStub{routes: []service.CompositeModelRoute{ + {PublicModel: "route-only", MatchType: service.CompositeRouteMatchExact, Enabled: true}, + {PublicModel: "route-only", MatchType: service.CompositeRouteMatchExact, Enabled: true}, + {PublicModel: "mapped-model", MatchType: service.CompositeRouteMatchExact, Enabled: true}, + {PublicModel: "image-only", MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointImages, Enabled: true}, + {PublicModel: "prefix-", MatchType: service.CompositeRouteMatchPrefix, Enabled: true}, + {PublicModel: "disabled", MatchType: service.CompositeRouteMatchExact, Enabled: false}, + }} + for _, tc := range []struct { + name string + repo service.CompositeModelRouteRepository + allowlist []string + want []string + }{ + {"routes", routes, nil, []string{"mapped-model", "route-only", "image-only"}}, + {"allowlist", routes, []string{"route-only", "disabled", "unknown"}, []string{"route-only"}}, + {"lookup failure", &gatewayModelsRouteRepoStub{err: errors.New("unavailable")}, nil, []string{"mapped-model"}}, + {"allowlisted lookup failure", &gatewayModelsRouteRepoStub{err: errors.New("unavailable")}, []string{"mapped-model", "unknown"}, []string{"mapped-model"}}, + {"no route repo", nil, nil, []string{"mapped-model"}}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newGatewayModelsHandlerForTest(accounts, tc.repo) + group := &service.Group{ID: groupID, Platform: service.PlatformComposite} + if tc.allowlist != nil { + group.ModelAllowlist = service.GroupModelAllowlist{Enabled: true, Models: tc.allowlist} + } + for _, endpoint := range []string{"models", "codex"} { + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/models", nil) + c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{Group: group}) + if endpoint == "models" { + h.Models(c) + var got gatewayModelsResponseForTest + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + require.Equal(t, tc.want, modelIDsForTest(got.Data)) + } else { + h.CodexModels(c) + var got codexModelsResponseForTest + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + want := tc.want + if tc.name == "routes" { + want = []string{"mapped-model", "route-only"} + } + require.ElementsMatch(t, want, codexModelSlugsForTest(got.Models)) + } + require.Equal(t, http.StatusOK, rec.Code) + } + }) + } +} + func TestDefaultModelIDsForCompositeIncludesAntigravityDefaults(t *testing.T) { antigravityIDs := defaultModelIDsForPlatform(service.PlatformAntigravity) require.NotEmpty(t, antigravityIDs) @@ -1580,3 +1655,188 @@ func TestDefaultModelIDsForPlatform_CompositeFallbackExcludesTypeSafe(t *testing require.NotContains(t, defaultModelIDsForPlatform(service.PlatformComposite), "jev-latest") require.NotContains(t, defaultCodexModelIDsForPlatform(service.PlatformComposite), "jev-latest") } + +func TestGatewayModels_CompositeExactRoutesRespectTypeSafeIsolation(t *testing.T) { + gin.SetMode(gin.TestMode) + const groupID int64 = 7784 + routes := &gatewayModelsRouteRepoStub{routes: []service.CompositeModelRoute{ + {PublicModel: "systemone-any-alias", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointAny, Enabled: true}, + {PublicModel: "systemone-responses-alias", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + {PublicModel: "systemone-legacy-alias", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Enabled: true}, + {PublicModel: "jev-latest", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Enabled: true}, + {PublicModel: "responses-alias", TargetPlatform: service.PlatformOpenAI, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + {PublicModel: "image-alias", TargetPlatform: service.PlatformOpenAI, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointImages, Enabled: true}, + }} + h := newGatewayModelsHandlerForTest(&gatewayModelsAccountRepoStub{}, routes) + wantModels := make([]string, 0, len(routes.routes)) + for _, route := range routes.routes { + wantModels = append(wantModels, route.PublicModel) + } + for _, allowlisted := range []bool{false, true} { + name := "unrestricted" + if allowlisted { + name = "allowlisted" + } + t.Run(name, func(t *testing.T) { + group := &service.Group{ID: groupID, Platform: service.PlatformComposite} + if allowlisted { + group.ModelAllowlist = service.GroupModelAllowlist{Enabled: true, Models: []string{ + "systemone-any-alias", "systemone-responses-alias", "systemone-legacy-alias", "jev-latest", "responses-alias", "image-alias", + }} + } + for _, codex := range []bool{false, true} { + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/models", nil) + ctx.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{Group: group}) + if codex { + h.CodexModels(ctx) + var manifest codexModelsResponseForTest + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &manifest)) + require.Equal(t, []string{"responses-alias"}, codexModelSlugsForTest(manifest.Models)) + } else { + h.Models(ctx) + var models gatewayModelsResponseForTest + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &models)) + require.ElementsMatch(t, wantModels, modelIDsForTest(models.Data)) + } + require.Equal(t, http.StatusOK, rec.Code) + } + }) + } +} + +func TestGatewayModels_CompositeCodexFiltersEffectiveResponsesRoutes(t *testing.T) { + gin.SetMode(gin.TestMode) + const groupID int64 = 7793 + defaultModel := defaultModelIDsForPlatform(service.PlatformOpenAI)[0] + for _, scenario := range []struct { + name string + model string + accounts []service.Account + routes []service.CompositeModelRoute + }{ + { + name: "colliding exact routes", + model: "shared-alias", + routes: []service.CompositeModelRoute{ + {PublicModel: "shared-alias", TargetPlatform: service.PlatformOpenAI, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointAny, Priority: 1, Enabled: true}, + {PublicModel: "shared-alias", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Priority: 100, Enabled: true}, + }, + }, + { + name: "account alias exact override", + model: "account-alias", + accounts: []service.Account{{ID: 1, Platform: service.PlatformOpenAI, Credentials: map[string]any{ + "model_mapping": map[string]any{"account-alias": "gpt-5"}, + }}}, + routes: []service.CompositeModelRoute{ + {PublicModel: "account-alias", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + }, + }, + { + name: "account alias prefix override", + model: "account-alias", + accounts: []service.Account{{ID: 1, Platform: service.PlatformOpenAI, Credentials: map[string]any{ + "model_mapping": map[string]any{"account-alias": "gpt-5"}, + }}}, + routes: []service.CompositeModelRoute{ + {PublicModel: "account-", TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchPrefix, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + }, + }, + { + name: "account default override", + model: defaultModel, + accounts: []service.Account{{ID: 1, Platform: service.PlatformOpenAI}}, + routes: []service.CompositeModelRoute{ + {PublicModel: defaultModel, TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + }, + }, + { + name: "fallback default override", + model: defaultModel, + routes: []service.CompositeModelRoute{ + {PublicModel: defaultModel, TargetPlatform: service.PlatformTypeSafe, MatchType: service.CompositeRouteMatchPrefix, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true}, + }, + }, + } { + t.Run(scenario.name, func(t *testing.T) { + for _, allowlisted := range []bool{false, true} { + name := "unrestricted" + if allowlisted { + name = "allowlisted" + } + t.Run(name, func(t *testing.T) { + routes := append([]service.CompositeModelRoute(nil), scenario.routes...) + if scenario.name != "fallback default override" { + routes = append(routes, service.CompositeModelRoute{ + PublicModel: "responses-control", TargetPlatform: service.PlatformOpenAI, MatchType: service.CompositeRouteMatchExact, Endpoint: service.CompositeRouteEndpointResponses, Enabled: true, + }) + } + h := newGatewayModelsHandlerForTest(&gatewayModelsAccountRepoStub{ + byGroup: map[int64][]service.Account{groupID: scenario.accounts}, + }, &gatewayModelsRouteRepoStub{routes: routes}) + group := &service.Group{ID: groupID, Platform: service.PlatformComposite} + if allowlisted { + group.ModelAllowlist = service.GroupModelAllowlist{Enabled: true, Models: []string{scenario.model, "responses-control"}} + } + for _, codex := range []bool{false, true} { + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/models", nil) + ctx.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{Group: group}) + if codex { + h.CodexModels(ctx) + var manifest codexModelsResponseForTest + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &manifest)) + slugs := codexModelSlugsForTest(manifest.Models) + require.NotContains(t, slugs, scenario.model) + if scenario.name != "fallback default override" { + require.Contains(t, slugs, "responses-control") + } else if allowlisted { + require.Empty(t, slugs) + } + } else { + h.Models(ctx) + var models gatewayModelsResponseForTest + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &models)) + require.Contains(t, modelIDsForTest(models.Data), scenario.model) + } + require.Equal(t, http.StatusOK, recorder.Code) + } + }) + } + }) + } +} + +func TestGatewayModels_CompositeCodexRouteLookupErrorKeepsFallback(t *testing.T) { + gin.SetMode(gin.TestMode) + const groupID int64 = 7793 + model := defaultCodexModelIDsForPlatform(service.PlatformComposite)[0] + for _, allowlisted := range []bool{false, true} { + name := "unrestricted" + if allowlisted { + name = "allowlisted" + } + t.Run(name, func(t *testing.T) { + h := newGatewayModelsHandlerForTest(&gatewayModelsAccountRepoStub{}, &gatewayModelsRouteRepoStub{err: errors.New("unavailable")}) + group := &service.Group{ID: groupID, Platform: service.PlatformComposite} + if allowlisted { + group.ModelAllowlist = service.GroupModelAllowlist{Enabled: true, Models: []string{model}} + } + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/models", nil) + ctx.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{Group: group}) + h.CodexModels(ctx) + require.Equal(t, http.StatusOK, recorder.Code) + var manifest codexModelsResponseForTest + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &manifest)) + require.Contains(t, codexModelSlugsForTest(manifest.Models), model) + if allowlisted { + require.Len(t, manifest.Models, 1) + } + }) + } +} diff --git a/backend/internal/handler/handler.go b/backend/internal/handler/handler.go index 6d01386b2010..71d59a1c44c4 100644 --- a/backend/internal/handler/handler.go +++ b/backend/internal/handler/handler.go @@ -45,7 +45,6 @@ type AdminHandlers struct { Compliance *admin.ComplianceHandler AuditLog *admin.AuditLogHandler SystemPrompt *admin.SystemPromptHandler - ReasoningRecovery *admin.ReasoningRecoveryHandler CodexGatewayBorrow *admin.CodexGatewayBorrowHandler PelicanTest *admin.PelicanTestHandler } diff --git a/backend/internal/handler/openai_codex_models_handler_test.go b/backend/internal/handler/openai_codex_models_handler_test.go index cfd20eca47cf..cce22c23f571 100644 --- a/backend/internal/handler/openai_codex_models_handler_test.go +++ b/backend/internal/handler/openai_codex_models_handler_test.go @@ -1065,3 +1065,32 @@ func TestCodexModelsPinnedAccountsETagMatchReturns304(t *testing.T) { require.Equal(t, http.StatusNotModified, second.Code, second.Body.String()) require.Empty(t, second.Body.Bytes()) } + +func TestCodexModelsDiscoveryProjectsWildcardMappingWithoutAllowlist(t *testing.T) { + for _, mode := range []string{"ordinary", "pinned", "fallback"} { + t.Run(mode, func(t *testing.T) { + account := newPinnedCodexAccount(1, service.StatusActive, true, false) + account.Credentials["model_mapping"] = map[string]any{"gpt-*": "gpt-5.4"} + upstream := &codexModelsPinnedHTTPUpstream{bodies: map[int64]string{ + 1: `{"models":[{"slug":"gpt-5.4"},{"slug":"blocked"},{"slug":"codex-auto-review"},{"slug":"gpt-image-2"}]}`, + }} + handler := newPinnedCodexTestHandler([]service.Account{account}, upstream, 3) + group := &service.Group{ID: 84, Platform: service.PlatformOpenAI, + ModelAllowlist: service.GroupModelAllowlist{Models: []string{"blocked"}}, + } + if mode != "ordinary" { + group.CodexModelsManifestConfig = service.GroupCodexModelsManifestConfig{Enabled: true, AccountIDs: []int64{1}} + } + if mode == "fallback" { + group.CodexModelsManifestConfig.AccountIDs = []int64{999} + group.CodexModelsManifestConfig.FallbackToScheduler = true + } + first := performCodexModelsRequestForGroup(t, handler, group, "") + require.Equal(t, http.StatusOK, first.Code, first.Body.String()) + require.Equal(t, []string{"gpt-5.4"}, codexHandlerManifestSlugs(t, first)) + require.NotEmpty(t, first.Header().Get("ETag")) + second := performCodexModelsRequestForGroup(t, handler, group, first.Header().Get("ETag")) + require.Equal(t, http.StatusNotModified, second.Code) + }) + } +} diff --git a/backend/internal/handler/openai_continuation_account_mismatch_test.go b/backend/internal/handler/openai_continuation_account_mismatch_test.go deleted file mode 100644 index 14fa9f60f0e9..000000000000 --- a/backend/internal/handler/openai_continuation_account_mismatch_test.go +++ /dev/null @@ -1,414 +0,0 @@ -//go:build unit - -package handler - -import ( - "io" - "net/http" - "strings" - "sync" - "testing" - - "github.com/Wei-Shaw/sub2api/internal/config" - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" -) - -// continuationMismatchUpstream answers by account: an accepting account -// completes the request, every other account answers with its scripted reply -// or, by default, rejects the ciphertext in the request. -type continuationMismatchUpstream struct { - service.HTTPUpstream - mu sync.Mutex - accountIDs []int64 - bodies [][]byte - accepting map[int64]bool - replies map[int64]continuationMismatchReply - // sequence scripts an account's answers call by call; when it runs out the - // account falls back to replies and rejection. - sequence map[int64][]continuationMismatchReply - rejection continuationMismatchReply -} - -type continuationMismatchReply struct { - status int - payload string - // stream, when set, is sent as an event stream in place of payload. - stream string -} - -func (u *continuationMismatchUpstream) Do(req *http.Request, _ string, accountID int64, _ int) (*http.Response, error) { - body, err := io.ReadAll(req.Body) - if err != nil { - return nil, err - } - u.mu.Lock() - u.accountIDs = append(u.accountIDs, accountID) - u.bodies = append(u.bodies, body) - var next *continuationMismatchReply - if pending := u.sequence[accountID]; len(pending) > 0 { - next, u.sequence[accountID] = &pending[0], pending[1:] - } - u.mu.Unlock() - if next == nil && u.accepting[accountID] { - return openAIOfficialHTTPSuccess(openAIOfficialHTTPAttempt{accountID: accountID, path: req.URL.Path, body: body}), nil - } - reply, scripted := u.replies[accountID] - if next != nil { - reply = *next - } else if !scripted { - reply = u.rejection - } - if reply.stream != "" { - return openAIOfficialHTTPResponse(reply.status, "text/event-stream", reply.stream), nil - } - return openAIOfficialHTTPResponse(reply.status, "application/json", reply.payload), nil -} - -func (u *continuationMismatchUpstream) distinctAccounts() []int64 { - u.mu.Lock() - defer u.mu.Unlock() - var out []int64 - seen := map[int64]bool{} - for _, id := range u.accountIDs { - if !seen[id] { - seen[id] = true - out = append(out, id) - } - } - return out -} - -var ( - // No item is named: with another ciphertext carrier nothing is removed. - continuationMismatchUnrecoverable = continuationMismatchReply{status: http.StatusBadRequest, - payload: `{"error":{"message":"invalid encrypted content","type":"invalid_request_error","param":null,"code":"invalid_encrypted_content"}}`} - // The rejection names the reasoning item, so its ciphertext is removed and - // the request is sent once more. - continuationMismatchNamedReasoning = continuationMismatchReply{status: http.StatusBadRequest, - payload: `{"error":{"message":"The encrypted content for item rs_a could not be verified.","type":"invalid_request_error","param":null,"code":"invalid_encrypted_content"}}`} - // The wording for an inter-agent message body, which carries no code. - continuationMismatchUndecryptable = continuationMismatchReply{status: http.StatusBadRequest, - payload: `{"error":{"type":"invalid_request_error","message":"Encrypted function output content could not be decrypted or decoded."}}`} -) - -func continuationMismatchAccounts(count int, accountType string) []service.Account { - accounts := make([]service.Account, 0, count) - for id := int64(1); id <= int64(count); id++ { - account := service.Account{ - ID: id, Name: "organisation", Platform: service.PlatformOpenAI, Type: accountType, - Status: service.StatusActive, Schedulable: true, Priority: int(id), GroupIDs: []int64{openAIOfficialHTTPGroupID}, - Credentials: map[string]any{"api_key": "fixture", "base_url": "https://api.example.test"}, - Extra: map[string]any{"use_responses_api": true}, - } - if accountType == service.AccountTypeOAuth { - account.Credentials = map[string]any{"access_token": "fixture"} - } - accounts = append(accounts, account) - } - return accounts -} - -// newContinuationMismatchHandler runs the real handler, selector and forwarder -// over in-memory accounts and a sticky-session cache the test can read. -func newContinuationMismatchHandler(t *testing.T, upstream service.HTTPUpstream, accounts []service.Account) (*OpenAIGatewayHandler, *openAIOfficialHTTPStickyCache) { - t.Helper() - repo := &openAIOfficialHTTPAccountRepo{grokCredentialHandlerRepo: grokCredentialHandlerRepo{accounts: accounts}} - cache := &openAIOfficialHTTPStickyCache{grokCredentialHandlerGatewayCache: grokCredentialHandlerGatewayCache{ - sessions: make(map[grokCredentialHandlerGatewayCacheKey]int64), - }} - cfg := &config.Config{RunMode: config.RunModeSimple} - cfg.Gateway.MaxAccountSwitches = 10 - billing := service.NewBillingCacheService(nil, nil, nil, nil, nil, nil, cfg, nil) - t.Cleanup(billing.Stop) - gateway := service.NewOpenAIGatewayService( - repo, nil, nil, nil, nil, nil, cache, cfg, nil, nil, - service.NewBillingService(cfg, nil), service.NewRateLimitService(repo, nil, cfg, nil, nil), billing, upstream, - &service.DeferredService{}, nil, nil, nil, nil, nil, nil, nil, - ) - h := NewOpenAIGatewayHandler(gateway, service.NewConcurrencyService(nil), billing, - service.NewAPIKeyService(nil, nil, nil, nil, nil, nil, cfg), nil, nil, nil, nil, cfg) - return h, cache -} - -func serveContinuationMismatch(t *testing.T, h *OpenAIGatewayHandler, body string) *responseRecorderResult { - t.Helper() - return serveContinuationMismatchSession(t, h, body, "one-conversation") -} - -func serveContinuationMismatchSession(t *testing.T, h *OpenAIGatewayHandler, body, session string) *responseRecorderResult { - t.Helper() - c, recorder := newReplayableEndpointContext(t, openAIOfficialHTTPGroupID, http.MethodPost, "/v1/responses", []byte(body)) - c.Request.Header.Set("session_id", session) - h.Responses(c) - return &responseRecorderResult{code: recorder.Code, body: recorder.Body.String()} -} - -type responseRecorderResult struct { - code int - body string -} - -func continuationMismatchBoundAccount(t *testing.T, cache *openAIOfficialHTTPStickyCache) int64 { - t.Helper() - // The cache also holds response-owner entries; the session binding is the - // entry keyed by the session hash alone. - var bound []int64 - for key, accountID := range cache.sessions { - if strings.Count(key.sessionHash, ":") == 1 { - bound = append(bound, accountID) - } - } - require.Len(t, bound, 1, "one conversation has one binding") - return bound[0] -} - -// A Codex multi-agent history: reasoning ciphertext plus an inter-agent message -// whose body is ciphertext the gateway cannot remove. -const continuationMismatchBody = `{"model":"gpt-5.1","stream":false,"input":[{"type":"reasoning","id":"rs_a","encrypted_content":"cipher-reasoning","summary":[]},{"type":"agent_message","id":"amsg_b","author":"/root","recipient":"/root/worker","content":[{"type":"input_text","text":"Payload:"},{"type":"encrypted_content","encrypted_content":"cipher-agent"}]}]}` - -// A history whose only ciphertext is reasoning: any rejection of it can be -// repaired by the stripped retry, also where reasoning ids are not sent. -const continuationMismatchReasoningOnlyBody = `{"model":"gpt-5.1","stream":false,"input":[{"type":"reasoning","id":"rs_a","encrypted_content":"cipher-reasoning","summary":[]},{"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]}]}` - -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchMovesToAnAccountThatAccepts(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, test := range []struct { - name string - rejection continuationMismatchReply - callsOnFirst int - }{ - // The stripped retry is rejected again, as when the message body is foreign too. - {"stripped_retry_rejected", continuationMismatchNamedReasoning, 2}, - {"recovery_not_possible", continuationMismatchUnrecoverable, 1}, - } { - t.Run(test.name, func(t *testing.T) { - upstream := &continuationMismatchUpstream{accepting: map[int64]bool{2: true}, rejection: test.rejection} - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, continuationMismatchBody) - - require.Equal(t, http.StatusOK, response.code, response.body) - require.Contains(t, response.body, "healthy answer") - require.NotContains(t, response.body, service.OpenAIContinuationStateUnavailableCode) - require.Len(t, upstream.accountIDs, test.callsOnFirst+1) - require.Equal(t, []int64{1, 2}, upstream.distinctAccounts()) - accepted := upstream.bodies[len(upstream.bodies)-1] - for _, path := range []string{"input.0.encrypted_content", "input.1"} { - require.Equal(t, gjson.GetBytes(upstream.bodies[0], path).Raw, gjson.GetBytes(accepted, path).Raw, "the next account receives the request unchanged: "+path) - } - require.Equal(t, "cipher-reasoning", gjson.GetBytes(accepted, "input.0.encrypted_content").String()) - require.Equal(t, int64(2), continuationMismatchBoundAccount(t, cache), "the conversation continues on the account that accepted it") - }) - } -} - -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchEndsWhenNoAccountAccepts(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{rejection: continuationMismatchUnrecoverable} - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(maxOpenAICiphertextAccountSwitches+3, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, continuationMismatchBody) - - require.Len(t, upstream.distinctAccounts(), maxOpenAICiphertextAccountSwitches+1, "the first account and a bounded number of others") - require.Len(t, upstream.accountIDs, maxOpenAICiphertextAccountSwitches+1, "an account that cannot repair the request is asked once") - require.Equal(t, http.StatusBadRequest, response.code) - require.Equal(t, service.OpenAIContinuationStateUnavailableCode, gjson.Get(response.body, "error.code").String()) - require.NotContains(t, response.body, "invalid encrypted content", "the client receives the fixed sentence, not upstream text") - require.Equal(t, upstream.accountIDs[0], continuationMismatchBoundAccount(t, cache), "the accounts that rejected the conversation are no home for it") -} - -// An account that reads the ciphertext and rejects the request for another -// reason has answered: that answer ends the request. -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchKeepsALaterAccountsOwnAnswer(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{ - rejection: continuationMismatchUnrecoverable, - replies: map[int64]continuationMismatchReply{2: {status: http.StatusBadRequest, - payload: `{"error":{"type":"invalid_request_error","code":"missing_required_parameter","param":"input[3].call_id","message":"do-not-leak-validation"}}`}}, - } - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(3, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, continuationMismatchBody) - - require.Equal(t, []int64{1, 2}, upstream.accountIDs, "a validation rejection is the same on every account") - require.Equal(t, http.StatusBadRequest, response.code) - require.Equal(t, service.OpenAIRequestRejectedCode, gjson.Get(response.body, "error.code").String()) - require.NotContains(t, response.body, "do-not-leak") - require.Equal(t, int64(1), continuationMismatchBoundAccount(t, cache)) -} - -// A ciphertext rejection wrapped in a 5xx is still no fault of the account: -// the account is not asked again before the next one is tried. -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchSkipsTheSameAccountRetry(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{ - accepting: map[int64]bool{2: true}, - rejection: continuationMismatchReply{status: http.StatusServiceUnavailable, payload: `{"error":{"message":"Encrypted content could not be verified"}}`}, - } - handler, _ := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeOAuth)) - response := serveContinuationMismatch(t, handler, strings.Replace(continuationMismatchBody, `"stream":false`, `"stream":true`, 1)) - - require.Equal(t, []int64{1, 2}, upstream.accountIDs) - require.Contains(t, response.body, "response.completed") -} - -// When the accounts run out, the client receives what the first account -// answered, in that failure's own classification, and the session is where it -// was before the others were tried. -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchRendersTheFirstFailureWhenAccountsRunOut(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, test := range []struct { - name string - first, second continuationMismatchReply - code string - }{ - {"continuation_state_first", continuationMismatchUnrecoverable, continuationMismatchUndecryptable, service.OpenAIContinuationStateUnavailableCode}, - {"request_rejected_first", continuationMismatchUndecryptable, continuationMismatchUnrecoverable, service.OpenAIRequestRejectedCode}, - } { - t.Run(test.name, func(t *testing.T) { - upstream := &continuationMismatchUpstream{replies: map[int64]continuationMismatchReply{1: test.first, 2: test.second}} - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, continuationMismatchBody) - - require.Equal(t, []int64{1, 2}, upstream.accountIDs) - require.Equal(t, http.StatusBadRequest, response.code) - require.Equal(t, test.code, gjson.Get(response.body, "error.code").String()) - require.NotContains(t, response.body, "could not be decrypted") - require.Equal(t, int64(1), continuationMismatchBoundAccount(t, cache)) - }) - } -} - -// An account that read the ciphertext and was answering when its stream broke -// is where the conversation runs: the session stays with it. -func TestOpenAIGatewayHandler_ContinuationCiphertextMismatchKeepsTheSessionOnAnAccountThatWasAnswering(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{ - rejection: continuationMismatchUnrecoverable, - replies: map[int64]continuationMismatchReply{2: {status: http.StatusOK, stream: "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_partial\",\"status\":\"in_progress\"}}\n\n" + - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"partial-answer\"}\n\n"}}, - } - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(3, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, strings.Replace(continuationMismatchBody, `"stream":false`, `"stream":true`, 1)) - - require.Equal(t, []int64{1, 2}, upstream.accountIDs) - require.Contains(t, response.body, "partial-answer") - require.Equal(t, int64(2), continuationMismatchBoundAccount(t, cache)) -} - -// The stripped retry can fail for a reason of the account's own. The answer is -// then the same as on a first send: the request goes to the next account, which -// receives it unchanged, and the first account is not asked a third time. -func TestOpenAIGatewayHandler_ContinuationStrippedRetryAccountFailureMovesToTheNextAccount(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, test := range []struct { - name string - stream bool - accountType string - body string - first continuationMismatchReply - retry continuationMismatchReply - }{ - // Production, 2026-10-05: the retry is accepted at HTTP level and fails in the stream. - {"credit_exhausted_in_stream", true, service.AccountTypeAPIKey, continuationMismatchBody, continuationMismatchNamedReasoning, - continuationMismatchReply{status: http.StatusOK, stream: "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_failed\",\"status\":\"failed\",\"error\":{\"code\":\"credit_balance_exhausted\",\"message\":\"You have no credits remaining. do-not-leak-billing\"}}}\n\n"}}, - {"payment_required", false, service.AccountTypeAPIKey, continuationMismatchBody, continuationMismatchNamedReasoning, - continuationMismatchReply{status: http.StatusPaymentRequired, payload: `{"error":{"type":"insufficient_quota","code":"insufficient_quota","message":"do-not-leak-billing"}}`}}, - // An OAuth account would otherwise get a same-account retry for a 5xx. - {"server_error_on_oauth", true, service.AccountTypeOAuth, continuationMismatchReasoningOnlyBody, continuationMismatchUnrecoverable, - continuationMismatchReply{status: http.StatusServiceUnavailable, payload: `{"error":{"type":"server_error","message":"do-not-leak-provider"}}`}}, - } { - t.Run(test.name, func(t *testing.T) { - upstream := &continuationMismatchUpstream{ - accepting: map[int64]bool{2: true}, - sequence: map[int64][]continuationMismatchReply{1: {test.first, test.retry}}, - rejection: continuationMismatchUnrecoverable, - } - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, test.accountType)) - body := test.body - if test.stream { - body = strings.Replace(body, `"stream":false`, `"stream":true`, 1) - } - response := serveContinuationMismatch(t, handler, body) - - require.Equal(t, []int64{1, 1, 2}, upstream.accountIDs, "first send, stripped retry, then the next account") - require.False(t, gjson.GetBytes(upstream.bodies[1], "input.0.encrypted_content").Exists(), "the retry on the first account is the stripped one") - require.Equal(t, gjson.GetBytes(upstream.bodies[0], "input").Raw, gjson.GetBytes(upstream.bodies[2], "input").Raw, "the next account receives the request unchanged") - require.Equal(t, http.StatusOK, response.code, response.body) - require.NotContains(t, response.body, "do-not-leak") - require.NotContains(t, response.body, "response.failed") - if test.stream { - require.Contains(t, response.body, "response.completed") - } else { - require.Contains(t, response.body, "healthy answer") - } - require.Equal(t, int64(2), continuationMismatchBoundAccount(t, cache)) - }) - } -} - -// An account failure of the stripped retry is an ordinary account failure, not -// a mismatch: when nobody answers, the last account's failure is the answer and -// the session stays where selection put it. -func TestOpenAIGatewayHandler_ContinuationStrippedRetryAccountFailureIsNotAMismatch(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{ - sequence: map[int64][]continuationMismatchReply{1: {continuationMismatchNamedReasoning, - {status: http.StatusPaymentRequired, payload: `{"error":{"type":"insufficient_quota","code":"insufficient_quota","message":"do-not-leak-billing"}}`}}}, - replies: map[int64]continuationMismatchReply{2: {status: http.StatusServiceUnavailable, payload: `{"error":{"type":"server_error","message":"do-not-leak-provider"}}`}}, - } - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, continuationMismatchBody) - - require.Equal(t, []int64{1, 1, 2}, upstream.accountIDs) - require.GreaterOrEqual(t, response.code, http.StatusInternalServerError, response.body) - require.NotEqual(t, service.OpenAIContinuationStateUnavailableCode, gjson.Get(response.body, "error.code").String()) - require.NotContains(t, response.body, "do-not-leak") - require.Equal(t, int64(2), continuationMismatchBoundAccount(t, cache), "no mismatch, so no binding is moved back") -} - -// An OAuth account that failed on its stripped retry gets the cooldown of an -// account whose same-account attempts are used up: after a 401 the next request -// is not sent to it. (For an OpenAI API-key account that cooldown does nothing.) -func TestOpenAIGatewayHandler_ContinuationStrippedRetryAccountFailureCoolsTheAccount(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &continuationMismatchUpstream{ - accepting: map[int64]bool{1: true, 2: true}, - // An authentication failure blocks the whole account for a while, which - // the next selection can be seen to respect. - sequence: map[int64][]continuationMismatchReply{1: {continuationMismatchUnrecoverable, - {status: http.StatusUnauthorized, payload: `{"error":{"type":"authentication_error","message":"credentials rejected"}}`}}}, - } - handler, _ := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeOAuth)) - body := strings.Replace(continuationMismatchReasoningOnlyBody, `"stream":false`, `"stream":true`, 1) - serveContinuationMismatch(t, handler, body) - require.Equal(t, []int64{1, 1, 2}, upstream.accountIDs) - - serveContinuationMismatchSession(t, handler, body, "another-conversation") - require.Equal(t, []int64{1, 1, 2, 2}, upstream.accountIDs, "account 1 would accept again, but is cooling") -} - -// The upstream cannot find a reasoning item that the stripped retry left behind -// without ciphertext. The same account is asked once more, without the id, and -// answers; no other account is involved and the session stays where it was. -func TestOpenAIGatewayHandler_ContinuationStrippedItemTheUpstreamCannotFindIsRepairedOnTheSameAccount(t *testing.T) { - gin.SetMode(gin.TestMode) - const relayID = "rs_0123456789abcdef0123456789abcdef" - body := strings.Replace(continuationMismatchReasoningOnlyBody, `"rs_a"`, `"`+relayID+`"`, 1) - unfound := continuationMismatchReply{status: http.StatusNotFound, - payload: `{"error":{"message":"Item with id '` + relayID + `' not found. Items are not persisted when ` + "`store`" + ` is set to false.","type":"invalid_request_error","param":"input","code":null}}`} - upstream := &continuationMismatchUpstream{ - accepting: map[int64]bool{1: true, 2: true}, - sequence: map[int64][]continuationMismatchReply{1: {continuationMismatchUnrecoverable, unfound}}, - } - handler, cache := newContinuationMismatchHandler(t, upstream, continuationMismatchAccounts(2, service.AccountTypeAPIKey)) - response := serveContinuationMismatch(t, handler, body) - - require.Equal(t, http.StatusOK, response.code, response.body) - require.Equal(t, []int64{1, 1, 1}, upstream.accountIDs, "first send, stripped retry, then the repair") - require.Equal(t, relayID, gjson.GetBytes(upstream.bodies[1], "input.0.id").String()) - require.False(t, gjson.GetBytes(upstream.bodies[1], "input.0.encrypted_content").Exists()) - require.Equal(t, "reasoning", gjson.GetBytes(upstream.bodies[2], "input.0.type").String(), "the item is kept") - require.False(t, gjson.GetBytes(upstream.bodies[2], "input.0.id").Exists(), "without the id the upstream could not find") - require.Equal(t, int64(1), continuationMismatchBoundAccount(t, cache)) -} diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 6c15320dae1c..8be6e043a5f9 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -17,6 +17,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" "github.com/Wei-Shaw/sub2api/internal/pkg/ip" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" @@ -218,13 +219,6 @@ type grokMediaEligibilityProber interface { const maxOpenAIFirstOutputTimeoutSwitches = 1 -// maxOpenAICiphertextAccountSwitches bounds how often one request is handed -// to another account because an account could not read its ciphertext, within -// the ordinary switch budget. Every rejection costs an upload of the whole -// history, and when the account that issued the ciphertext is gone no account -// accepts. -const maxOpenAICiphertextAccountSwitches = 3 - func openAIForwardSucceededForScheduling(result *service.OpenAIForwardResult) bool { return result.SucceededForScheduling() } @@ -378,8 +372,9 @@ func allowOpenAICompatibleMessagesDispatch(c *gin.Context, apiKey *service.APIKe func openAICompatibleTextTargetAllowed(c *gin.Context, apiKey *service.APIKey, model string) bool { return compositeTargetPlatformAllowed(c, apiKey, model, - service.PlatformOpenAI, service.PlatformGrok, - service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, service.PlatformMiniMax) + domain.PlatformIDsWhere(func(spec domain.PlatformSpec) bool { + return spec.Gateway == domain.PlatformGatewayOpenAI + })...) } // isResponsesWebSocketCompositePlatform 限定 composite 分组在 Responses WebSocket @@ -724,37 +719,6 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { var passthroughFailoverState openAIPassthroughFailoverState var accountTypePreference service.OpenAIAccountTypePreference var sameAccountRetrySelection *service.AccountSelectionResult - // An account that cannot read the ciphertext in the request hands it to - // the next account. When the accounts or the switch budget run out, the - // client receives the first such failure, as it did before accounts were - // switched for it; an account that ends the request with an answer of its - // own is still the answer. - var ciphertextFailoverErr *service.UpstreamFailoverError - ciphertextFirstAccountID, ciphertextSessionHash := int64(0), "" - ciphertextLaterAccounts := make(map[int64]struct{}) - ciphertextSwitchCount := 0 - ciphertextAccepted := false - exhaustedFailoverErr := func(current *service.UpstreamFailoverError) *service.UpstreamFailoverError { - if ciphertextFailoverErr != nil { - return ciphertextFailoverErr - } - return current - } - // Selecting one of the later accounts may have moved the session to it. - // When none of them answered, they are no better home for the conversation - // than the account that first rejected it, so a binding this request moved - // goes back there. A binding the scheduler kept elsewhere is not touched. - defer func() { - if ciphertextFailoverErr == nil || ciphertextAccepted { - return - } - restoreCtx, cancel := context.WithTimeout(context.WithoutCancel(c.Request.Context()), 2*time.Second) - defer cancel() - if err := h.gatewayService.MoveStickySessionIfBoundTo(restoreCtx, apiKey.GroupID, ciphertextSessionHash, ciphertextFirstAccountID, ciphertextLaterAccounts); err != nil { - reqLog.Warn("openai.ciphertext_mismatch_session_restore_failed", zap.Int64("account_id", ciphertextFirstAccountID), zap.Error(err)) - } - }() - // 分组利润控制:请求级装配定价上下文——pricingAt 固定本请求的 // D 与计费高峰因子,选号、槽位终检与全部 failover 重入共用同一门与阈值。 // 生图意图只影响能力路由与图片计费,不关门:混合 /v1/responses 请求的 @@ -813,7 +777,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { zap.Int("excluded_account_count", len(failedAccountIDs)), ) if retryingSameAccount && lastFailoverErr != nil { - h.handleFailoverExhausted(c, exhaustedFailoverErr(lastFailoverErr), streamStarted) + h.handleFailoverExhausted(c, lastFailoverErr, streamStarted) return } if len(failedAccountIDs) == 0 { @@ -830,16 +794,13 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { return } if lastFailoverErr != nil { - h.handleFailoverExhausted(c, exhaustedFailoverErr(lastFailoverErr), streamStarted) + h.handleFailoverExhausted(c, lastFailoverErr, streamStarted) } else { h.handleFailoverExhaustedSimple(c, 502, streamStarted) } return } - if (selection == nil || selection.Account == nil) && ciphertextFailoverErr != nil { - h.handleFailoverExhausted(c, ciphertextFailoverErr, streamStarted) - return - } + if selection == nil || selection.Account == nil { cls := classifyNoAccountErrorFromGin(c, h.gatewayService, apiKey, routingModel, reqModel, requestPlatform) if !cls.ModelNotFound { @@ -866,9 +827,6 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { sessionHash = ensureOpenAIPoolModeSessionHash(sessionHash, account) reqLog.Debug("openai.account_selected", zap.Int64("account_id", account.ID), zap.String("account_name", account.Name)) setOpsSelectedAccount(c, account.ID, account.Platform) - if ciphertextFailoverErr != nil && account.ID != ciphertextFirstAccountID { - ciphertextLaterAccounts[account.ID] = struct{}{} - } var accountReleaseFunc func() var slotResult openAISlotAcquireResult @@ -891,7 +849,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { c, account, account.GetMappedModel(routingModel), lastFailoverErr, failedAccountIDs, &switchCount, maxAccountSwitches, &oauth429FailoverState, "responses", reqLog, true, - func() { h.handleFailoverExhausted(c, exhaustedFailoverErr(lastFailoverErr), streamStarted) }, + func() { h.handleFailoverExhausted(c, lastFailoverErr, streamStarted) }, ) { continue } @@ -958,11 +916,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { }) } if err != nil { - if ciphertextFailoverErr != nil && service.OpenAICompactKeepaliveAdjustedWrittenSize(c) != writerSizeBeforeForward { - // This account read the ciphertext and was answering when the - // attempt failed: the conversation stays with it. - ciphertextAccepted = true - } + if result != nil && result.ClientDisconnect { reqLog.Info("openai.client_disconnected", zap.Int64("account_id", account.ID), @@ -986,9 +940,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { zap.Error(err), ) } else { - if h.handleReasoningRecoveryTerminal(c, err, selection, account, account.GetMappedModel(routingModel), streamStarted, h.gatewayService) { - return - } + var failoverErr *service.UpstreamFailoverError if errors.As(err, &failoverErr) { if failoverClientGone(c) { @@ -1038,31 +990,17 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { h.handleFailoverExhausted(c, failoverErr, streamStarted) return } - retryAction := openAIFailoverRetrySwitchAccount - switch { - case failoverErr.IsOpenAICiphertextAccountMismatch(): - // A mismatch says nothing against the account: no - // same-account retry and no cooldown, the next account is tried. - case failoverErr.IsOpenAIRecoveryRetrySpent(): - // The account failed on its stripped retry and is not asked - // again for this request. The retry-exhausted cooldown runs as - // for used-up same-account attempts; it does nothing for an - // OpenAI API-key account, whose penalties the forwarding - // layer writes on a first send. - h.gatewayService.CooldownOpenAIRetryExhausted(c.Request.Context(), account, account.GetMappedModel(routingModel), failoverErr) - default: - retryAction = retryState.HandleHTTP( - c.Request.Context(), - h.gatewayService, - account, - account.GetMappedModel(routingModel), - failoverErr, - true, - sameAccountRetryDelay, - "responses", - ) - } - h.finalizeOpenAIHTTPFailoverSelection(c, selection, account, account.GetMappedModel(routingModel), failoverErr, retryAction) + retryAction := retryState.HandleHTTP( + c.Request.Context(), + h.gatewayService, + account, + account.GetMappedModel(routingModel), + failoverErr, + true, + sameAccountRetryDelay, + "responses", + ) + switch retryAction { case openAIFailoverRetryReselect: continue @@ -1088,28 +1026,14 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { AllowCompatibleFallback: true, } } - if failoverErr.IsOpenAICiphertextAccountMismatch() { - if ciphertextFailoverErr == nil { - ciphertextFailoverErr = failoverErr - ciphertextFirstAccountID, ciphertextSessionHash = account.ID, sessionHash - } - if ciphertextSwitchCount >= maxOpenAICiphertextAccountSwitches { - h.handleFailoverExhausted(c, ciphertextFailoverErr, streamStarted) - return - } - ciphertextSwitchCount++ - service.ResetOpenAIReasoningRecoveryAttempt(c) - } - if failoverErr.IsOpenAIRecoveryRetrySpent() { - service.ResetOpenAIReasoningRecoveryAttempt(c) - } + if switchCount >= maxAccountSwitches { - h.handleFailoverExhausted(c, exhaustedFailoverErr(failoverErr), streamStarted) + h.handleFailoverExhausted(c, failoverErr, streamStarted) return } switchCount++ if h.gatewayService.ShouldStopOpenAIOAuth429Failover(account, failoverErr.StatusCode, switchCount, &oauth429FailoverState) { - h.handleFailoverExhausted(c, exhaustedFailoverErr(failoverErr), streamStarted) + h.handleFailoverExhausted(c, failoverErr, streamStarted) return } h.gatewayService.RecordOpenAIAccountSwitch() @@ -1118,7 +1042,6 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { zap.Int("upstream_status", failoverErr.StatusCode), zap.Int("switch_count", switchCount), zap.Int("max_switches", maxAccountSwitches), - zap.Bool("ciphertext_account_mismatch", failoverErr.IsOpenAICiphertextAccountMismatch()), } if account.Proxy != nil { failoverSwitchFields = append(failoverSwitchFields, @@ -1171,15 +1094,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { if shouldSubmitOpenAIUsage(err, result) { submitResponsesUsage(result) } - ciphertextAccepted = true - if ciphertextFailoverErr != nil { - // Selection usually bound the session to this account already. Where - // it leaves a binding in place, move one that still points at the - // account that could not read the conversation. - if bindErr := h.gatewayService.MoveStickySessionIfBoundTo(c.Request.Context(), apiKey.GroupID, sessionHash, account.ID, map[int64]struct{}{ciphertextFirstAccountID: {}}); bindErr != nil { - reqLog.Warn("openai.ciphertext_mismatch_session_bind_failed", zap.Int64("account_id", account.ID), zap.Error(bindErr)) - } - } + reqLog.Debug("openai.request_completed", zap.Int64("account_id", account.ID), zap.Int("switch_count", switchCount), @@ -2399,6 +2314,101 @@ func (p *openAIWSTurnPricing) currentOr(fallback time.Time) time.Time { return fallback } +// openAIWSTurnAPIKeyLookup 是后续 turn 重取 API Key 认证快照的入口 +// (APIKeyService.GetByKey:经 L1/L2 认证缓存,未命中才回源)。 +type openAIWSTurnAPIKeyLookup interface { + GetByKey(ctx context.Context, key string) (*service.APIKey, error) +} + +// openAIWSTurnBillingAPIKeys 按 turn 号保存每个 turn 计费用的 API Key 快照。 +// +// API Key 认证快照只在建连时取一次。若连接内所有 turn 共用它,分组定价(倍率、 +// 高峰、图片定价、利润门参数)会停留在建连时刻:管理员调价后,已打开的长连接 +// 在客户端重连前一直按旧价计费、按旧售价过利润门,而 HTTP 请求每次都经认证缓存 +// 取快照,缓存失效后即生效。 +// +// 因此后续 turn 在 BeforeTurn(与 HTTP 请求进入认证中间件同位)经同一认证缓存 +// 重取快照,只采用其中的分组,作为该 turn 利润门准入与用量计费共同的分组;用户、 +// Key 限额、订阅仍沿用建连快照。只在同一把 Key、同一分组且平台与订阅类型未变时 +// 采用:Key 换组或分组改平台/订阅类型不是调价,该连接按建连分组调度,继续按建连 +// 快照计费;重取失败同样保留建连快照,不断连。 +// +// 首轮沿用刚经认证中间件取得的建连快照;没有经过 BeforeTurn 的 turn 回退建连 +// 快照。只保留当前与上一个 turn:下一 turn 的 BeforeTurn 先于上一 turn 的 +// AfterTurn 执行时,上一 turn 仍取到自己的快照。 +type openAIWSTurnBillingAPIKeys struct { + mu sync.Mutex + keys map[int]*service.APIKey +} + +// begin 在 BeforeTurn 重装利润门前调用:后续 turn 重取计费分组并按 turn 记下, +// 返回换入该分组的上下文供本 turn 的利润门使用。 +func (k *openAIWSTurnBillingAPIKeys) begin(ctx context.Context, apiKeyService *service.APIKeyService, turn int, conn *service.APIKey) context.Context { + turnKey := conn + if turn > 1 && apiKeyService != nil { + turnKey = refreshOpenAIWSTurnBillingAPIKey(ctx, apiKeyService, conn) + } + k.set(turn, turnKey) + return withOpenAIWSTurnBillingGroup(ctx, turnKey) +} + +func (k *openAIWSTurnBillingAPIKeys) set(turn int, key *service.APIKey) { + k.mu.Lock() + defer k.mu.Unlock() + if k.keys == nil { + k.keys = make(map[int]*service.APIKey, 2) + } + for t := range k.keys { + if t < turn-1 { + delete(k.keys, t) + } + } + k.keys[turn] = key +} + +func (k *openAIWSTurnBillingAPIKeys) forTurn(turn int, conn *service.APIKey) *service.APIKey { + k.mu.Lock() + defer k.mu.Unlock() + if key := k.keys[turn]; key != nil { + return key + } + return conn +} + +// refreshOpenAIWSTurnBillingAPIKey 返回本 turn 计费用的 API Key:分组取当前认证 +// 快照,其余字段与建连快照共享。不满足采用条件时原样返回建连快照。 +func refreshOpenAIWSTurnBillingAPIKey(ctx context.Context, lookup openAIWSTurnAPIKeyLookup, conn *service.APIKey) *service.APIKey { + if lookup == nil || conn == nil || conn.Key == "" || conn.GroupID == nil || conn.Group == nil { + return conn + } + latest, err := lookup.GetByKey(ctx, conn.Key) + if err != nil || latest == nil || latest.ID != conn.ID || latest.GroupID == nil || *latest.GroupID != *conn.GroupID { + return conn + } + group := latest.Group + if group == nil || group.ID != conn.Group.ID || + group.Platform != conn.Group.Platform || + group.SubscriptionType != conn.Group.SubscriptionType { + return conn + } + turnKey := *conn + turnKey.Group = group + return &turnKey +} + +// withOpenAIWSTurnBillingGroup 把本 turn 的计费分组换进认证分组上下文,使利润门 +// 的售价与本 turn 计费同源。上下文里没有同 ID 的认证分组时不改动。 +func withOpenAIWSTurnBillingGroup(ctx context.Context, turnKey *service.APIKey) context.Context { + if turnKey == nil || turnKey.Group == nil { + return ctx + } + current, ok := ctx.Value(ctxkey.Group).(*service.Group) + if !ok || current == nil || current == turnKey.Group || current.ID != turnKey.Group.ID { + return ctx + } + return context.WithValue(ctx, ctxkey.Group, turnKey.Group) +} + // recordOpenAIProfitVeto 记录 OpenAI 侧选号循环的一次利润门终检否决:把账号 // 加入本请求排除集并递增否决计数。返回 false 表示否决次数已达 // maxProfitVetoAttempts,调用方必须停止重选并按「无可用账号」终止。 @@ -3193,6 +3203,9 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) { // turn 级定价:首轮回退到 TurnStarted 的所属 turn 时刻;后续 turn 由 // BeforeTurn 重新冻结 pricingAt 并按最新门复核当前账号。 var turnPricing openAIWSTurnPricing + // 后续 turn 的计费分组同样在 BeforeTurn 经认证缓存重取,使分组调价对 + // 已打开的连接生效。 + var turnBillingAPIKeys openAIWSTurnBillingAPIKeys // Passthrough ingress does not invoke BeforeTurn for the first frame. if err := checkSimpleModeTurnBilling(); err != nil { closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "billing check failed") @@ -3280,7 +3293,8 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) { // 长连接跨峰谷/倍率刷新防护:每个 turn 按当前时刻重装门并复核 // 当前账号,越线即要求客户端重连重选(连接绑定单一上游账号, // 无法中途换号)。本 turn 的准入与计费共用同一 pricingAt。 - turnCtx, turnAt := h.gatewayService.WithOpenAITurnPricingContext(ctx, apiKey.GroupID) + turnBillingCtx := turnBillingAPIKeys.begin(ctx, h.apiKeyService, turn, apiKey) + turnCtx, turnAt := h.gatewayService.WithOpenAITurnPricingContext(turnBillingCtx, apiKey.GroupID) if _, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(turnCtx, account); vetoed { reqLog.Info("openai.websocket_turn_profit_vetoed", zap.Int("turn", turn), @@ -3413,8 +3427,9 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) { } h.gatewayService.ReportOpenAIAccountScheduleResultForSelection(selection, account.ID, scheduleModel, openAIForwardSucceededForScheduling(result), result.FirstTokenMs) turnRecordPricingAt := turnPricing.currentOr(turnStart) + turnBillingAPIKey := turnBillingAPIKeys.forTurn(turn, apiKey) cyberBlocked := service.GetOpsCyberPolicy(c) != nil - turnUsageSnapshot := snapshotOpenAIUsageMetadataWithHash(c, apiKey, account, subscription, turnMapping, turnRequestedModel, result, requestPayloadHash) + turnUsageSnapshot := snapshotOpenAIUsageMetadataWithHash(c, turnBillingAPIKey, account, subscription, turnMapping, turnRequestedModel, result, requestPayloadHash) turnUsageSnapshot.channelFields = turnUsageFields turnUsageSnapshot.cyberBlocked = cyberBlocked turnUsageInput := turnUsageSnapshot.Input(result, h.apiKeyService, turnRecordPricingAt) @@ -4045,30 +4060,6 @@ func (h *OpenAIGatewayHandler) handleOpenAINoAccountError(c *gin.Context, classi h.handleStreamingAwareErrorWithCode(c, classification.Status, classification.ErrType, code, classification.Message, streamStarted, false) } -// A recovery terminal ends the request: it is an attempt that spent its -// stripped retry, or one whose ciphertext rejection arrived after output was -// committed, and that was not handed on. Preserve its classification without -// allowing refusal, account, or transport retries to re-enter the loop. An -// account mismatch and an account failure of the stripped retry, both -// possible only before anything reached the client, are returned as failovers -// instead and never reach this function. -func (h *OpenAIGatewayHandler) handleReasoningRecoveryTerminal(c *gin.Context, err error, selection *service.AccountSelectionResult, account *service.Account, model string, streamStarted bool, reporter openAIFailoverSelectionReporter) bool { - var terminal *service.OpenAIReasoningRecoveryTerminalError - if !errors.As(err, &terminal) { - return false - } - if failoverClientGone(c) { - reporter.ReleaseOpenAIRuntimeBreakerProbeForSelection(selection) - return true - } - finalizeOpenAIFailoverSelection(reporter, selection, account, model, terminal.Failure, openAIFailoverRetryStop) - if terminal.FailureTerminalForwarded { - return true - } - h.handleFailoverExhausted(c, terminal.Failure, streamStarted || c.Writer.Written()) - return true -} - func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverErr *service.UpstreamFailoverError, streamStarted bool) { if failoverErr == nil { h.handleFailoverExhaustedSimple(c, http.StatusBadGateway, streamStarted) diff --git a/backend/internal/handler/openai_gateway_handler_test.go b/backend/internal/handler/openai_gateway_handler_test.go index 29a667e3147e..1649e3108200 100644 --- a/backend/internal/handler/openai_gateway_handler_test.go +++ b/backend/internal/handler/openai_gateway_handler_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" pkghttputil "github.com/Wei-Shaw/sub2api/internal/pkg/httputil" "github.com/Wei-Shaw/sub2api/internal/pkg/pagination" "github.com/Wei-Shaw/sub2api/internal/pkg/xai" @@ -2110,6 +2111,12 @@ type openAIResponsesWSUsageLogCase struct { afterFirstUpstreamRequest func(channelSvc *service.ChannelService) error // group 覆盖 apiKey.Group(分组级模型白名单测试用);nil 保持原有无分组行为。 group *service.Group + // apiKeyService 非 nil 时模拟 API Key 认证中间件:连接认证快照经它按 + // apiKeyCredential 取得,并把其分组放入请求 ctx;handler 也使用它。 + apiKeyService *service.APIKeyService + apiKeyCredential string + // accountRateMultiplier 覆盖账号倍率(利润门测试用)。 + accountRateMultiplier *float64 // firstFrameCloseExpected:首帧即被拒(连接被 1008 关闭),不期待任何响应帧。 firstFrameCloseExpected bool // secondTurnCloseExpected:第二个 turn 被拒(连接被 1008 关闭)。 @@ -3455,6 +3462,7 @@ func runOpenAIResponsesWebSocketUsageLogCase(t *testing.T, tc openAIResponsesWSU if tc.accountPlatform != "" { account.Platform = tc.accountPlatform } + account.RateMultiplier = tc.accountRateMultiplier if strings.TrimSpace(tc.ingressMode) != "" { account.Extra["openai_apikey_responses_websockets_v2_mode"] = tc.ingressMode } @@ -3547,6 +3555,12 @@ func runOpenAIResponsesWebSocketUsageLogCase(t *testing.T, tc openAIResponsesWSU Platform: firstNonEmpty(tc.groupPlatform, service.PlatformOpenAI), }, } + if tc.apiKeyService != nil { + h.apiKeyService = tc.apiKeyService + authKey, err := tc.apiKeyService.GetByKey(context.Background(), tc.apiKeyCredential) + require.NoError(t, err) + apiKey = authKey + } if tc.simpleModeRejectAtRead > 0 { apiKey.RateLimit5h = 1 } @@ -3558,6 +3572,9 @@ func runOpenAIResponsesWebSocketUsageLogCase(t *testing.T, tc openAIResponsesWSU router.Use(func(c *gin.Context) { c.Set(string(middleware.ContextKeyAPIKey), apiKey) c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: apiKey.User.ID, Concurrency: 1}) + if tc.apiKeyService != nil && apiKey.Group != nil { + c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), ctxkey.Group, apiKey.Group)) + } c.Next() }) router.GET("/openai/v1/responses", h.ResponsesWebSocket) diff --git a/backend/internal/handler/openai_gateway_ws_group_pricing_test.go b/backend/internal/handler/openai_gateway_ws_group_pricing_test.go new file mode 100644 index 000000000000..7b1a581532f6 --- /dev/null +++ b/backend/internal/handler/openai_gateway_ws_group_pricing_test.go @@ -0,0 +1,293 @@ +package handler + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/stretchr/testify/require" +) + +// 分组调价对已打开的 WebSocket 连接生效:同一条连接的两个 turn 之间由管理端 +// 改分组倍率(并按分组失效认证缓存),第二个 turn 必须按新倍率计费、按新售价 +// 过利润门;HTTP 请求每次经认证缓存取快照,本来就是这个行为。 + +const ( + wsGroupPricingGroupID = int64(4201) // 与 runOpenAIResponsesWebSocketUsageLogCase 的调度分组一致 + wsGroupPricingKey = "sk-ws-group-pricing" + wsGroupPricingFirst = `{"type":"response.create","model":"gpt-5.6-sol","stream":false}` + wsGroupPricingSecond = `{"type":"response.create","model":"gpt-5.6-sol","stream":false}` +) + +// wsGroupPricingAPIKeyRepoStub 只实现认证查询与按分组列 Key(认证缓存失效用)。 +type wsGroupPricingAPIKeyRepoStub struct { + service.APIKeyRepository + mu sync.Mutex + apiKey service.APIKey + group service.Group + err error + lookups int +} + +func newWSGroupPricingAPIKeyRepoStub(rate float64) *wsGroupPricingAPIKeyRepoStub { + groupID := wsGroupPricingGroupID + return &wsGroupPricingAPIKeyRepoStub{ + apiKey: service.APIKey{ + ID: 1801, + UserID: 1701, + Key: wsGroupPricingKey, + Status: service.StatusActive, + GroupID: &groupID, + User: &service.User{ID: 1701, Status: service.StatusActive, Concurrency: 1}, + }, + group: service.Group{ + ID: groupID, + Name: "ws-group-pricing", + Platform: service.PlatformOpenAI, + Status: service.StatusActive, + RateMultiplier: rate, + Hydrated: true, + }, + } +} + +func (s *wsGroupPricingAPIKeyRepoStub) GetByKeyForAuth(ctx context.Context, key string) (*service.APIKey, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.lookups++ + if s.err != nil { + return nil, s.err + } + if key != s.apiKey.Key { + return nil, service.ErrAPIKeyNotFound + } + apiKey := s.apiKey + user := *s.apiKey.User + apiKey.User = &user + group := s.group + apiKey.Group = &group + return &apiKey, nil +} + +func (s *wsGroupPricingAPIKeyRepoStub) ListKeysByGroupID(ctx context.Context, groupID int64) ([]string, error) { + return []string{s.apiKey.Key}, nil +} + +// adminUpdate 模拟管理端改 Key/分组后按分组失效认证缓存(与 admin 分组更新同一入口)。 +func (s *wsGroupPricingAPIKeyRepoStub) adminUpdate(apiKeyService *service.APIKeyService, fn func(apiKey *service.APIKey, group *service.Group)) error { + s.mu.Lock() + fn(&s.apiKey, &s.group) + s.mu.Unlock() + apiKeyService.InvalidateAuthCacheByGroupID(context.Background(), wsGroupPricingGroupID) + return nil +} + +func (s *wsGroupPricingAPIKeyRepoStub) lookupCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.lookups +} + +func newWSGroupPricingAPIKeyService(repo *wsGroupPricingAPIKeyRepoStub) *service.APIKeyService { + return service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, &config.Config{}) +} + +func TestOpenAIResponsesWebSocket_GroupRateChangeAppliesToNextTurn(t *testing.T) { + for _, mode := range []string{service.OpenAIWSIngressModePassthrough, service.OpenAIWSIngressModeCtxPool} { + t.Run(mode, func(t *testing.T) { + repo := newWSGroupPricingAPIKeyRepoStub(3.0) + apiKeyService := newWSGroupPricingAPIKeyService(repo) + got := runOpenAIResponsesWebSocketUsageLogCase(t, openAIResponsesWSUsageLogCase{ + firstPayload: wsGroupPricingFirst, + secondPayload: wsGroupPricingSecond, + ingressMode: mode, + apiKeyService: apiKeyService, + apiKeyCredential: wsGroupPricingKey, + afterFirstUpstreamRequest: func(*service.ChannelService) error { + return repo.adminUpdate(apiKeyService, func(_ *service.APIKey, group *service.Group) { group.RateMultiplier = 0.3 }) + }, + }) + + require.Len(t, got.logs, 2) + require.InDelta(t, 3.0, got.logs[0].RateMultiplier, 1e-12, "turn 1 bills at the rate in force at connect") + require.InDelta(t, 0.3, got.logs[1].RateMultiplier, 1e-12, "turn 2 must bill at the group rate in force when it starts") + require.Greater(t, got.logs[0].ActualCost, 0.0) + require.InDelta(t, got.logs[0].ActualCost/10, got.logs[1].ActualCost, 1e-15) + }) + } +} + +func TestOpenAIResponsesWebSocket_GroupRateChangeReachesProfitGateOnNextTurn(t *testing.T) { + repo := newWSGroupPricingAPIKeyRepoStub(3.0) + repo.group.ProfitControlEnabled = true + repo.group.ProfitMinMargin = 0.1 + apiKeyService := newWSGroupPricingAPIKeyService(repo) + accountRate := 1.0 + runOpenAIResponsesWebSocketUsageLogCase(t, openAIResponsesWSUsageLogCase{ + firstPayload: wsGroupPricingFirst, + secondPayload: wsGroupPricingSecond, + apiKeyService: apiKeyService, + apiKeyCredential: wsGroupPricingKey, + accountRateMultiplier: &accountRate, + // 3.0 → 0.3:账号倍率 1.0 在新售价下已越线,第二个 turn 必须在转发上游前被拒。 + afterFirstUpstreamRequest: func(*service.ChannelService) error { + return repo.adminUpdate(apiKeyService, func(_ *service.APIKey, group *service.Group) { group.RateMultiplier = 0.3 }) + }, + secondTurnCloseExpected: true, + closeStatus: coderws.StatusTryAgainLater, + closeReason: "no longer eligible for this connection", + }) +} + +func TestOpenAIResponsesWebSocket_KeyMovedToAnotherGroupKeepsConnectionGroup(t *testing.T) { + repo := newWSGroupPricingAPIKeyRepoStub(3.0) + apiKeyService := newWSGroupPricingAPIKeyService(repo) + got := runOpenAIResponsesWebSocketUsageLogCase(t, openAIResponsesWSUsageLogCase{ + firstPayload: wsGroupPricingFirst, + secondPayload: wsGroupPricingSecond, + apiKeyService: apiKeyService, + apiKeyCredential: wsGroupPricingKey, + afterFirstUpstreamRequest: func(*service.ChannelService) error { + return repo.adminUpdate(apiKeyService, func(apiKey *service.APIKey, group *service.Group) { + other := wsGroupPricingGroupID + 1 + apiKey.GroupID = &other + group.ID = other + group.RateMultiplier = 0.3 + }) + }, + }) + + require.Len(t, got.logs, 2) + require.InDelta(t, 3.0, got.logs[1].RateMultiplier, 1e-12, + "a key moved to another group keeps the connection's group: the connection was scheduled from it") +} + +func TestOpenAIResponsesWebSocket_FailedKeyRefreshKeepsConnectionGroup(t *testing.T) { + repo := newWSGroupPricingAPIKeyRepoStub(3.0) + apiKeyService := newWSGroupPricingAPIKeyService(repo) + got := runOpenAIResponsesWebSocketUsageLogCase(t, openAIResponsesWSUsageLogCase{ + firstPayload: wsGroupPricingFirst, + secondPayload: wsGroupPricingSecond, + apiKeyService: apiKeyService, + apiKeyCredential: wsGroupPricingKey, + afterFirstUpstreamRequest: func(*service.ChannelService) error { + return repo.adminUpdate(apiKeyService, func(_ *service.APIKey, group *service.Group) { + group.RateMultiplier = 0.3 + repo.err = errors.New("auth store unavailable") + }) + }, + }) + + require.Len(t, got.logs, 2) + require.InDelta(t, 3.0, got.logs[1].RateMultiplier, 1e-12, "a failed refresh keeps the connection snapshot") + require.GreaterOrEqual(t, repo.lookupCount(), 2, "the second turn must have attempted a refresh") +} + +type wsTurnAPIKeyLookupFunc func(ctx context.Context, key string) (*service.APIKey, error) + +func (f wsTurnAPIKeyLookupFunc) GetByKey(ctx context.Context, key string) (*service.APIKey, error) { + return f(ctx, key) +} + +func TestRefreshOpenAIWSTurnBillingAPIKey(t *testing.T) { + groupID := int64(77) + conn := &service.APIKey{ + ID: 5, + Key: "sk-conn", + GroupID: &groupID, + User: &service.User{ID: 9}, + Group: &service.Group{ID: groupID, Platform: service.PlatformOpenAI, SubscriptionType: service.SubscriptionTypeStandard, RateMultiplier: 3}, + } + latestWith := func(mutate func(k *service.APIKey)) *service.APIKey { + gid := groupID + k := &service.APIKey{ + ID: 5, + Key: "sk-conn", + GroupID: &gid, + User: &service.User{ID: 9, Balance: 123}, + Group: &service.Group{ID: groupID, Platform: service.PlatformOpenAI, SubscriptionType: service.SubscriptionTypeStandard, RateMultiplier: 0.3}, + } + if mutate != nil { + mutate(k) + } + return k + } + + t.Run("same group adopts the latest group and nothing else", func(t *testing.T) { + got := refreshOpenAIWSTurnBillingAPIKey(context.Background(), wsTurnAPIKeyLookupFunc(func(ctx context.Context, key string) (*service.APIKey, error) { + require.Equal(t, "sk-conn", key) + return latestWith(nil), nil + }), conn) + require.NotSame(t, conn, got) + require.InDelta(t, 0.3, got.Group.RateMultiplier, 1e-12) + require.Same(t, conn.User, got.User, "only the group snapshot is refreshed") + require.Equal(t, conn.ID, got.ID) + require.InDelta(t, 3.0, conn.Group.RateMultiplier, 1e-12, "the connection snapshot is never mutated") + }) + + keep := map[string]wsTurnAPIKeyLookupFunc{ + "lookup error": func(ctx context.Context, key string) (*service.APIKey, error) { + return nil, errors.New("boom") + }, + "not found": func(ctx context.Context, key string) (*service.APIKey, error) { + return nil, service.ErrAPIKeyNotFound + }, + "nil key": func(ctx context.Context, key string) (*service.APIKey, error) { return nil, nil }, + "different key id": func(ctx context.Context, key string) (*service.APIKey, error) { + return latestWith(func(k *service.APIKey) { k.ID = 6 }), nil + }, + "moved group": func(ctx context.Context, key string) (*service.APIKey, error) { + return latestWith(func(k *service.APIKey) { other := int64(78); k.GroupID = &other; k.Group.ID = other }), nil + }, + "no group": func(ctx context.Context, key string) (*service.APIKey, error) { + return latestWith(func(k *service.APIKey) { k.Group = nil }), nil + }, + "platform changed": func(ctx context.Context, key string) (*service.APIKey, error) { + return latestWith(func(k *service.APIKey) { k.Group.Platform = service.PlatformAnthropic }), nil + }, + "billing type changed": func(ctx context.Context, key string) (*service.APIKey, error) { + return latestWith(func(k *service.APIKey) { k.Group.SubscriptionType = service.SubscriptionTypeSubscription }), nil + }, + } + for name, lookup := range keep { + t.Run(name+" keeps the connection snapshot", func(t *testing.T) { + require.Same(t, conn, refreshOpenAIWSTurnBillingAPIKey(context.Background(), lookup, conn)) + }) + } + + t.Run("no credential or no group skips the lookup", func(t *testing.T) { + called := false + lookup := wsTurnAPIKeyLookupFunc(func(ctx context.Context, key string) (*service.APIKey, error) { + called = true + return latestWith(nil), nil + }) + noKey := *conn + noKey.Key = "" + require.Same(t, &noKey, refreshOpenAIWSTurnBillingAPIKey(context.Background(), lookup, &noKey)) + noGroup := *conn + noGroup.Group = nil + require.Same(t, &noGroup, refreshOpenAIWSTurnBillingAPIKey(context.Background(), lookup, &noGroup)) + require.Nil(t, refreshOpenAIWSTurnBillingAPIKey(context.Background(), lookup, nil)) + require.Same(t, conn, refreshOpenAIWSTurnBillingAPIKey(context.Background(), nil, conn)) + require.False(t, called) + }) +} + +func TestOpenAIWSTurnBillingAPIKeysKeepPreviousTurnUntilItIsRecorded(t *testing.T) { + conn := &service.APIKey{ID: 1} + turn2 := &service.APIKey{ID: 2} + turn3 := &service.APIKey{ID: 3} + var keys openAIWSTurnBillingAPIKeys + require.Same(t, conn, keys.forTurn(1, conn), "a turn without BeforeTurn bills with the connection snapshot") + keys.set(2, turn2) + keys.set(3, turn3) // turn 3 starts before turn 2's usage is submitted + require.Same(t, turn2, keys.forTurn(2, conn)) + require.Same(t, turn3, keys.forTurn(3, conn)) + keys.set(4, conn) + require.Same(t, conn, keys.forTurn(2, conn), "only the current and previous turn are kept") + require.Len(t, keys.keys, 2) +} diff --git a/backend/internal/handler/openai_reasoning_recovery_terminal_handler_test.go b/backend/internal/handler/openai_reasoning_recovery_terminal_handler_test.go deleted file mode 100644 index cddf1dbdc7db..000000000000 --- a/backend/internal/handler/openai_reasoning_recovery_terminal_handler_test.go +++ /dev/null @@ -1,195 +0,0 @@ -//go:build unit - -package handler - -import ( - "bytes" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" -) - -type recoveryTerminalHandlerUpstream struct { - service.HTTPUpstream - accountIDs []int64 - bodies [][]byte - finalCode int - committed bool -} - -func (u *recoveryTerminalHandlerUpstream) Do(req *http.Request, _ string, accountID int64, _ int) (*http.Response, error) { - body, err := io.ReadAll(req.Body) - if err != nil { - return nil, err - } - u.accountIDs = append(u.accountIDs, accountID) - u.bodies = append(u.bodies, body) - if u.committed { - return &http.Response{ - StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}}, - Body: io.NopCloser(strings.NewReader("event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"already delivered\"}\n\n" + - "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_partial\",\"status\":\"failed\",\"error\":{\"type\":\"invalid_request_error\",\"code\":\"invalid_encrypted_content\",\"message\":\"signature rejected\"}}}\n\n")), - }, nil - } - status := http.StatusBadRequest - payload := `{"error":{"type":"invalid_request_error","code":"invalid_encrypted_content","param":"input[0].encrypted_content","message":"do-not-leak-first"}}` - if len(u.accountIDs) > 1 { - status = u.finalCode - payload = `{"error":{"type":"invalid_request_error","param":"input","message":"do-not-leak-second"}}` - if status >= 500 { - payload = `{"error":{"type":"server_error","message":"do-not-leak-server"}}` - } - } - return &http.Response{ - StatusCode: status, Header: http.Header{"Content-Type": {"application/json"}}, - Body: io.NopCloser(strings.NewReader(payload)), - }, nil -} - -func TestOpenAIGatewayHandler_ReasoningFailureAlreadyForwardedIsNotDuplicated(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, passthrough := range []bool{false, true} { - mode := "native" - if passthrough { - mode = "passthrough" - } - t.Run(mode, func(t *testing.T) { - upstream := &recoveryTerminalHandlerUpstream{committed: true} - account := service.Account{ - ID: 1, Name: "response-already-forwarded", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey, - Status: service.StatusActive, Schedulable: true, - Credentials: map[string]any{"api_key": "fixture", "base_url": "https://api.example.test"}, - Extra: map[string]any{"use_responses_api": true, "openai_passthrough": passthrough}, - } - handler := newOpenAIResponsesFailoverTestHandler(t, upstream, account) - c, recorder := newOpenAIResponsesFailoverTestContextWithStream(t, nil, true) - handler.Responses(c) - require.Equal(t, []int64{1}, upstream.accountIDs, "semantic output forbids any replay") - require.Contains(t, recorder.Body.String(), "already delivered") - require.Equal(t, 1, strings.Count(recorder.Body.String(), `"type":"response.failed"`)) - require.NotContains(t, recorder.Body.String(), service.OpenAIContinuationStateUnavailableClientMessage) - require.Zero(t, handler.gatewayService.SnapshotOpenAIAccountSchedulerMetrics().RuntimeStatsAccountCount) - }) - } -} - -// After the stripped retry, a request rejection ends the request on that -// account; a provider failure is the account's own and the request goes to the -// next account, never back to the first. -func TestOpenAIGatewayHandler_ReasoningRecoveryOutcomeAfterTheStrippedRetry(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, passthrough := range []bool{false, true} { - mode := "native" - if passthrough { - mode = "passthrough" - } - for _, failure := range []struct { - name string - status int - clientStatus int - healthEvents int - accountIDs []int64 - }{ - {"request_rejected", http.StatusBadRequest, http.StatusBadRequest, 0, []int64{1, 1}}, - {"provider_error", http.StatusInternalServerError, http.StatusBadGateway, 1, []int64{1, 1, 2}}, - } { - t.Run(mode+"/"+failure.name, func(t *testing.T) { - setupOpsErrorLogTestQueue(t, 4) - upstream := &recoveryTerminalHandlerUpstream{finalCode: failure.status} - accounts := []service.Account{ - {ID: 1, Name: "recovery-source", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey, - Status: service.StatusActive, Schedulable: true, - Credentials: map[string]any{"api_key": "fixture", "base_url": "https://api.example.test", "pool_mode": true, "pool_mode_retry_count": 5}, - Extra: map[string]any{"use_responses_api": true, "openai_passthrough": passthrough}}, - {ID: 2, Name: "next-account", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey, - Status: service.StatusActive, Schedulable: true, Priority: 1, - Credentials: map[string]any{"api_key": "other-fixture", "base_url": "https://other.example.test"}, - Extra: map[string]any{"use_responses_api": true}}, - } - handler := newOpenAIResponsesFailoverTestHandler(t, upstream, accounts...) - body := []byte(`{"model":"gpt-5.1","stream":true,"input":[{"type":"reasoning","encrypted_content":"private-old-cipher","summary":[]},{"type":"function_call","call_id":"call-1","name":"probe","arguments":"{}"},{"type":"function_call_output","call_id":"call-1","output":"private-tool-output"}]}`) - authContext, _ := newOpenAIResponsesFailoverTestContextWithBody(t, nil, body) - ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) - router := gin.New() - router.Use(func(c *gin.Context) { - for key, value := range authContext.Keys { - c.Set(key, value) - } - c.Next() - }, OpsErrorLoggerMiddleware(ops)) - router.POST("/v1/responses", handler.Responses) - recorder := httptest.NewRecorder() - request := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) - request.Header.Set("Content-Type", "application/json") - router.ServeHTTP(recorder, request) - - require.Equal(t, failure.accountIDs, upstream.accountIDs, "no third POST or same-account scheduler retry on the first account") - require.True(t, gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").Exists()) - require.False(t, gjson.GetBytes(upstream.bodies[1], "input.0.encrypted_content").Exists()) - require.Equal(t, gjson.GetBytes(upstream.bodies[0], "input.1").Raw, gjson.GetBytes(upstream.bodies[1], "input.1").Raw) - require.Equal(t, gjson.GetBytes(upstream.bodies[0], "input.2").Raw, gjson.GetBytes(upstream.bodies[1], "input.2").Raw) - require.NotContains(t, recorder.Body.String(), "do-not-leak") - require.NotContains(t, recorder.Body.String(), "response.completed") - if failure.clientStatus == http.StatusBadRequest { - require.Equal(t, 1, strings.Count(recorder.Body.String(), "event: response.failed\n")) - require.Contains(t, recorder.Body.String(), service.OpenAIRequestRejectedCode) - } - require.Equal(t, int64(1), OpsErrorLogQueueLength()) - job := <-opsErrorLogQueue - require.Equal(t, failure.clientStatus, job.entry.StatusCode) - require.NotNil(t, job.entry.UpstreamStatusCode) - require.Equal(t, failure.status, *job.entry.UpstreamStatusCode) - // The minimal router fixture intentionally disables the optional - // advanced scheduler. Exercise the terminal finalizer with an - // observable reporter instead of treating inert metrics as evidence. - // A provider failure still ends as a terminal when the attempt did - // not classify it, so both classifications are probed directly. - reporter := &recordingFailoverSelectionReporter{} - classified := &service.UpstreamFailoverError{StatusCode: failure.status} - if failure.clientStatus == http.StatusBadRequest { - classified = service.NewOpenAIRequestRejectedError(failure.status, nil) - } - probe, _ := newOpenAIResponsesFailoverTestContextWithStream(t, nil, true) - terminal := &service.OpenAIReasoningRecoveryTerminalError{Failure: classified} - require.True(t, handler.handleReasoningRecoveryTerminal(probe, terminal, - &service.AccountSelectionResult{Account: &accounts[0]}, &accounts[0], "gpt-5.1", true, reporter)) - require.Equal(t, failure.healthEvents, reporter.reportedFailures, "request rejection must not emit account-health failure feedback") - require.Equal(t, 1-failure.healthEvents, reporter.releasedProbes) - require.Zero(t, reporter.sameAccountRetries) - }) - } - } -} - -type recordingFailoverSelectionReporter struct { - sameAccountRetries int - reportedFailures int - releasedProbes int -} - -func (r *recordingFailoverSelectionReporter) ReportOpenAIAccountSameAccountRetry( - _ *service.AccountSelectionResult, _ int64, _ string, -) { - r.sameAccountRetries++ -} - -func (r *recordingFailoverSelectionReporter) ReportOpenAIAccountScheduleResultForSelection( - _ *service.AccountSelectionResult, _ int64, _ string, success bool, _ *int, -) { - if !success { - r.reportedFailures++ - } -} - -func (r *recordingFailoverSelectionReporter) ReleaseOpenAIRuntimeBreakerProbeForSelection( - _ *service.AccountSelectionResult, -) { - r.releasedProbes++ -} diff --git a/backend/internal/handler/wire.go b/backend/internal/handler/wire.go index 2f027c6003d9..cbe4e7e5a931 100644 --- a/backend/internal/handler/wire.go +++ b/backend/internal/handler/wire.go @@ -49,7 +49,6 @@ func ProvideAdminHandlers( complianceHandler *admin.ComplianceHandler, auditLogHandler *admin.AuditLogHandler, systemPromptHandler *admin.SystemPromptHandler, - reasoningRecoveryHandler *admin.ReasoningRecoveryHandler, codexGatewayBorrowHandler *admin.CodexGatewayBorrowHandler, pelicanTestHandler *admin.PelicanTestHandler, upstreamBillingProbe *service.UpstreamBillingProbeService, @@ -100,7 +99,6 @@ func ProvideAdminHandlers( Compliance: complianceHandler, AuditLog: auditLogHandler, SystemPrompt: systemPromptHandler, - ReasoningRecovery: reasoningRecoveryHandler, CodexGatewayBorrow: codexGatewayBorrowHandler, PelicanTest: pelicanTestHandler, } @@ -302,7 +300,6 @@ var ProviderSet = wire.NewSet( admin.NewComplianceHandler, admin.NewAuditLogHandler, admin.NewSystemPromptHandler, - admin.NewReasoningRecoveryHandler, admin.NewCodexGatewayBorrowHandler, admin.NewPelicanTestHandler, ProvideAccountJobRuntime, diff --git a/backend/internal/model/error_passthrough_rule.go b/backend/internal/model/error_passthrough_rule.go index da928125f29b..c4ca164dc08a 100644 --- a/backend/internal/model/error_passthrough_rule.go +++ b/backend/internal/model/error_passthrough_rule.go @@ -49,21 +49,9 @@ const ( PlatformTypeSafe = domain.PlatformTypeSafe ) -// AllPlatforms 返回所有支持的平台列表 +// AllPlatforms 返回所有支持的平台列表(平台清单,按展示顺序)。 func AllPlatforms() []string { - return []string{ - PlatformAnthropic, - PlatformOpenAI, - PlatformGemini, - PlatformAntigravity, - PlatformGrok, - PlatformKimi, - PlatformZhipu, - PlatformDeepseek, - PlatformMiniMax, - PlatformOpenCodeGo, - PlatformTypeSafe, - } + return domain.ConcretePlatformIDs() } // Validate 验证规则配置的有效性 diff --git a/backend/internal/model/error_passthrough_rule_test.go b/backend/internal/model/error_passthrough_rule_test.go index 17e6fdcbf281..9b66d712432c 100644 --- a/backend/internal/model/error_passthrough_rule_test.go +++ b/backend/internal/model/error_passthrough_rule_test.go @@ -19,5 +19,7 @@ func TestAllPlatformsIncludesEveryConcretePlatform(t *testing.T) { "minimax", "opencode_go", "typesafe", + "command_code", + "cline", }, AllPlatforms()) } diff --git a/backend/internal/pkg/antigravity/claude_types.go b/backend/internal/pkg/antigravity/claude_types.go index 23601c045726..61e32b615db2 100644 --- a/backend/internal/pkg/antigravity/claude_types.go +++ b/backend/internal/pkg/antigravity/claude_types.go @@ -122,6 +122,20 @@ type ClaudeContentItem struct { Input any `json:"input,omitempty"` } +// MarshalJSON 保证 thinking 块始终带 thinking/signature 字段(即使为空), +// 与 Anthropic 官方一致;Grok Build 等严格客户端缺少 signature 会解析失败。 +func (c ClaudeContentItem) MarshalJSON() ([]byte, error) { + type claudeContentItem ClaudeContentItem + if c.Type != "thinking" { + return json.Marshal(claudeContentItem(c)) + } + return json.Marshal(struct { + Thinking string `json:"thinking"` + Signature string `json:"signature"` + claudeContentItem + }{Thinking: c.Thinking, Signature: c.Signature, claudeContentItem: claudeContentItem(c)}) +} + // ClaudeUsage Claude 用量统计 type ClaudeUsage struct { InputTokens int `json:"input_tokens"` diff --git a/backend/internal/pkg/antigravity/claude_types_test.go b/backend/internal/pkg/antigravity/claude_types_test.go index 4b49fdb9b10f..1aa5bca31277 100644 --- a/backend/internal/pkg/antigravity/claude_types_test.go +++ b/backend/internal/pkg/antigravity/claude_types_test.go @@ -1,6 +1,9 @@ package antigravity -import "testing" +import ( + "encoding/json" + "testing" +) func TestDefaultModels_ContainsNewAndLegacyImageModels(t *testing.T) { t.Parallel() @@ -44,3 +47,23 @@ func TestDefaultModels_ContainsNewAndLegacyImageModels(t *testing.T) { } } } + +func TestClaudeContentItem_ThinkingAlwaysHasSignature(t *testing.T) { + t.Parallel() + + data, err := json.Marshal(ClaudeContentItem{Type: "thinking", Thinking: "plan"}) + if err != nil { + t.Fatal(err) + } + if got, want := string(data), `{"thinking":"plan","signature":"","type":"thinking"}`; got != want { + t.Fatalf("thinking block = %s, want %s", got, want) + } + + data, err = json.Marshal(ClaudeContentItem{Type: "text", Text: "hi"}) + if err != nil { + t.Fatal(err) + } + if got, want := string(data), `{"type":"text","text":"hi"}`; got != want { + t.Fatalf("text block = %s, want %s", got, want) + } +} diff --git a/backend/internal/pkg/antigravity/stream_transformer.go b/backend/internal/pkg/antigravity/stream_transformer.go index 13471ddddc51..faecf233ecac 100644 --- a/backend/internal/pkg/antigravity/stream_transformer.go +++ b/backend/internal/pkg/antigravity/stream_transformer.go @@ -296,8 +296,9 @@ func (p *StreamingProcessor) processThinking(text, signature string) []byte { // 开始或继续 thinking 块 if p.blockType != BlockTypeThinking { _, _ = result.Write(p.startBlock(BlockTypeThinking, map[string]any{ - "type": "thinking", - "thinking": "", + "type": "thinking", + "thinking": "", + "signature": "", })) } @@ -475,8 +476,9 @@ func (p *StreamingProcessor) emitEmptyThinkingWithSignature(signature string) [] var result bytes.Buffer _, _ = result.Write(p.startBlock(BlockTypeThinking, map[string]any{ - "type": "thinking", - "thinking": "", + "type": "thinking", + "thinking": "", + "signature": "", })) _, _ = result.Write(p.emitDelta("thinking_delta", map[string]any{ "thinking": "", diff --git a/backend/internal/pkg/apicompat/anthropic_responses_test.go b/backend/internal/pkg/apicompat/anthropic_responses_test.go index 9dd586ad7841..bbfd9d282b4c 100644 --- a/backend/internal/pkg/apicompat/anthropic_responses_test.go +++ b/backend/internal/pkg/apicompat/anthropic_responses_test.go @@ -848,7 +848,7 @@ func TestStreamingReasoning(t *testing.T) { sse, err := ResponsesAnthropicEventToSSE(events[0]) require.NoError(t, err) - assert.Contains(t, sse, `"content_block":{"thinking":"","type":"thinking"}`) + assert.Contains(t, sse, `"content_block":{"thinking":"","signature":"","type":"thinking"}`) // reasoning text delta events = ResponsesEventToAnthropicEvents(&ResponsesStreamEvent{ diff --git a/backend/internal/pkg/apicompat/chatcompletions_to_responses.go b/backend/internal/pkg/apicompat/chatcompletions_to_responses.go index 7fc48cbc2705..5f6269eb8f00 100644 --- a/backend/internal/pkg/apicompat/chatcompletions_to_responses.go +++ b/backend/internal/pkg/apicompat/chatcompletions_to_responses.go @@ -93,10 +93,11 @@ func ChatCompletionsToResponses(req *ChatCompletionsRequest) (*ResponsesRequest, out.Tools = convertChatToolsToResponses(req.Tools, req.Functions) } - // tool_choice: already compatible format — pass through directly. + // tool_choice: strings and Responses-shaped objects pass through; a named + // Chat choice nests the name under "function" and must be flattened. // Legacy function_call needs mapping. if len(req.ToolChoice) > 0 { - out.ToolChoice = req.ToolChoice + out.ToolChoice = convertChatToolChoiceToResponses(req.ToolChoice) } else if len(req.FunctionCall) > 0 { tc, err := convertChatFunctionCallToToolChoice(req.FunctionCall) if err != nil { @@ -112,7 +113,18 @@ func ChatCompletionsToResponses(req *ChatCompletionsRequest) (*ResponsesRequest, // array into a Responses API input items array. func convertChatMessagesToResponsesInput(msgs []ChatMessage) ([]ResponsesInputItem, error) { var out []ResponsesInputItem + legacyIDs := legacyFunctionCallIDs{msgs: msgs} for _, m := range msgs { + switch { + case m.Role == "assistant" && m.FunctionCall != nil && len(m.ToolCalls) == 0: + m.ToolCalls = []ChatToolCall{{ + ID: legacyIDs.assign(m.FunctionCall.Name), + Type: "function", + Function: *m.FunctionCall, + }} + case m.Role == "function" && m.ToolCallID == "": + m.ToolCallID = legacyIDs.claim(m.Name) + } items, err := chatMessageToResponsesItems(m) if err != nil { return nil, err @@ -122,11 +134,63 @@ func convertChatMessagesToResponsesInput(msgs []ChatMessage) ([]ResponsesInputIt return out, nil } +// legacyFunctionCallIDs pairs legacy function calls with their results. Legacy +// function calling carries no call IDs: the assistant turn has function_call +// and the role=function result refers to it only by name, while the Responses +// API pairs function_call and function_call_output by call_id. IDs follow +// message order, so a replayed history converts to the same input, and skip +// IDs already used by tool_calls in the same conversation. +type legacyFunctionCallIDs struct { + msgs []ChatMessage + used map[string]bool + next int + pending map[string][]string +} + +// assign returns a new call ID for a legacy function call and queues it for +// the next result with the same function name. +func (ids *legacyFunctionCallIDs) assign(name string) string { + if ids.used == nil { + ids.used = make(map[string]bool) + ids.pending = make(map[string][]string) + for _, m := range ids.msgs { + for _, tc := range m.ToolCalls { + ids.used[tc.ID] = true + } + if m.ToolCallID != "" { + ids.used[m.ToolCallID] = true + } + } + } + var id string + for { + ids.next++ + id = fmt.Sprintf("call_legacy_%d", ids.next) + if !ids.used[id] { + break + } + } + ids.used[id] = true + ids.pending[name] = append(ids.pending[name], id) + return id +} + +// claim returns the oldest unanswered call ID for name, or "" when no legacy +// call with that name precedes the result. +func (ids *legacyFunctionCallIDs) claim(name string) string { + queue := ids.pending[name] + if len(queue) == 0 { + return "" + } + ids.pending[name] = queue[1:] + return queue[0] +} + // chatMessageToResponsesItems converts a single ChatMessage into one or more // ResponsesInputItem values. func chatMessageToResponsesItems(m ChatMessage) ([]ResponsesInputItem, error) { switch m.Role { - case "system": + case "system", "developer": return chatSystemToResponses(m) case "user": return chatUserToResponses(m) @@ -141,7 +205,9 @@ func chatMessageToResponsesItems(m ChatMessage) ([]ResponsesInputItem, error) { } } -// chatSystemToResponses converts a system message. +// chatSystemToResponses converts a system or developer message, keeping its +// role: the Responses API accepts both, and developer must not be demoted to +// a user turn. func chatSystemToResponses(m ChatMessage) ([]ResponsesInputItem, error) { parsed, err := parseChatMessageContent(m.Content) if err != nil { @@ -151,7 +217,7 @@ func chatSystemToResponses(m ChatMessage) ([]ResponsesInputItem, error) { if err != nil { return nil, err } - return []ResponsesInputItem{{Type: "message", Role: "system", Content: content}}, nil + return []ResponsesInputItem{{Type: "message", Role: m.Role, Content: content}}, nil } // chatUserToResponses converts a user message, handling both plain strings and @@ -308,8 +374,9 @@ func chatToolToResponses(m ChatMessage) ([]ResponsesInputItem, error) { } // chatFunctionToResponses converts a legacy function result message -// (role=function) into a function_call_output item. The Name field is used as -// call_id since legacy function calls do not carry a separate call_id. +// (role=function) into a function_call_output item. ToolCallID holds the ID +// assigned to the matching legacy call (see legacyFunctionCallIDs); without a +// matching call the Name field is used as call_id. func chatFunctionToResponses(m ChatMessage) ([]ResponsesInputItem, error) { output, err := parseChatContent(m.Content) if err != nil { @@ -318,9 +385,13 @@ func chatFunctionToResponses(m ChatMessage) ([]ResponsesInputItem, error) { if output == "" { output = "(empty)" } + callID := m.ToolCallID + if callID == "" { + callID = m.Name + } return []ResponsesInputItem{{ Type: "function_call_output", - CallID: m.Name, + CallID: callID, Output: output, }}, nil } @@ -495,6 +566,34 @@ func defaultStrictFalse(src *bool) *bool { return src } +// convertChatToolChoiceToResponses maps a Chat Completions tool_choice to the +// Responses API shape. +// +// {"type":"function","function":{"name":"X"}} → {"type":"function","name":"X"} +// +// Strings ("auto", "none", "required") and objects that already use the +// Responses shape are returned unchanged. +func convertChatToolChoiceToResponses(raw json.RawMessage) json.RawMessage { + var choice struct { + Type string `json:"type"` + Function *struct { + Name string `json:"name"` + } `json:"function"` + } + if err := json.Unmarshal(raw, &choice); err != nil || choice.Type != "function" || choice.Function == nil { + return raw + } + name := strings.TrimSpace(choice.Function.Name) + if name == "" { + return raw + } + flat, err := json.Marshal(map[string]string{"type": "function", "name": name}) + if err != nil { + return raw + } + return flat +} + // convertChatFunctionCallToToolChoice maps the legacy function_call field to a // Responses API tool_choice value. // diff --git a/backend/internal/pkg/apicompat/chatcompletions_to_responses_developer_role_test.go b/backend/internal/pkg/apicompat/chatcompletions_to_responses_developer_role_test.go new file mode 100644 index 000000000000..d6fbcad6cc92 --- /dev/null +++ b/backend/internal/pkg/apicompat/chatcompletions_to_responses_developer_role_test.go @@ -0,0 +1,43 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// The Responses API accepts role "developer" in input messages. Converting it +// to "user" silently demotes the caller's instructions to user turns. +func TestChatCompletionsToResponses_DeveloperRolePreserved(t *testing.T) { + tests := []struct { + name string + content string + wantContent string + }{ + {name: "string content", content: `"Answer in French."`, wantContent: `"Answer in French."`}, + {name: "text parts", content: `[{"type":"text","text":"Answer in French."}]`, wantContent: `[{"type":"input_text","text":"Answer in French."}]`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &ChatCompletionsRequest{ + Model: "gpt-4o", + Messages: []ChatMessage{ + {Role: "developer", Content: json.RawMessage(tt.content)}, + {Role: "user", Content: json.RawMessage(`"Hello"`)}, + }, + } + + resp, err := ChatCompletionsToResponses(req) + require.NoError(t, err) + + var items []ResponsesInputItem + require.NoError(t, json.Unmarshal(resp.Input, &items)) + require.Len(t, items, 2) + require.Equal(t, "message", items[0].Type) + require.Equal(t, "developer", items[0].Role) + require.JSONEq(t, tt.wantContent, string(items[0].Content)) + require.Equal(t, "user", items[1].Role) + }) + } +} diff --git a/backend/internal/pkg/apicompat/chatcompletions_to_responses_legacy_function_call_test.go b/backend/internal/pkg/apicompat/chatcompletions_to_responses_legacy_function_call_test.go new file mode 100644 index 000000000000..1c971fd087f5 --- /dev/null +++ b/backend/internal/pkg/apicompat/chatcompletions_to_responses_legacy_function_call_test.go @@ -0,0 +1,87 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func convertLegacyFunctionHistory(t *testing.T, msgs []ChatMessage) []ResponsesInputItem { + t.Helper() + resp, err := ChatCompletionsToResponses(&ChatCompletionsRequest{Model: "gpt-4o", Messages: msgs}) + require.NoError(t, err) + var items []ResponsesInputItem + require.NoError(t, json.Unmarshal(resp.Input, &items)) + return items +} + +// Legacy function calling has no call IDs: the assistant turn carries +// function_call and the result is a role=function message keyed by name. The +// Responses API needs a function_call item and a function_call_output with the +// same call_id, otherwise the output is rejected as having no matching call. +func TestChatCompletionsToResponses_LegacyFunctionCallHistoryPaired(t *testing.T) { + msgs := []ChatMessage{ + {Role: "user", Content: json.RawMessage(`"Weather in Paris and Rome?"`)}, + {Role: "assistant", Content: json.RawMessage(`"Checking Paris."`), FunctionCall: &ChatFunctionCall{Name: "get_weather", Arguments: `{"city":"Paris"}`}}, + {Role: "function", Name: "get_weather", Content: json.RawMessage(`"sunny"`)}, + {Role: "assistant", FunctionCall: &ChatFunctionCall{Name: "get_weather", Arguments: `{"city":"Rome"}`}}, + {Role: "function", Name: "get_weather", Content: json.RawMessage(`"rainy"`)}, + } + + items := convertLegacyFunctionHistory(t, msgs) + require.Len(t, items, 6) + + require.Equal(t, "user", items[0].Role) + require.Equal(t, "assistant", items[1].Role) + + firstCall, firstOutput := items[2], items[3] + require.Equal(t, "function_call", firstCall.Type) + require.Equal(t, "get_weather", firstCall.Name) + require.Equal(t, `{"city":"Paris"}`, firstCall.Arguments) + require.Equal(t, "function_call_output", firstOutput.Type) + require.Equal(t, "sunny", firstOutput.Output) + require.NotEmpty(t, firstCall.CallID) + require.Equal(t, firstCall.CallID, firstOutput.CallID) + + secondCall, secondOutput := items[4], items[5] + require.Equal(t, "function_call", secondCall.Type) + require.Equal(t, `{"city":"Rome"}`, secondCall.Arguments) + require.Equal(t, "function_call_output", secondOutput.Type) + require.Equal(t, "rainy", secondOutput.Output) + require.Equal(t, secondCall.CallID, secondOutput.CallID) + require.NotEqual(t, firstCall.CallID, secondCall.CallID) + + // Synthesized IDs must be stable across turns so the replayed history keeps + // the same prompt prefix. + require.Equal(t, items, convertLegacyFunctionHistory(t, msgs)) +} + +func TestChatCompletionsToResponses_LegacyFunctionCallIDAvoidsToolCallIDs(t *testing.T) { + items := convertLegacyFunctionHistory(t, []ChatMessage{ + {Role: "user", Content: json.RawMessage(`"hi"`)}, + {Role: "assistant", FunctionCall: &ChatFunctionCall{Name: "lookup", Arguments: `{}`}}, + {Role: "function", Name: "lookup", Content: json.RawMessage(`"legacy result"`)}, + {Role: "assistant", ToolCalls: []ChatToolCall{{ID: "call_legacy_1", Type: "function", Function: ChatFunctionCall{Name: "lookup", Arguments: `{}`}}}}, + {Role: "tool", ToolCallID: "call_legacy_1", Content: json.RawMessage(`"tool result"`)}, + }) + require.Len(t, items, 5) + + legacyCall, legacyOutput := items[1], items[2] + require.Equal(t, "function_call", legacyCall.Type) + require.Equal(t, legacyCall.CallID, legacyOutput.CallID) + require.NotEqual(t, "call_legacy_1", legacyCall.CallID) + + require.Equal(t, "call_legacy_1", items[3].CallID) + require.Equal(t, "call_legacy_1", items[4].CallID) +} + +func TestChatCompletionsToResponses_LegacyFunctionResultWithoutCallKeepsName(t *testing.T) { + items := convertLegacyFunctionHistory(t, []ChatMessage{ + {Role: "user", Content: json.RawMessage(`"hi"`)}, + {Role: "function", Name: "lookup", Content: json.RawMessage(`"orphan"`)}, + }) + require.Len(t, items, 2) + require.Equal(t, "function_call_output", items[1].Type) + require.Equal(t, "lookup", items[1].CallID) +} diff --git a/backend/internal/pkg/apicompat/chatcompletions_to_responses_tool_choice_test.go b/backend/internal/pkg/apicompat/chatcompletions_to_responses_tool_choice_test.go new file mode 100644 index 000000000000..736a0b5acf67 --- /dev/null +++ b/backend/internal/pkg/apicompat/chatcompletions_to_responses_tool_choice_test.go @@ -0,0 +1,43 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// Chat Completions nests a named choice under "function"; the Responses API +// expects the name at the top level and rejects the nested shape. +func TestChatCompletionsToResponses_ToolChoiceShape(t *testing.T) { + tests := []struct { + name string + choice string + want string + }{ + {name: "named chat function is flattened", choice: `{"type":"function","function":{"name":"get_weather"}}`, want: `{"type":"function","name":"get_weather"}`}, + {name: "flat responses function is kept", choice: `{"type":"function","name":"get_weather"}`, want: `{"type":"function","name":"get_weather"}`}, + {name: "auto string is kept", choice: `"auto"`, want: `"auto"`}, + {name: "required string is kept", choice: `"required"`, want: `"required"`}, + {name: "none string is kept", choice: `"none"`, want: `"none"`}, + {name: "allowed_tools policy is kept", choice: `{"type":"allowed_tools","mode":"auto","tools":[{"type":"function","name":"get_weather"}]}`, want: `{"type":"allowed_tools","mode":"auto","tools":[{"type":"function","name":"get_weather"}]}`}, + {name: "function without name is kept", choice: `{"type":"function","function":{}}`, want: `{"type":"function","function":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &ChatCompletionsRequest{ + Model: "gpt-4o", + Messages: []ChatMessage{{Role: "user", Content: json.RawMessage(`"weather?"`)}}, + Tools: []ChatTool{{ + Type: "function", + Function: &ChatFunction{Name: "get_weather", Parameters: json.RawMessage(`{"type":"object"}`)}, + }}, + ToolChoice: json.RawMessage(tt.choice), + } + + resp, err := ChatCompletionsToResponses(req) + require.NoError(t, err) + require.JSONEq(t, tt.want, string(resp.ToolChoice)) + }) + } +} diff --git a/backend/internal/pkg/apicompat/responses_to_chatcompletions.go b/backend/internal/pkg/apicompat/responses_to_chatcompletions.go index 78aadfd495bb..d1a7c5d5cee7 100644 --- a/backend/internal/pkg/apicompat/responses_to_chatcompletions.go +++ b/backend/internal/pkg/apicompat/responses_to_chatcompletions.go @@ -16,7 +16,8 @@ import ( // ResponsesToChatCompletions converts a Responses API response into a Chat // Completions response. Text output items are concatenated into -// choices[0].message.content; function_call items become tool_calls. +// choices[0].message.content, refusal parts into choices[0].message.refusal; +// function_call items become tool_calls. func ResponsesToChatCompletions(resp *ResponsesResponse, model string) *ChatCompletionsResponse { id := resp.ID if id == "" { @@ -32,6 +33,7 @@ func ResponsesToChatCompletions(resp *ResponsesResponse, model string) *ChatComp } var contentText string + var refusalText string var reasoningText string var toolCalls []ChatToolCall @@ -39,8 +41,11 @@ func ResponsesToChatCompletions(resp *ResponsesResponse, model string) *ChatComp switch item.Type { case "message": for _, part := range item.Content { - if part.Type == "output_text" && part.Text != "" { + switch { + case part.Type == "output_text" && part.Text != "": contentText += part.Text + case part.Type == "refusal" && part.Refusal != "": + refusalText += part.Refusal } } case "function_call": @@ -63,7 +68,7 @@ func ResponsesToChatCompletions(resp *ResponsesResponse, model string) *ChatComp } } - msg := ChatMessage{Role: "assistant"} + msg := ChatMessage{Role: "assistant", Refusal: refusalText} if len(toolCalls) > 0 { msg.ToolCalls = toolCalls } @@ -154,6 +159,8 @@ func ResponsesEventToChatChunks(evt *ResponsesStreamEvent, state *ResponsesEvent return resToChatHandleCreated(evt, state) case "response.output_text.delta": return resToChatHandleTextDelta(evt, state) + case "response.refusal.delta": + return resToChatHandleRefusalDelta(evt, state) case "response.output_item.added": return resToChatHandleOutputItemAdded(evt, state) case "response.output_item.done": @@ -256,6 +263,14 @@ func resToChatHandleTextDelta(evt *ResponsesStreamEvent, state *ResponsesEventTo return []ChatCompletionsChunk{makeChatDeltaChunk(state, ChatDelta{Content: &content})} } +func resToChatHandleRefusalDelta(evt *ResponsesStreamEvent, state *ResponsesEventToChatState) []ChatCompletionsChunk { + if evt.Delta == "" { + return nil + } + refusal := evt.Delta + return []ChatCompletionsChunk{makeChatDeltaChunk(state, ChatDelta{Refusal: &refusal})} +} + func resToChatHandleOutputItemAdded(evt *ResponsesStreamEvent, state *ResponsesEventToChatState) []ChatCompletionsChunk { if evt.Item != nil && evt.Item.Type == "function_call" { return resToChatFunctionItem(evt.Item, evt.OutputIndex, false, state) @@ -546,6 +561,7 @@ type bufferedFuncCall struct { // (response.completed / response.done) carries an empty output array. type BufferedResponseAccumulator struct { text strings.Builder + refusal strings.Builder reasoning strings.Builder funcCalls []bufferedFuncCall outputIndexToFuncIdx map[int]int @@ -567,6 +583,10 @@ func (a *BufferedResponseAccumulator) ProcessEvent(event *ResponsesStreamEvent) if event.Delta != "" { _, _ = a.text.WriteString(event.Delta) } + case "response.refusal.delta": + if event.Delta != "" { + _, _ = a.refusal.WriteString(event.Delta) + } case "response.output_item.added": if event.Item != nil && (event.Item.Type == "function_call" || event.Item.Type == "custom_tool_call") { idx := len(a.funcCalls) @@ -605,7 +625,7 @@ func (a *BufferedResponseAccumulator) ProcessEvent(event *ResponsesStreamEvent) // HasContent reports whether any content has been accumulated. func (a *BufferedResponseAccumulator) HasContent() bool { - return a.text.Len() > 0 || len(a.funcCalls) > 0 || a.reasoning.Len() > 0 + return a.text.Len() > 0 || a.refusal.Len() > 0 || len(a.funcCalls) > 0 || a.reasoning.Len() > 0 } // BuildOutput constructs a []ResponsesOutput from the accumulated delta @@ -624,14 +644,18 @@ func (a *BufferedResponseAccumulator) BuildOutput() []ResponsesOutput { }) } - if a.text.Len() > 0 { + if a.text.Len() > 0 || a.refusal.Len() > 0 { + var content []ResponsesContentPart + if a.text.Len() > 0 { + content = append(content, ResponsesContentPart{Type: "output_text", Text: a.text.String()}) + } + if a.refusal.Len() > 0 { + content = append(content, ResponsesContentPart{Type: "refusal", Refusal: a.refusal.String()}) + } out = append(out, ResponsesOutput{ - Type: "message", - Role: "assistant", - Content: []ResponsesContentPart{{ - Type: "output_text", - Text: a.text.String(), - }}, + Type: "message", + Role: "assistant", + Content: content, }) } diff --git a/backend/internal/pkg/apicompat/responses_to_chatcompletions_refusal_test.go b/backend/internal/pkg/apicompat/responses_to_chatcompletions_refusal_test.go new file mode 100644 index 000000000000..cf8f0bfa4aa4 --- /dev/null +++ b/backend/internal/pkg/apicompat/responses_to_chatcompletions_refusal_test.go @@ -0,0 +1,114 @@ +package apicompat + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const refusalText = "I can't help with that." + +func decodeResponsesEvent(t *testing.T, raw string) ResponsesStreamEvent { + t.Helper() + var evt ResponsesStreamEvent + require.NoError(t, json.Unmarshal([]byte(raw), &evt)) + return evt +} + +func chatMessageJSON(t *testing.T, resp *ChatCompletionsResponse) map[string]any { + t.Helper() + raw, err := json.Marshal(resp) + require.NoError(t, err) + var decoded struct { + Choices []struct { + Message map[string]any `json:"message"` + } `json:"choices"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Len(t, decoded.Choices, 1) + return decoded.Choices[0].Message +} + +// A refusal arrives as a "refusal" content part, not output_text. Dropping it +// leaves the Chat client with an empty assistant message. +func TestResponsesToChatCompletions_RefusalPreserved(t *testing.T) { + var resp ResponsesResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "id":"resp_refusal", + "status":"completed", + "output":[{"type":"message","role":"assistant","content":[{"type":"refusal","refusal":"`+refusalText+`"}]}] + }`), &resp)) + + chat := ResponsesToChatCompletions(&resp, "gpt-4o") + + msg := chatMessageJSON(t, chat) + require.Equal(t, refusalText, msg["refusal"]) + require.NotContains(t, msg, "content") + require.Equal(t, "stop", chat.Choices[0].FinishReason) +} + +func TestResponsesEventToChatChunks_RefusalDelta(t *testing.T) { + state := NewResponsesEventToChatState() + events := []string{ + `{"type":"response.created","response":{"id":"resp_refusal","model":"gpt-4o"}}`, + `{"type":"response.output_item.added","output_index":0,"item":{"type":"message","role":"assistant"}}`, + `{"type":"response.refusal.delta","output_index":0,"content_index":0,"delta":"I can't "}`, + `{"type":"response.refusal.delta","output_index":0,"content_index":0,"delta":"help with that."}`, + `{"type":"response.refusal.done","output_index":0,"content_index":0,"refusal":"` + refusalText + `"}`, + `{"type":"response.completed","response":{"id":"resp_refusal","status":"completed"}}`, + } + + var refusal strings.Builder + var finishReason string + for _, raw := range events { + evt := decodeResponsesEvent(t, raw) + for _, chunk := range ResponsesEventToChatChunks(&evt, state) { + encoded, err := json.Marshal(chunk) + require.NoError(t, err) + var decoded struct { + Choices []struct { + Delta struct { + Refusal *string `json:"refusal"` + } `json:"delta"` + FinishReason *string `json:"finish_reason"` + } `json:"choices"` + } + require.NoError(t, json.Unmarshal(encoded, &decoded)) + for _, choice := range decoded.Choices { + if choice.Delta.Refusal != nil { + _, _ = refusal.WriteString(*choice.Delta.Refusal) + } + if choice.FinishReason != nil { + finishReason = *choice.FinishReason + } + } + } + } + + require.Equal(t, refusalText, refusal.String()) + require.Equal(t, "stop", finishReason) +} + +// Non-streaming Chat requests are served from a forced upstream stream; when +// the terminal event carries an empty output array the message is rebuilt from +// the accumulated deltas, which must include refusal deltas. +func TestBufferedResponseAccumulator_RefusalRebuiltFromDeltas(t *testing.T) { + acc := NewBufferedResponseAccumulator() + for _, raw := range []string{ + `{"type":"response.output_item.added","output_index":0,"item":{"type":"message","role":"assistant"}}`, + `{"type":"response.refusal.delta","output_index":0,"content_index":0,"delta":"I can't "}`, + `{"type":"response.refusal.delta","output_index":0,"content_index":0,"delta":"help with that."}`, + } { + evt := decodeResponsesEvent(t, raw) + acc.ProcessEvent(&evt) + } + require.True(t, acc.HasContent()) + + final := &ResponsesResponse{ID: "resp_refusal", Status: "completed"} + acc.SupplementResponseOutput(final) + chat := ResponsesToChatCompletions(final, "gpt-4o") + + require.Equal(t, refusalText, chatMessageJSON(t, chat)["refusal"]) +} diff --git a/backend/internal/pkg/apicompat/thinking_signature_wire_test.go b/backend/internal/pkg/apicompat/thinking_signature_wire_test.go new file mode 100644 index 000000000000..33924fd4989f --- /dev/null +++ b/backend/internal/pkg/apicompat/thinking_signature_wire_test.go @@ -0,0 +1,24 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// Thinking blocks must always carry the `signature` key (Anthropic wire shape); +// strict clients such as Grok Build fail with "missing field `signature`". +func TestAnthropicContentBlock_ThinkingAlwaysHasSignature(t *testing.T) { + data, err := json.Marshal(AnthropicContentBlock{Type: "thinking", Thinking: "plan"}) + require.NoError(t, err) + require.JSONEq(t, `{"type":"thinking","thinking":"plan","signature":""}`, string(data)) + + data, err = json.Marshal(AnthropicContentBlock{Type: "thinking", Signature: "sig"}) + require.NoError(t, err) + require.JSONEq(t, `{"type":"thinking","thinking":"","signature":"sig"}`, string(data)) + + data, err = json.Marshal(AnthropicContentBlock{Type: "text", Text: "hi"}) + require.NoError(t, err) + require.JSONEq(t, `{"type":"text","text":"hi"}`, string(data)) +} diff --git a/backend/internal/pkg/apicompat/types.go b/backend/internal/pkg/apicompat/types.go index fb6433c964a8..f02b3fcbe37c 100644 --- a/backend/internal/pkg/apicompat/types.go +++ b/backend/internal/pkg/apicompat/types.go @@ -95,10 +95,14 @@ func (b AnthropicContentBlock) MarshalJSON() ([]byte, error) { anthropicContentBlock }{Text: b.Text, anthropicContentBlock: anthropicContentBlock(b)}) case "thinking": + // Anthropic always sends `signature` on thinking blocks (empty on + // content_block_start); strict clients such as Grok Build reject the + // block with "missing field `signature`" when the key is absent. return json.Marshal(struct { - Thinking string `json:"thinking"` + Thinking string `json:"thinking"` + Signature string `json:"signature"` anthropicContentBlock - }{Thinking: b.Thinking, anthropicContentBlock: anthropicContentBlock(b)}) + }{Thinking: b.Thinking, Signature: b.Signature, anthropicContentBlock: anthropicContentBlock(b)}) default: return json.Marshal(base) } @@ -338,8 +342,9 @@ func (i *ResponsesInputItem) UnmarshalJSON(data []byte) error { // ResponsesContentPart is a typed content part in a Responses message. type ResponsesContentPart struct { PromptCacheBreakpoint json.RawMessage `json:"prompt_cache_breakpoint,omitempty"` - Type string `json:"type"` // "input_text" | "output_text" | "input_image" | "input_file" + Type string `json:"type"` // "input_text" | "output_text" | "refusal" | "input_image" | "input_file" Text string `json:"text,omitempty"` + Refusal string `json:"refusal,omitempty"` // type=refusal ImageURL string `json:"image_url,omitempty"` // data URI for input_image // input_file fields. @@ -718,6 +723,7 @@ type ChatStreamOptions struct { type ChatMessage struct { Role string `json:"role"` // "system" | "user" | "assistant" | "tool" | "function" Content json.RawMessage `json:"content,omitempty"` + Refusal string `json:"refusal,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"` Reasoning string `json:"reasoning,omitempty"` Name string `json:"name,omitempty"` @@ -858,6 +864,7 @@ type ChatChunkChoice struct { type ChatDelta struct { Role string `json:"role,omitempty"` Content *string `json:"content,omitempty"` // pointer: omit when not present, null vs "" matters + Refusal *string `json:"refusal,omitempty"` ReasoningContent *string `json:"reasoning_content,omitempty"` Reasoning *string `json:"reasoning,omitempty"` ToolCalls []ChatToolCall `json:"tool_calls,omitempty"` diff --git a/backend/internal/pkg/claude/constants.go b/backend/internal/pkg/claude/constants.go index 02e0533e09a8..3578cedc9cf2 100644 --- a/backend/internal/pkg/claude/constants.go +++ b/backend/internal/pkg/claude/constants.go @@ -20,6 +20,9 @@ const ( BetaFastMode = "fast-mode-2026-02-01" // Legacy structured output compatibility; forwarded only when explicitly requested. BetaStructuredOutputs = "structured-outputs-2025-11-13" + // Tool-change compatibility tokens; never enabled without an explicit client request. + BetaMidConversationToolChanges = "mid-conversation-tool-changes-2026-07-01" + BetaInlineTools = "inline-tools-2026-09-15" // 新增(对齐官方 CLI 2.1.9x 以来的流量) BetaPromptCachingScope = "prompt-caching-scope-2026-01-05" diff --git a/backend/internal/pkg/xai/sso_device.go b/backend/internal/pkg/xai/sso_device.go index 825616b93c39..5a06d24f90bb 100644 --- a/backend/internal/pkg/xai/sso_device.go +++ b/backend/internal/pkg/xai/sso_device.go @@ -1,6 +1,7 @@ package xai import ( + "bytes" "context" "encoding/base64" "encoding/json" @@ -14,6 +15,8 @@ import ( "strconv" "strings" "time" + + "golang.org/x/net/html" ) const ( @@ -147,7 +150,7 @@ func (f *ssoDeviceFlow) convert(ctx context.Context) (*TokenResponse, error) { return nil, fmt.Errorf("open xAI device verification page: %w", SSOHTTPError{Status: status}) } - status, finalURL, _, err = f.do(ctx, http.MethodPost, SSOVerifyURL, url.Values{"user_code": {device.UserCode}}) + status, finalURL, body, err = f.do(ctx, http.MethodPost, SSOVerifyURL, url.Values{"user_code": {device.UserCode}}) if err != nil { return nil, err } @@ -158,11 +161,19 @@ func (f *ssoDeviceFlow) convert(ctx context.Context) (*TokenResponse, error) { return nil, errors.New("xAI device verification did not reach consent page") } - status, finalURL, _, err = f.do(ctx, http.MethodPost, SSOApproveURL, url.Values{ + approval := url.Values{ "user_code": {device.UserCode}, "action": {"allow"}, "principal_type": {"User"}, "principal_id": {""}, + } + if consentToken := ssoConsentToken(body); consentToken != "" { + approval.Set("consent_token", consentToken) + } + consentURL, _ := url.Parse(finalURL) // f.do only returns validated xAI URLs. + status, finalURL, _, err = f.do(ctx, http.MethodPost, SSOApproveURL, approval, http.Header{ + "Origin": {consentURL.Scheme + "://" + consentURL.Host}, + "Referer": {finalURL}, }) if err != nil { return nil, err @@ -177,6 +188,37 @@ func (f *ssoDeviceFlow) convert(ctx context.Context) (*TokenResponse, error) { return f.pollToken(ctx, device.DeviceCode, time.Duration(device.Interval)*time.Second, time.Duration(device.ExpiresIn)*time.Second) } +// xAI binds device approval to a hidden token on the consent page. Keep it in +// this flow only; older consent pages without the field remain supported. +func ssoConsentToken(body []byte) string { + tokenizer := html.NewTokenizer(bytes.NewReader(body)) + for { + switch tokenizer.Next() { + case html.ErrorToken: + return "" + case html.StartTagToken, html.SelfClosingTagToken: + token := tokenizer.Token() + if token.Data != "input" { + continue + } + var name, value, inputType string + for _, attr := range token.Attr { + switch attr.Key { + case "name": + name = attr.Val + case "value": + value = attr.Val + case "type": + inputType = attr.Val + } + } + if name == "consent_token" && strings.EqualFold(inputType, "hidden") { + return value + } + } + } +} + func (f *ssoDeviceFlow) pollToken(ctx context.Context, deviceCode string, interval, expiresIn time.Duration) (*TokenResponse, error) { if interval < time.Second { interval = time.Second @@ -241,7 +283,7 @@ func (f *ssoDeviceFlow) pollToken(ctx context.Context, deviceCode string, interv return nil, errors.New("xAI device flow token polling timed out") } -func (f *ssoDeviceFlow) do(ctx context.Context, method, endpoint string, form url.Values) (int, string, []byte, error) { +func (f *ssoDeviceFlow) do(ctx context.Context, method, endpoint string, form url.Values, headers ...http.Header) (int, string, []byte, error) { if !safeXAIAuthURL(endpoint) { return 0, "", nil, errors.New("xAI OAuth URL is not trusted") } @@ -260,6 +302,15 @@ func (f *ssoDeviceFlow) do(ctx context.Context, method, endpoint string, form ur request.Header.Set("Accept", "application/json, text/html;q=0.9, */*;q=0.8") request.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") request.Header.Set("User-Agent", f.userAgent) + if redirects == 0 { + for _, header := range headers { + for name, values := range header { + for _, value := range values { + request.Header.Add(name, value) + } + } + } + } if cookie := f.cookieHeader(request.URL); cookie != "" { request.Header.Set("Cookie", cookie) } diff --git a/backend/internal/pkg/xai/sso_device_test.go b/backend/internal/pkg/xai/sso_device_test.go index cd3bae3aa36f..4b59dd4a5724 100644 --- a/backend/internal/pkg/xai/sso_device_test.go +++ b/backend/internal/pkg/xai/sso_device_test.go @@ -23,6 +23,10 @@ type ssoDeviceFakeClient struct { func (c *ssoDeviceFakeClient) Do(req *http.Request) (*http.Response, error) { c.cookieHeaders = append(c.cookieHeaders, req.Header.Get("Cookie")) + if req.URL.String() != SSOApproveURL { + require.Empty(c.t, req.Header.Get("Origin")) + require.Empty(c.t, req.Header.Get("Referer")) + } switch req.URL.String() { case SSOAccountsURL: require.Equal(c.t, http.MethodGet, req.Method) @@ -43,13 +47,16 @@ func (c *ssoDeviceFakeClient) Do(req *http.Request) (*http.Response, error) { return ssoDeviceResponse(http.StatusFound, http.Header{"Location": {"/oauth2/device/consent"}}, ``), nil case "https://auth.x.ai/oauth2/device/consent": require.Equal(c.t, http.MethodGet, req.Method) - return ssoDeviceResponse(http.StatusOK, nil, `consent`), nil + return ssoDeviceResponse(http.StatusOK, nil, `
`), nil case SSOApproveURL: require.Equal(c.t, http.MethodPost, req.Method) + require.Equal(c.t, "https://auth.x.ai", req.Header.Get("Origin")) + require.Equal(c.t, "https://auth.x.ai/oauth2/device/consent", req.Header.Get("Referer")) values := readSSODeviceForm(c.t, req) require.Equal(c.t, "USER-1", values.Get("user_code")) require.Equal(c.t, "allow", values.Get("action")) require.Equal(c.t, "User", values.Get("principal_type")) + require.Equal(c.t, "consent&token", values.Get("consent_token")) return ssoDeviceResponse(http.StatusSeeOther, http.Header{"Location": {"/oauth2/device/done"}}, ``), nil case "https://auth.x.ai/oauth2/device/done": require.Equal(c.t, http.MethodGet, req.Method) @@ -87,6 +94,24 @@ func TestConvertSSOToBuildCompletesDeviceFlow(t *testing.T) { require.Contains(t, client.cookieHeaders[0], "sso-rw=sso-token") require.Contains(t, client.cookieHeaders[len(client.cookieHeaders)-1], "session=web-session") require.Contains(t, client.cookieHeaders[len(client.cookieHeaders)-1], "csrf=csrf-token") + for _, cookie := range client.cookieHeaders { + require.NotContains(t, cookie, "consent&token") + } +} + +func TestSSOConsentToken(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + {"legacy page", `consent`, ""}, + {"encoded hidden value", ``, "a&b+c"}, + {"case insensitive HTML", ``, "token"}, + {"ignore script and visible fields", ``, "real"}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ssoConsentToken([]byte(tc.body))) + }) + } } func TestNormalizeSSOTokenAcceptsCookieHeader(t *testing.T) { diff --git a/backend/internal/repository/gateway_cache.go b/backend/internal/repository/gateway_cache.go index 7f075218ce95..81549ef0ae73 100644 --- a/backend/internal/repository/gateway_cache.go +++ b/backend/internal/repository/gateway_cache.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "io" "strconv" "strings" "time" @@ -23,27 +22,12 @@ const openAIRuntimeBreakerHalfOpenRetention = 5 * time.Minute type gatewayCache struct { rdb *redis.Client - // reasoningStateRDB serves only the rejected-cipher memory; see - // newReasoningStateClient. - reasoningStateRDB *redis.Client } func NewGatewayCache(rdb *redis.Client) service.GatewayCache { - return &gatewayCache{rdb: rdb, reasoningStateRDB: newReasoningStateClient(rdb)} + return &gatewayCache{rdb: rdb} } -// Close releases the client of the rejected-cipher memory, the only resource -// the cache owns; application cleanup calls it through io.Closer. The shared -// client is closed by its owner. -func (c *gatewayCache) Close() error { - if c == nil || c.reasoningStateRDB == nil { - return nil - } - return c.reasoningStateRDB.Close() -} - -var _ io.Closer = (*gatewayCache)(nil) - // buildSessionKey 构建 session key,包含 groupID 实现分组隔离 // 格式: sticky_session:{groupID}:{sessionHash} func buildSessionKey(groupID int64, sessionHash string) string { diff --git a/backend/internal/repository/gateway_cache_reasoning_state.go b/backend/internal/repository/gateway_cache_reasoning_state.go deleted file mode 100644 index 5035a9c81cac..000000000000 --- a/backend/internal/repository/gateway_cache_reasoning_state.go +++ /dev/null @@ -1,245 +0,0 @@ -package repository - -import ( - "context" - "errors" - "strconv" - "strings" - "time" - - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/redis/go-redis/v9" -) - -const rejectedReasoningPrefix = "openai_reasoning_state:v1:{openai_rejected_reasoning}:" - -var _ service.OpenAIReasoningStateStore = (*gatewayCache)(nil) - -// Every rejection has its own hard TTL. Index TTLs and lazy expiry cleanup are -// bookkeeping only: touching LRU never extends a rejection's lifetime. The size, -// owner, expiry, and LRU indexes are updated atomically with every replacement -// and eviction, including records whose Redis entry TTL already elapsed. -var rejectedReasoningScript = redis.NewScript(` -local prefix, op, tenant = ARGV[1], ARGV[2], ARGV[3] -local ttl, maxBytes, maxEntries = tonumber(ARGV[4]), tonumber(ARGV[5]), tonumber(ARGV[6]) -local maxTenantBytes, maxEntryBytes = tonumber(ARGV[7]), tonumber(ARGV[8]) -local clock = redis.call('TIME') -local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) -local tick = tonumber(clock[1]) * 1000000 + tonumber(clock[2]) - -local function remove(id) - local size = tonumber(redis.call('HGET', KEYS[3], id)) or 0 - local owner = redis.call('HGET', KEYS[4], id) - redis.call('DEL', prefix .. 'entry:' .. id) - redis.call('ZREM', KEYS[1], id) - redis.call('ZREM', KEYS[2], id) - redis.call('HDEL', KEYS[3], id) - redis.call('HDEL', KEYS[4], id) - if size > 0 then - if redis.call('HINCRBY', KEYS[5], '_global', -size) <= 0 then - redis.call('HDEL', KEYS[5], '_global') - end - end - if owner then - redis.call('ZREM', prefix .. 'tenant:' .. owner, id) - if size > 0 and redis.call('HINCRBY', KEYS[5], owner, -size) <= 0 then - redis.call('HDEL', KEYS[5], owner) - end - end -end - --- Cap opportunistic GC work; stale entries remain bounded by maxEntries and --- are also removed on direct lookup or while selecting eviction victims. -for _, id in ipairs(redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', now, 'LIMIT', 0, 128)) do - remove(id) -end - -if op == 'get' then - local result = {} - for index = 9, #ARGV do - local id = ARGV[index] - local payload = redis.call('HGET', prefix .. 'entry:' .. id, 'payload') - local owner = redis.call('HGET', KEYS[4], id) - if payload and owner == tenant then - redis.call('ZADD', KEYS[1], tick, id) - redis.call('ZADD', prefix .. 'tenant:' .. tenant, tick, id) - table.insert(result, id) - table.insert(result, payload) - elseif not payload then - remove(id) - end - end - return result -end - -if op ~= 'put' then return redis.error_reply('invalid reasoning state operation') end - -local payload = string.format('%.0f:%.0f', now, now + ttl) -local records = {} -for index = 9, #ARGV do - local id = ARGV[index] - local size = string.len(payload) + string.len(id) - if size > maxEntryBytes or size > maxBytes or size > maxTenantBytes then return 0 end - table.insert(records, {id, size}) -end - -for _, record in ipairs(records) do - local id, size = record[1], record[2] - remove(id) - while (tonumber(redis.call('HGET', KEYS[5], tenant)) or 0) + size > maxTenantBytes do - local victim = redis.call('ZRANGE', prefix .. 'tenant:' .. tenant, 0, 0)[1] - if not victim then return redis.error_reply('reasoning tenant index inconsistent') end - remove(victim) - end - while (tonumber(redis.call('HGET', KEYS[5], '_global')) or 0) + size > maxBytes or - redis.call('ZCARD', KEYS[1]) >= maxEntries do - local victim = redis.call('ZRANGE', KEYS[1], 0, 0)[1] - if not victim then return redis.error_reply('reasoning global index inconsistent') end - remove(victim) - end - redis.call('HSET', prefix .. 'entry:' .. id, 'payload', payload) - redis.call('PEXPIRE', prefix .. 'entry:' .. id, ttl) - redis.call('HSET', KEYS[3], id, size) - redis.call('HSET', KEYS[4], id, tenant) - redis.call('HINCRBY', KEYS[5], '_global', size) - redis.call('HINCRBY', KEYS[5], tenant, size) - redis.call('ZADD', KEYS[1], tick, id) - redis.call('ZADD', KEYS[2], now + ttl, id) - redis.call('ZADD', prefix .. 'tenant:' .. tenant, tick, id) - redis.call('PEXPIRE', prefix .. 'tenant:' .. tenant, ttl) -end -for _, key in ipairs(KEYS) do redis.call('PEXPIRE', key, ttl) end -return #records -`) - -type reasoningStateLimits struct { - bytes, entries, tenantBytes, entryBytes int -} - -var rejectedReasoningLimits = reasoningStateLimits{ - bytes: service.OpenAIRejectedReasoningMaxBytes, - entries: service.OpenAIRejectedReasoningMaxEntries, - tenantBytes: service.OpenAIRejectedReasoningMaxBytes, - entryBytes: 256, -} - -func rejectedReasoningKeys() []string { - return []string{ - rejectedReasoningPrefix + "lru", rejectedReasoningPrefix + "expiry", rejectedReasoningPrefix + "sizes", - rejectedReasoningPrefix + "owners", rejectedReasoningPrefix + "totals", - } -} - -func rejectedReasoningArguments(operation string, scope service.OpenAIReasoningCacheScope, limits reasoningStateLimits) []any { - return []any{rejectedReasoningPrefix, operation, scope.TenantHash, service.OpenAIReasoningStateTTL.Milliseconds(), - limits.bytes, limits.entries, limits.tenantBytes, limits.entryBytes} -} - -func validateReasoningStateKeys(scope service.OpenAIReasoningCacheScope, keys []string) error { - if !service.IsOpenAIReasoningCacheDigest(scope.ScopeHash) || !service.IsOpenAIReasoningCacheDigest(scope.TenantHash) || - len(keys) > service.OpenAIReasoningStateMaxLookupEntries { - return service.ErrOpenAIReasoningCacheInput - } - for _, key := range keys { - if !service.IsOpenAIReasoningCacheDigest(key) { - return service.ErrOpenAIReasoningCacheInput - } - } - return nil -} - -func (c *gatewayCache) GetOpenAIRejectedReasoning(ctx context.Context, scope service.OpenAIReasoningCacheScope, cipherHashes []string) (map[string]service.OpenAIRejectedReasoning, error) { - if err := validateReasoningStateKeys(scope, cipherHashes); err != nil { - return nil, err - } - if len(cipherHashes) == 0 { - return nil, nil - } - args := rejectedReasoningArguments("get", scope, rejectedReasoningLimits) - for _, hash := range cipherHashes { - args = append(args, scope.ScopeHash+":"+hash) - } - reply, err := c.runRejectedReasoning(ctx, args) - if err != nil { - return nil, err - } - values, ok := reply.([]any) - if !ok || len(values)%2 != 0 { - return nil, errors.New("invalid rejected reasoning cache result") - } - result := make(map[string]service.OpenAIRejectedReasoning, len(values)/2) - for index := 0; index < len(values); index += 2 { - key, keyOK := values[index].(string) - payload, payloadOK := values[index+1].(string) - if !keyOK || !payloadOK { - return nil, errors.New("invalid rejected reasoning cache record") - } - rejected, expires, found := strings.Cut(payload, ":") - rejectedMS, rejectedErr := strconv.ParseInt(rejected, 10, 64) - expiresMS, expiresErr := strconv.ParseInt(expires, 10, 64) - if !found || rejectedErr != nil || expiresErr != nil || expiresMS-rejectedMS != service.OpenAIReasoningStateTTL.Milliseconds() { - return nil, errors.New("invalid rejected reasoning cache record") - } - result[strings.TrimPrefix(key, scope.ScopeHash+":")] = service.OpenAIRejectedReasoning{ - RejectedAt: time.UnixMilli(rejectedMS), ExpiresAt: time.UnixMilli(expiresMS), - } - } - return result, nil -} - -func (c *gatewayCache) PutOpenAIRejectedReasoning(ctx context.Context, scope service.OpenAIReasoningCacheScope, cipherHashes []string) error { - if err := validateReasoningStateKeys(scope, cipherHashes); err != nil { - return err - } - if len(cipherHashes) == 0 { - return nil - } - args := rejectedReasoningArguments("put", scope, rejectedReasoningLimits) - seen := make(map[string]bool, len(cipherHashes)) - for _, hash := range cipherHashes { - if !seen[hash] { - args = append(args, scope.ScopeHash+":"+hash) - seen[hash] = true - } - } - _, err := c.runRejectedReasoning(ctx, args) - return err -} - -// newReasoningStateClient builds the client the rejected-cipher memory runs on, -// once for the gateway cache, which owns it and closes it in Close. The -// shared client does not apply a context deadline to socket reads and writes -// (ContextTimeoutEnabled is off): on it every write and read gets a timeout of -// its own, counted from the moment it starts. This client applies the context -// deadline, so every write and read of an operation ends at the operation's -// deadline however long the connection took to acquire. It sends a command -// once and dials once, so an unreachable Redis fails at the refused dial. It -// opens connections on demand and keeps them for the next operation. The -// shared options are copied, never changed. -func newReasoningStateClient(shared *redis.Client) *redis.Client { - if shared == nil { - return nil - } - opts := *shared.Options() - opts.ContextTimeoutEnabled = true - opts.MaxRetries = -1 - opts.DialerRetries = 1 - opts.MinIdleConns = 0 - // The shared client's notification processor holds that client's handlers. - opts.PushNotificationProcessor = nil - return redis.NewClient(&opts) -} - -func (c *gatewayCache) runRejectedReasoning(ctx context.Context, args []any) (any, error) { - if c == nil || c.reasoningStateRDB == nil { - return nil, errors.New("reasoning state cache unavailable") - } - if err := ctx.Err(); err != nil { - return nil, err - } - // Callers supply the request-wide remaining I/O budget. Bound direct users - // too; never inherit the ordinary multi-second Redis defaults for this cache. - ioCtx, cancel := context.WithTimeout(ctx, service.OpenAIReasoningStateIOBudget) - defer cancel() - return rejectedReasoningScript.Run(ioCtx, c.reasoningStateRDB, rejectedReasoningKeys(), args...).Result() -} diff --git a/backend/internal/repository/gateway_cache_reasoning_state_test.go b/backend/internal/repository/gateway_cache_reasoning_state_test.go deleted file mode 100644 index a10db366a6f2..000000000000 --- a/backend/internal/repository/gateway_cache_reasoning_state_test.go +++ /dev/null @@ -1,321 +0,0 @@ -package repository - -import ( - "context" - "errors" - "io" - "net" - "strconv" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/Wei-Shaw/sub2api/internal/service" - "github.com/alicebob/miniredis/v2" - "github.com/redis/go-redis/v9" - "github.com/stretchr/testify/require" -) - -// One stored rejection is charged its payload (":", -// 13 digits each) plus its id (":"). -const reasoningStateTestEntryBytes = 13 + 1 + 13 + 64 + 1 + 64 - -func newReasoningStateTestCache(t *testing.T) (*gatewayCache, *miniredis.Miniredis) { - t.Helper() - server := miniredis.RunT(t) - return newReasoningStateTestCacheOn(t, redis.NewClient(&redis.Options{Addr: server.Addr()})), server -} - -// The cache is built and closed as production does it: the rejection memory -// gets its own client derived from the shared one, and the cache closes it -// before the shared client is closed. -func newReasoningStateTestCacheOn(t *testing.T, client *redis.Client) *gatewayCache { - t.Helper() - cache, ok := NewGatewayCache(client).(*gatewayCache) - require.True(t, ok) - t.Cleanup(func() { - _ = cache.Close() - _ = client.Close() - }) - return cache -} - -func reasoningStateTestScope() service.OpenAIReasoningCacheScope { - return service.OpenAIReasoningCacheScope{ScopeHash: strings.Repeat("a", 64), TenantHash: strings.Repeat("b", 64)} -} - -func TestGatewayCacheReasoningStateHardTTLAndIsolation(t *testing.T) { - cache, server := newReasoningStateTestCache(t) - scope, key, ctx := reasoningStateTestScope(), strings.Repeat("d", 64), context.Background() - require.NoError(t, cache.PutOpenAIRejectedReasoning(ctx, scope, []string{key})) - entryKey := rejectedReasoningPrefix + "entry:" + scope.ScopeHash + ":" + key - require.Equal(t, 24*time.Hour, server.TTL(entryKey)) - - server.FastForward(23 * time.Hour) - rejected, err := cache.GetOpenAIRejectedReasoning(ctx, scope, []string{key, strings.Repeat("e", 64)}) - require.NoError(t, err) - require.Len(t, rejected, 1) - require.Equal(t, 24*time.Hour, rejected[key].ExpiresAt.Sub(rejected[key].RejectedAt)) - require.Equal(t, time.Hour, server.TTL(entryKey), "a lookup must not extend the rejection") - other := scope - other.ScopeHash = strings.Repeat("f", 64) - none, err := cache.GetOpenAIRejectedReasoning(ctx, other, []string{key}) - require.NoError(t, err) - require.Empty(t, none) - other = scope - other.TenantHash = strings.Repeat("f", 64) - none, err = cache.GetOpenAIRejectedReasoning(ctx, other, []string{key}) - require.NoError(t, err) - require.Empty(t, none) - - // Only another real rejection write renews the evidence. - require.NoError(t, cache.PutOpenAIRejectedReasoning(ctx, scope, []string{key})) - require.Equal(t, 24*time.Hour, server.TTL(entryKey)) - server.FastForward(time.Hour + time.Millisecond) - rejected, err = cache.GetOpenAIRejectedReasoning(ctx, scope, []string{key}) - require.NoError(t, err) - require.Len(t, rejected, 1) - server.FastForward(23 * time.Hour) - rejected, err = cache.GetOpenAIRejectedReasoning(ctx, scope, []string{key}) - require.NoError(t, err) - require.Empty(t, rejected) -} - -func TestGatewayCacheReasoningStateBoundedAtomicLRU(t *testing.T) { - cache, server := newReasoningStateTestCache(t) - ctx, scope := context.Background(), reasoningStateTestScope() - other := service.OpenAIReasoningCacheScope{ScopeHash: strings.Repeat("c", 64), TenantHash: strings.Repeat("d", 64)} - const unit = reasoningStateTestEntryBytes - limits := reasoningStateLimits{bytes: 3 * unit, entries: 3, tenantBytes: 2 * unit, entryBytes: unit} - clock := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - put := func(owner service.OpenAIReasoningCacheScope, key string) { - t.Helper() - clock = clock.Add(time.Millisecond) - server.SetTime(clock) - args := append(rejectedReasoningArguments("put", owner, limits), owner.ScopeHash+":"+key) - result, err := cache.runRejectedReasoning(ctx, args) - require.NoError(t, err) - require.Equal(t, int64(1), result) - } - entry := func(owner service.OpenAIReasoningCacheScope, key string) string { - return rejectedReasoningPrefix + "entry:" + owner.ScopeHash + ":" + key - } - first, second, third, fourth, fifth := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64), strings.Repeat("4", 64), strings.Repeat("5", 64) - put(scope, first) - put(scope, second) - put(other, third) - clock = clock.Add(time.Millisecond) - server.SetTime(clock) - _, err := cache.runRejectedReasoning(ctx, append(rejectedReasoningArguments("get", scope, limits), scope.ScopeHash+":"+first)) - require.NoError(t, err) - put(scope, fourth) // The per-key bound of two entries evicts second, not the touched first. - require.True(t, server.Exists(entry(scope, first))) - require.False(t, server.Exists(entry(scope, second))) - put(other, fifth) // The global bound of three entries evicts the oldest third. - require.False(t, server.Exists(entry(other, third))) - require.Equal(t, strconv.Itoa(3*unit), server.HGet(rejectedReasoningPrefix+"totals", "_global")) - require.Equal(t, strconv.Itoa(2*unit), server.HGet(rejectedReasoningPrefix+"totals", scope.TenantHash)) - require.Equal(t, strconv.Itoa(unit), server.HGet(rejectedReasoningPrefix+"totals", other.TenantHash)) - count, err := cache.rdb.ZCard(ctx, rejectedReasoningPrefix+"lru").Result() - require.NoError(t, err) - require.Equal(t, int64(3), count) - - // Refuse an oversized incoming record without deleting an existing one. - stored := server.HGet(entry(scope, first), "payload") - oversized := limits - oversized.entryBytes = unit - 1 - result, err := cache.runRejectedReasoning(ctx, append(rejectedReasoningArguments("put", scope, oversized), scope.ScopeHash+":"+first)) - require.NoError(t, err) - require.Equal(t, int64(0), result) - require.Equal(t, stored, server.HGet(entry(scope, first), "payload")) - - // Lower entry limit independently from bytes to exercise the count gate. - limits.entries, limits.bytes, limits.tenantBytes = 2, 100*unit, 100*unit - put(scope, strings.Repeat("6", 64)) - count, err = cache.rdb.ZCard(ctx, rejectedReasoningPrefix+"lru").Result() - require.NoError(t, err) - require.Equal(t, int64(2), count) -} - -func TestGatewayCacheReasoningStateBoundAndDigestOnly(t *testing.T) { - cache, server := newReasoningStateTestCache(t) - scope, ctx := reasoningStateTestScope(), context.Background() - limits := reasoningStateLimits{bytes: 350, entries: 2, tenantBytes: 350, entryBytes: 256} - args := rejectedReasoningArguments("put", scope, limits) - for _, digit := range []string{"1", "2", "3"} { - args = append(args, scope.ScopeHash+":"+strings.Repeat(digit, 64)) - } - _, err := cache.runRejectedReasoning(ctx, args) - require.NoError(t, err) - count, err := cache.rdb.ZCard(ctx, rejectedReasoningPrefix+"lru").Result() - require.NoError(t, err) - require.Equal(t, int64(2), count) - total, err := strconv.Atoi(server.HGet(rejectedReasoningPrefix+"totals", "_global")) - require.NoError(t, err) - require.LessOrEqual(t, total, 350) - for _, key := range server.Keys() { - require.True(t, strings.HasPrefix(key, rejectedReasoningPrefix)) - require.Positive(t, server.TTL(key), "every key of the rejection memory expires") - if strings.Contains(key, ":entry:") { - require.Regexp(t, `^\d+:\d+$`, server.HGet(key, "payload")) - } - } -} - -func TestGatewayCacheReasoningStateConcurrentCapacity(t *testing.T) { - cache, server := newReasoningStateTestCache(t) - scope, ctx := reasoningStateTestScope(), context.Background() - const unit = reasoningStateTestEntryBytes - limits := reasoningStateLimits{bytes: 4 * unit, entries: 4, tenantBytes: 2 * unit, entryBytes: unit} - results := make(chan error, 8) - for index := range cap(results) { - go func() { - key := strings.Repeat(strconv.Itoa(index), 64) - _, err := cache.runRejectedReasoning(ctx, append(rejectedReasoningArguments("put", scope, limits), scope.ScopeHash+":"+key)) - results <- err - }() - } - for range cap(results) { - require.NoError(t, <-results) - } - require.Equal(t, strconv.Itoa(2*unit), server.HGet(rejectedReasoningPrefix+"totals", "_global")) - require.Equal(t, strconv.Itoa(2*unit), server.HGet(rejectedReasoningPrefix+"totals", scope.TenantHash)) - count, err := cache.rdb.ZCard(ctx, rejectedReasoningPrefix+"lru").Result() - require.NoError(t, err) - require.Equal(t, int64(2), count) -} - -// The rejection memory keeps its connections: operations that follow each other -// reuse one connection instead of dialing. -func TestGatewayCacheReasoningStateReusesItsConnection(t *testing.T) { - server := miniredis.RunT(t) - var dials atomic.Int32 - cache := newReasoningStateTestCacheOn(t, redis.NewClient(&redis.Options{ - Addr: server.Addr(), - Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) { - dials.Add(1) - return (&net.Dialer{}).DialContext(ctx, network, addr) - }, - })) - scope, key, ctx := reasoningStateTestScope(), strings.Repeat("d", 64), context.Background() - for range 3 { - require.NoError(t, cache.PutOpenAIRejectedReasoning(ctx, scope, []string{key})) - rejected, err := cache.GetOpenAIRejectedReasoning(ctx, scope, []string{key}) - require.NoError(t, err) - require.Len(t, rejected, 1) - } - require.Equal(t, int32(1), dials.Load()) -} - -// Closing the cache closes the rejection memory's client and nothing else: the -// shared client belongs to whoever built it. -func TestGatewayCacheReasoningStateCloseKeepsTheSharedClient(t *testing.T) { - cache, _ := newReasoningStateTestCache(t) - scope, key, ctx := reasoningStateTestScope(), strings.Repeat("d", 64), context.Background() - require.NoError(t, cache.PutOpenAIRejectedReasoning(ctx, scope, []string{key})) - require.NoError(t, cache.Close()) - _, err := cache.GetOpenAIRejectedReasoning(ctx, scope, []string{key}) - require.ErrorIs(t, err, redis.ErrClosed) - require.NoError(t, cache.rdb.Ping(ctx).Err(), "the shared client stays open") -} - -// reasoningStateDeadlineConn reports every deadline the client sets on its socket. -type reasoningStateDeadlineConn struct { - net.Conn - record func(time.Time) -} - -func (c reasoningStateDeadlineConn) SetDeadline(deadline time.Time) error { - c.record(deadline) - return c.Conn.SetDeadline(deadline) -} - -func (c reasoningStateDeadlineConn) SetReadDeadline(deadline time.Time) error { - c.record(deadline) - return c.Conn.SetReadDeadline(deadline) -} - -func (c reasoningStateDeadlineConn) SetWriteDeadline(deadline time.Time) error { - c.record(deadline) - return c.Conn.SetWriteDeadline(deadline) -} - -// A Redis that is slow to connect and then never answers costs the request no -// more than its I/O budget: no write or read gets a deadline later than the -// budget's own, although acquiring the connection already spent part of it. -// The bound belongs to the rejection memory's client: building it leaves the -// shared client's options as they were. -func TestGatewayCacheReasoningStateBlockedTransportBudget(t *testing.T) { - var socketDeadlines []time.Time - client := redis.NewClient(&redis.Options{ - Addr: "synthetic.invalid:6379", - Dialer: func(context.Context, string, string) (net.Conn, error) { - time.Sleep(20 * time.Millisecond) - local, peer := net.Pipe() - go func() { - defer func() { _ = peer.Close() }() - _, _ = io.Copy(io.Discard, peer) // Accept writes, never send a reply. - }() - return reasoningStateDeadlineConn{Conn: local, record: func(deadline time.Time) { - socketDeadlines = append(socketDeadlines, deadline) - }}, nil - }, - }) - sharedOptions := *client.Options() - cache := newReasoningStateTestCacheOn(t, client) - budget := service.NewOpenAIReasoningCacheBudget() - started := time.Now() - require.NoError(t, budget.Do(context.Background(), func(ctx context.Context) error { - select { - case <-time.After(20 * time.Millisecond): - return nil - case <-ctx.Done(): - return ctx.Err() - } - })) - var budgetDeadline time.Time - err := budget.Do(context.Background(), func(ctx context.Context) error { - budgetDeadline, _ = ctx.Deadline() - _, err := cache.GetOpenAIRejectedReasoning(ctx, reasoningStateTestScope(), []string{strings.Repeat("d", 64)}) - return err - }) - require.Error(t, err) - require.LessOrEqual(t, time.Since(started), service.OpenAIReasoningStateIOBudget+50*time.Millisecond) - require.NotEmpty(t, socketDeadlines, "the exchange started inside the budget") - for _, deadline := range socketDeadlines { - require.False(t, deadline.After(budgetDeadline), "a write or read may not outlast the budget") - } - require.ErrorIs(t, budget.Do(context.Background(), func(context.Context) error { return nil }), service.ErrOpenAIReasoningCacheBudget, - "a spent budget starts no further cache I/O for the request") - options := client.Options() - require.Equal(t, sharedOptions.ContextTimeoutEnabled, options.ContextTimeoutEnabled, "shared Redis options must remain unchanged") - require.Equal(t, sharedOptions.MaxRetries, options.MaxRetries) - require.Equal(t, sharedOptions.DialerRetries, options.DialerRetries) - require.Equal(t, sharedOptions.PushNotificationProcessor, options.PushNotificationProcessor) -} - -// An unreachable Redis fails the operation at the first refused dial: nothing -// is retried and nothing waits for the budget to run out. A request that is -// already cancelled starts no Redis work. -func TestGatewayCacheReasoningStateRefusedDialFailsAtOnce(t *testing.T) { - var dials atomic.Int32 - cache := newReasoningStateTestCacheOn(t, redis.NewClient(&redis.Options{ - Addr: "synthetic.invalid:6379", - Dialer: func(context.Context, string, string) (net.Conn, error) { - dials.Add(1) - return nil, errors.New("synthetic cache unavailable") - }, - })) - started := time.Now() - _, err := cache.GetOpenAIRejectedReasoning(context.Background(), reasoningStateTestScope(), []string{strings.Repeat("d", 64)}) - require.Error(t, err) - require.Less(t, time.Since(started), service.OpenAIReasoningStateIOBudget) - require.Equal(t, int32(1), dials.Load()) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err = cache.GetOpenAIRejectedReasoning(ctx, reasoningStateTestScope(), []string{strings.Repeat("d", 64)}) - require.ErrorIs(t, err, context.Canceled) - require.Equal(t, int32(1), dials.Load()) -} diff --git a/backend/internal/repository/ops_repo_dashboard.go b/backend/internal/repository/ops_repo_dashboard.go index a22235a805c6..6fa037d37055 100644 --- a/backend/internal/repository/ops_repo_dashboard.go +++ b/backend/internal/repository/ops_repo_dashboard.go @@ -33,18 +33,63 @@ func (r *opsRepository) GetDashboardOverview(ctx context.Context, filter *servic mode = service.OpsQueryModeRaw } + var overview *service.OpsDashboardOverview + var err error switch mode { case service.OpsQueryModePreagg: - return r.getDashboardOverviewPreaggregated(ctx, filter) + overview, err = r.getDashboardOverviewPreaggregated(ctx, filter) case service.OpsQueryModeAuto: - out, err := r.getDashboardOverviewPreaggregated(ctx, filter) - if err != nil && errors.Is(err, service.ErrOpsPreaggregatedNotPopulated) { - return r.getDashboardOverviewRaw(ctx, filter) + overview, err = r.getDashboardOverviewPreaggregated(ctx, filter) + if errors.Is(err, service.ErrOpsPreaggregatedNotPopulated) { + overview, err = r.getDashboardOverviewRaw(ctx, filter) } - return out, err default: - return r.getDashboardOverviewRaw(ctx, filter) + overview, err = r.getDashboardOverviewRaw(ctx, filter) + } + if err != nil { + return nil, err + } + // Percentiles must use individual rates over the entire window, even in + // preagg mode: hourly percentiles cannot be averaged into window percentiles. + rateCtx, cancelRate := context.WithTimeout(ctx, opsRawLatencyQueryTimeout) + defer cancelRate() + overview.OutputTPS, err = r.queryOutputTPS(rateCtx, filter) + if err != nil && !isQueryTimeoutErr(err) { + return nil, err + } + return overview, nil +} + +func (r *opsRepository) queryOutputTPS(ctx context.Context, filter *service.OpsDashboardFilter) (*service.OpsOutputTPS, error) { + join, where, args, _ := buildUsageWhere(filter, filter.StartTime.UTC(), filter.EndTime.UTC(), 1) + query := ` +SELECT + percentile_cont(0.05) WITHIN GROUP (ORDER BY output_tps), + percentile_cont(0.10) WITHIN GROUP (ORDER BY output_tps), + percentile_cont(0.50) WITHIN GROUP (ORDER BY output_tps), + AVG(output_tps), + COUNT(*) +FROM ( + SELECT ul.output_tokens * 1000.0 / NULLIF(ul.duration_ms, 0) AS output_tps + FROM usage_logs ul + ` + join + ` + ` + where + ` + AND ul.output_tokens > 0 AND ul.duration_ms > 0 + AND ul.image_count = 0 AND ul.image_output_tokens = 0 + AND COALESCE(ul.billing_mode, '') <> 'image' + ` + fmt.Sprintf("AND ul.request_type IN (%d, %d, %d, %d)", + service.RequestTypeSync, service.RequestTypeStream, service.RequestTypeWSV2, service.RequestTypeCyberBlocked) + ` +) rates` + var stats service.OpsOutputTPS + if err := r.db.QueryRowContext(ctx, query, args...).Scan(&stats.P5, &stats.P10, &stats.P50, &stats.Avg, &stats.SampleCount); err != nil { + // lib/pq reports query cancellation without wrapping the context error. + // Preserve deadline vs caller cancellation for the overview's timeout policy. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, err } + return &stats, nil } func (r *opsRepository) getDashboardOverviewRaw(ctx context.Context, filter *service.OpsDashboardFilter) (*service.OpsDashboardOverview, error) { diff --git a/backend/internal/repository/ops_repo_dashboard_integration_test.go b/backend/internal/repository/ops_repo_dashboard_integration_test.go new file mode 100644 index 000000000000..5cc331f52497 --- /dev/null +++ b/backend/internal/repository/ops_repo_dashboard_integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +package repository + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/stretchr/testify/require" +) + +func TestOpsOutputTPSDistribution(t *testing.T) { + ctx := context.Background() + client := testEntClient(t) + user := mustCreateUser(t, client, &service.User{}) + key := mustCreateApiKey(t, client, &service.APIKey{UserID: user.ID}) + account := mustCreateAccount(t, client, &service.Account{Name: "output-tps", Platform: service.PlatformAnthropic}) + group := mustCreateGroup(t, client, &service.Group{Name: "output-tps", Platform: service.PlatformOpenAI}) + otherGroup := mustCreateGroup(t, client, &service.Group{Name: "other-output-tps", Platform: service.PlatformOpenAI}) + start := time.Date(2021, 1, 17, 0, 30, 0, 0, time.UTC) + end := start.Add(6 * time.Hour) + t.Cleanup(func() { + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM usage_logs WHERE user_id = $1", user.ID) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM accounts WHERE id = $1", account.ID) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM ops_metrics_hourly WHERE bucket_start >= $1 AND bucket_start < $2", start.Truncate(time.Hour), end) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id IN ($1, $2)", group.ID, otherGroup.ID) + _, _ = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID) + }) + insert := func(at time.Time, groupID *int64, output int, duration *int, requestType service.RequestType, imageCount, imageTokens int, billingMode string) { + t.Helper() + _, err := integrationDB.ExecContext(ctx, `INSERT INTO usage_logs + (user_id, api_key_id, account_id, group_id, request_id, model, input_tokens, output_tokens, duration_ms, request_type, image_count, image_output_tokens, billing_mode, created_at, image_size) + VALUES ($1,$2,$3,$4,$5,'gpt-test',100000,$6,$7,$8,$9,$10,$11,$12,CASE WHEN $9 > 0 THEN '1K' ELSE NULL END)`, + user.ID, key.ID, account.ID, groupID, fmt.Sprintf("output-tps-%d-%d", user.ID, time.Now().UnixNano()), output, duration, requestType, imageCount, imageTokens, billingMode, at) + require.NoError(t, err) + } + // Rates 1, 10, 100, 1000: unequal durations catch ratio-of-sums mistakes. + // Put samples in the raw head, stable hourly buckets, and raw tail. + for _, sample := range []struct { + output, duration int + kind service.RequestType + at time.Time + }{ + {1, 1000, service.RequestTypeSync, start}, + {20, 2000, service.RequestTypeStream, start.Add(time.Hour)}, + {100, 1000, service.RequestTypeWSV2, start.Add(2 * time.Hour)}, + {500, 500, service.RequestTypeCyberBlocked, end.Add(-time.Minute)}, + } { + insert(sample.at, &group.ID, sample.output, &sample.duration, sample.kind, 0, 0, "token") + } + positive, zero, negative := 1000, 0, -1 + for _, d := range []*int{nil, &zero, &negative} { + insert(start, &group.ID, 9999, d, service.RequestTypeSync, 0, 0, "token") + } + for _, n := range []int{0, -1} { + insert(start, &group.ID, n, &positive, service.RequestTypeSync, 0, 0, "token") + } + for _, kind := range []service.RequestType{service.RequestTypeLive, service.RequestTypeUnknown} { + insert(start, &group.ID, 9999, &positive, kind, 0, 0, "token") + } + insert(start, &group.ID, 9999, &positive, service.RequestTypeSync, 1, 0, "token") + insert(start, &group.ID, 9999, &positive, service.RequestTypeSync, 0, 1, "token") + insert(start, &group.ID, 9999, &positive, service.RequestTypeSync, 0, 0, "image") + insert(start.Add(-time.Second), &group.ID, 9999, &positive, service.RequestTypeSync, 0, 0, "token") + insert(end, &group.ID, 9999, &positive, service.RequestTypeSync, 0, 0, "token") + insert(start, &otherGroup.ID, 9999, &positive, service.RequestTypeSync, 0, 0, "token") + insert(start, nil, 50, &positive, service.RequestTypeSync, 0, 0, "token") + repo := NewOpsRepository(integrationDB).(*opsRepository) + filter := &service.OpsDashboardFilter{StartTime: start, EndTime: end, Platform: " OpenAI ", GroupID: &group.ID} + assertStats := func(stats *service.OpsOutputTPS) { + t.Helper() + require.NotNil(t, stats) + require.EqualValues(t, 4, stats.SampleCount) + require.NotNil(t, stats.P5) + require.NotNil(t, stats.P10) + require.NotNil(t, stats.P50) + require.NotNil(t, stats.Avg) + require.InDelta(t, 2.35, *stats.P5, 1e-9) + require.InDelta(t, 3.7, *stats.P10, 1e-9) + require.InDelta(t, 55, *stats.P50, 1e-9) + require.InDelta(t, 277.75, *stats.Avg, 1e-9) + } + // Auto first exercises fallback before hourly rows exist. + for _, mode := range []service.OpsQueryMode{service.OpsQueryModeRaw, service.OpsQueryModeAuto} { + filter.QueryMode = mode + overview, err := repo.GetDashboardOverview(ctx, filter) + require.NoError(t, err) + assertStats(overview.OutputTPS) + } + require.NoError(t, repo.UpsertHourlyMetrics(ctx, start.Truncate(time.Hour), end)) + for _, mode := range []service.OpsQueryMode{service.OpsQueryModePreagg, service.OpsQueryModeAuto} { + filter.QueryMode = mode + overview, err := repo.GetDashboardOverview(ctx, filter) + require.NoError(t, err) + assertStats(overview.OutputTPS) + } + filter.Platform = service.PlatformAnthropic + empty, err := repo.queryOutputTPS(ctx, filter) + require.NoError(t, err) + require.Equal(t, &service.OpsOutputTPS{}, empty) + // Ungrouped logs use the account platform; group platform takes precedence. + filter.GroupID = nil + fallback, err := repo.queryOutputTPS(ctx, filter) + require.NoError(t, err) + require.EqualValues(t, 1, fallback.SampleCount) + require.Equal(t, 50.0, *fallback.P50) +} + +func TestOpsOutputTPSQueryDeadline(t *testing.T) { + tx := testTx(t) + _, err := tx.ExecContext(context.Background(), "LOCK TABLE usage_logs IN ACCESS EXCLUSIVE MODE") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + repo := NewOpsRepository(integrationDB).(*opsRepository) + now := time.Now().UTC() + stats, err := repo.queryOutputTPS(ctx, &service.OpsDashboardFilter{StartTime: now.Add(-time.Hour), EndTime: now}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Nil(t, stats) +} diff --git a/backend/internal/repository/ops_repo_dashboard_timeout_test.go b/backend/internal/repository/ops_repo_dashboard_timeout_test.go index 76332ca0a620..d0763b3f9611 100644 --- a/backend/internal/repository/ops_repo_dashboard_timeout_test.go +++ b/backend/internal/repository/ops_repo_dashboard_timeout_test.go @@ -2,8 +2,15 @@ package repository import ( "context" + "database/sql/driver" + "errors" "fmt" "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/stretchr/testify/require" ) func TestIsQueryTimeoutErr(t *testing.T) { @@ -20,3 +27,42 @@ func TestIsQueryTimeoutErr(t *testing.T) { t.Fatalf("wrapped context.Canceled should not be treated as query timeout") } } + +func TestOpsOutputTPSOverviewFailurePolicy(t *testing.T) { + for _, tc := range []struct { + name string + err error + wantError bool + }{ + {"timeout keeps overview", context.DeadlineExceeded, false}, + {"cancellation propagates", context.Canceled, true}, + {"database errors propagate", errors.New("database unavailable"), true}, + } { + t.Run(tc.name, func(t *testing.T) { + db, mock := newSQLMock(t) + repo := &opsRepository{db: db} + // The six existing raw overview queries run before the output-rate query. + for _, count := range []int{2, 13, 6, 2, 6, 2} { + columns := make([]string, count) + values := make([]driver.Value, count) + for i := range columns { + columns[i] = fmt.Sprint(i) + values[i] = int64(0) + } + mock.ExpectQuery("SELECT|WITH").WillReturnRows(sqlmock.NewRows(columns).AddRow(values...)) + } + mock.ExpectQuery(`percentile_cont\(0.05\)`).WillReturnError(tc.err) + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + overview, err := repo.GetDashboardOverview(context.Background(), &service.OpsDashboardFilter{StartTime: start, EndTime: start.Add(time.Hour), QueryMode: service.OpsQueryModeRaw}) + if tc.wantError { + require.ErrorIs(t, err, tc.err) + require.Nil(t, overview) + } else { + require.NoError(t, err) + require.NotNil(t, overview) + require.Nil(t, overview.OutputTPS) + } + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} diff --git a/backend/internal/repository/platform_list_db_integration_test.go b/backend/internal/repository/platform_list_db_integration_test.go new file mode 100644 index 000000000000..2bfb68861cf7 --- /dev/null +++ b/backend/internal/repository/platform_list_db_integration_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package repository + +import ( + "context" + "testing" + + dbent "github.com/Wei-Shaw/sub2api/ent" + "github.com/Wei-Shaw/sub2api/internal/service" + dbmigrations "github.com/Wei-Shaw/sub2api/migrations" + "github.com/stretchr/testify/require" +) + +func platformConstraintExists(t *testing.T, ctx context.Context, client *dbent.Client, table, constraint string) bool { + t.Helper() + rows, err := client.QueryContext(ctx, ` +SELECT 1 +FROM pg_constraint c +JOIN pg_class tbl ON tbl.oid = c.conrelid +JOIN pg_namespace ns ON ns.oid = tbl.relnamespace +WHERE ns.nspname = 'public' AND tbl.relname = $1 AND c.conname = $2`, table, constraint) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + exists := rows.Next() + require.NoError(t, rows.Err()) + return exists +} + +// 迁移 242 删除平台白名单 CHECK 约束:旧库(约束仍在)上可重复执行,执行后 +// 新登记的平台无需再做数据库迁移即可写入。 +func TestMigration242DropsPlatformCheckConstraints(t *testing.T) { + ctx := context.Background() + tx := testEntTx(t) + client := tx.Client() + + // 还原迁移 241_add_typesafe_platform 之后、242 之前的约束状态。 + for _, stmt := range []string{ + `ALTER TABLE user_platform_quotas DROP CONSTRAINT IF EXISTS user_platform_quotas_platform_check`, + `ALTER TABLE user_platform_quotas ADD CONSTRAINT user_platform_quotas_platform_check + CHECK (platform IN ('anthropic', 'openai', 'gemini', 'antigravity', 'grok', + 'kimi', 'zhipu', 'deepseek', 'minimax', 'opencode_go', 'typesafe'))`, + `ALTER TABLE composite_model_routes DROP CONSTRAINT IF EXISTS composite_model_routes_target_platform_check`, + `ALTER TABLE composite_model_routes ADD CONSTRAINT composite_model_routes_target_platform_check + CHECK (target_platform IN ('anthropic', 'openai', 'gemini', 'antigravity', 'grok', + 'kimi', 'zhipu', 'deepseek', 'minimax', 'opencode_go', 'typesafe'))`, + } { + _, err := client.ExecContext(ctx, stmt) + require.NoError(t, err) + } + require.True(t, platformConstraintExists(t, ctx, client, "user_platform_quotas", "user_platform_quotas_platform_check")) + require.True(t, platformConstraintExists(t, ctx, client, "composite_model_routes", "composite_model_routes_target_platform_check")) + + migrationSQL, err := dbmigrations.FS.ReadFile("242_drop_platform_check_constraints.sql") + require.NoError(t, err) + for range 2 { + _, err = client.ExecContext(ctx, string(migrationSQL)) + require.NoError(t, err) + } + require.False(t, platformConstraintExists(t, ctx, client, "user_platform_quotas", "user_platform_quotas_platform_check")) + require.False(t, platformConstraintExists(t, ctx, client, "composite_model_routes", "composite_model_routes_target_platform_check")) + // 渠道监控的 provider 约束表示探测能力,保留。 + require.True(t, platformConstraintExists(t, ctx, client, "channel_monitors", "channel_monitors_provider_check")) + require.True(t, platformConstraintExists(t, ctx, client, "channel_monitor_request_templates", "channel_monitor_request_templates_provider_check")) + + // 数据库层不再限制平台取值:模拟未来新登记的平台。 + userID := mustCreateUserForQuota(t, client) + _, err = client.ExecContext(ctx, ` +INSERT INTO user_platform_quotas (user_id, platform, daily_limit_usd, daily_usage_usd, weekly_usage_usd, monthly_usage_usd, created_at, updated_at) +VALUES ($1, 'future_platform', 1, 0, 0, 0, NOW(), NOW())`, userID) + require.NoError(t, err) + group := mustCreateGroup(t, client, &service.Group{Name: "platform-list-composite", Platform: service.PlatformComposite}) + _, err = client.ExecContext(ctx, ` +INSERT INTO composite_model_routes (group_id, public_model, target_platform) +VALUES ($1, 'future-model', 'future_platform')`, group.ID) + require.NoError(t, err) +} + +// 删除 CHECK 约束后,平台合法性由应用层保证:原生 SQL 写入前的 repository 校验。 +func TestUserPlatformQuotaRepository_RejectsUnregisteredPlatform(t *testing.T) { + ctx := context.Background() + tx := testEntTx(t) + txCtx := dbent.NewTxContext(ctx, tx) + client := tx.Client() + + userID := mustCreateUserForQuota(t, client) + repo := NewUserPlatformQuotaRepository(client) + daily := 5.0 + require.NoError(t, repo.UpsertForUser(txCtx, userID, []UserPlatformQuotaRecord{ + {UserID: userID, Platform: service.PlatformOpenCodeGo, DailyLimitUSD: &daily}, + })) + + for _, platform := range []string{"bogus", "moonshot", "Kimi", service.PlatformComposite} { + err := repo.BulkInsertInitial(txCtx, []UserPlatformQuotaRecord{ + {UserID: userID, Platform: service.PlatformAnthropic, DailyLimitUSD: &daily}, + {UserID: userID, Platform: platform, DailyLimitUSD: &daily}, + }) + require.ErrorContains(t, err, "is not allowed", platform) + err = repo.UpsertForUser(txCtx, userID, []UserPlatformQuotaRecord{ + {UserID: userID, Platform: platform, DailyLimitUSD: &daily}, + }) + require.ErrorContains(t, err, "is not allowed", platform) + } + + // 被拒绝的写入不产生任何副作用:既有记录保留、未新增任何行。 + got, err := repo.ListByUser(txCtx, userID) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, service.PlatformOpenCodeGo, got[0].Platform) + + // 三档全空的记录不写入,也不参与校验(与既有 configuredRecords 语义一致)。 + require.NoError(t, repo.BulkInsertInitial(txCtx, []UserPlatformQuotaRecord{{UserID: userID, Platform: "bogus"}})) +} + +func TestCompositeModelRouteRepository_RejectsUnregisteredTargetPlatform(t *testing.T) { + ctx := context.Background() + tx := testEntTx(t) + txCtx := dbent.NewTxContext(ctx, tx) + client := tx.Client() + + group := mustCreateGroup(t, client, &service.Group{Name: "platform-list-routes", Platform: service.PlatformComposite}) + repo := NewCompositeModelRouteRepository(client) + + route := &service.CompositeModelRoute{ + GroupID: group.ID, PublicModel: "glm-5", MatchType: "exact", TargetPlatform: service.PlatformZhipu, + UpstreamModel: "glm-5", Endpoint: "any", Priority: 100, Enabled: true, + } + require.NoError(t, repo.Create(txCtx, route)) + + bad := &service.CompositeModelRoute{ + GroupID: group.ID, PublicModel: "bogus-model", MatchType: "exact", TargetPlatform: "bogus", + UpstreamModel: "bogus-model", Endpoint: "any", Priority: 100, Enabled: true, + } + require.Error(t, repo.Create(txCtx, bad)) + + route.TargetPlatform = service.PlatformComposite + require.Error(t, repo.Update(txCtx, route)) + + routes, err := repo.ListByGroup(txCtx, group.ID, true) + require.NoError(t, err) + require.Len(t, routes, 1) + require.Equal(t, service.PlatformZhipu, routes[0].TargetPlatform) +} diff --git a/backend/internal/repository/user_platform_quota_repo.go b/backend/internal/repository/user_platform_quota_repo.go index 50e0660edaba..a24865074d3b 100644 --- a/backend/internal/repository/user_platform_quota_repo.go +++ b/backend/internal/repository/user_platform_quota_repo.go @@ -9,6 +9,7 @@ import ( dbent "github.com/Wei-Shaw/sub2api/ent" "github.com/Wei-Shaw/sub2api/ent/userplatformquota" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/timezone" "github.com/jackc/pgx/v5/pgconn" ) @@ -96,6 +97,9 @@ func (r *userPlatformQuotaRepository) BulkInsertInitial(ctx context.Context, rec if len(records) == 0 { return nil } + if err := validateQuotaPlatforms(records); err != nil { + return err + } client := clientFromContext(ctx, r.client) @@ -319,6 +323,9 @@ func monthlyMaybeReset(prevUsage float64, prevStart *time.Time, cost float64, no // 清空某平台的限额即软删该行并放弃其累计用量;之后再次配置限额会新建行、从新窗口起算。 func (r *userPlatformQuotaRepository) UpsertForUser(ctx context.Context, userID int64, records []UserPlatformQuotaRecord) error { records = configuredRecords(records) + if err := validateQuotaPlatforms(records); err != nil { + return err + } return r.withTx(ctx, func(txCtx context.Context, txClient *dbent.Client) error { platforms := make([]string, 0, len(records)) for _, rec := range records { @@ -489,6 +496,17 @@ func (r *userPlatformQuotaRepository) BatchSnapshotUsage(ctx context.Context, sn return nil } +// validateQuotaPlatforms 在原生 SQL 写入前校验平台须为平台清单中的具体平台, +// 取代迁移 242 删除的 user_platform_quotas_platform_check 约束(原生 SQL 不经过 ent Validate)。 +func validateQuotaPlatforms(records []UserPlatformQuotaRecord) error { + for _, rec := range records { + if !domain.IsConcretePlatform(rec.Platform) { + return fmt.Errorf("user_platform_quotas: platform %q is not allowed", rec.Platform) + } + } + return nil +} + // configuredRecords 过滤出至少配置了一档限额的记录;三档全空的记录视为未配置。 func configuredRecords(records []UserPlatformQuotaRecord) []UserPlatformQuotaRecord { out := make([]UserPlatformQuotaRecord, 0, len(records)) diff --git a/backend/internal/server/api_contract_test.go b/backend/internal/server/api_contract_test.go index 0865dd261ae5..ddfeabec08bf 100644 --- a/backend/internal/server/api_contract_test.go +++ b/backend/internal/server/api_contract_test.go @@ -863,7 +863,7 @@ func TestAPIContracts(t *testing.T) { "force_email_on_third_party_signup": false, "default_concurrency": 5, "default_balance": 1.25, - "default_platform_quotas": {"anthropic":{"daily":null,"weekly":null,"monthly":null},"antigravity":{"daily":null,"weekly":null,"monthly":null},"deepseek":{"daily":null,"weekly":null,"monthly":null},"gemini":{"daily":null,"weekly":null,"monthly":null},"grok":{"daily":null,"weekly":null,"monthly":null},"kimi":{"daily":null,"weekly":null,"monthly":null},"minimax":{"daily":null,"weekly":null,"monthly":null},"openai":{"daily":null,"weekly":null,"monthly":null},"opencode_go":{"daily":null,"weekly":null,"monthly":null},"typesafe":{"daily":null,"weekly":null,"monthly":null},"zhipu":{"daily":null,"weekly":null,"monthly":null}}, + "default_platform_quotas": {"anthropic":{"daily":null,"weekly":null,"monthly":null},"antigravity":{"daily":null,"weekly":null,"monthly":null},"cline":{"daily":null,"weekly":null,"monthly":null},"command_code":{"daily":null,"weekly":null,"monthly":null},"deepseek":{"daily":null,"weekly":null,"monthly":null},"gemini":{"daily":null,"weekly":null,"monthly":null},"grok":{"daily":null,"weekly":null,"monthly":null},"kimi":{"daily":null,"weekly":null,"monthly":null},"minimax":{"daily":null,"weekly":null,"monthly":null},"openai":{"daily":null,"weekly":null,"monthly":null},"opencode_go":{"daily":null,"weekly":null,"monthly":null},"typesafe":{"daily":null,"weekly":null,"monthly":null},"zhipu":{"daily":null,"weekly":null,"monthly":null}}, "auth_source_default_email_platform_quotas": null, "auth_source_default_github_platform_quotas": null, "auth_source_default_google_platform_quotas": null, @@ -1194,7 +1194,7 @@ func TestAPIContracts(t *testing.T) { "purchase_subscription_url": "", "table_default_page_size": 20, "table_page_size_options": [10, 20, 50], - "default_platform_quotas": {"anthropic":{"daily":null,"weekly":null,"monthly":null},"antigravity":{"daily":null,"weekly":null,"monthly":null},"deepseek":{"daily":null,"weekly":null,"monthly":null},"gemini":{"daily":null,"weekly":null,"monthly":null},"grok":{"daily":null,"weekly":null,"monthly":null},"kimi":{"daily":null,"weekly":null,"monthly":null},"minimax":{"daily":null,"weekly":null,"monthly":null},"openai":{"daily":null,"weekly":null,"monthly":null},"opencode_go":{"daily":null,"weekly":null,"monthly":null},"typesafe":{"daily":null,"weekly":null,"monthly":null},"zhipu":{"daily":null,"weekly":null,"monthly":null}}, + "default_platform_quotas": {"anthropic":{"daily":null,"weekly":null,"monthly":null},"antigravity":{"daily":null,"weekly":null,"monthly":null},"cline":{"daily":null,"weekly":null,"monthly":null},"command_code":{"daily":null,"weekly":null,"monthly":null},"deepseek":{"daily":null,"weekly":null,"monthly":null},"gemini":{"daily":null,"weekly":null,"monthly":null},"grok":{"daily":null,"weekly":null,"monthly":null},"kimi":{"daily":null,"weekly":null,"monthly":null},"minimax":{"daily":null,"weekly":null,"monthly":null},"openai":{"daily":null,"weekly":null,"monthly":null},"opencode_go":{"daily":null,"weekly":null,"monthly":null},"typesafe":{"daily":null,"weekly":null,"monthly":null},"zhipu":{"daily":null,"weekly":null,"monthly":null}}, "auth_source_default_email_platform_quotas": null, "auth_source_default_github_platform_quotas": null, "auth_source_default_google_platform_quotas": null, diff --git a/backend/internal/server/routes/admin.go b/backend/internal/server/routes/admin.go index 397eaafb90f9..ffaa02daf49f 100644 --- a/backend/internal/server/routes/admin.go +++ b/backend/internal/server/routes/admin.go @@ -132,7 +132,6 @@ func RegisterAdminRoutes( registerSystemPromptRoutes(admin, h) // 推理恢复全局开关 - registerReasoningRecoveryRoutes(admin, h) registerCodexGatewayBorrowRoutes(admin, h) registerPelicanTestRoutes(admin, h) @@ -177,14 +176,6 @@ func registerSystemPromptRoutes(admin *gin.RouterGroup, h *handler.Handlers) { } } -func registerReasoningRecoveryRoutes(admin *gin.RouterGroup, h *handler.Handlers) { - recovery := admin.Group("/reasoning-recovery") - { - recovery.GET("", h.Admin.ReasoningRecovery.Get) - recovery.PUT("", h.Admin.ReasoningRecovery.Save) - } -} - func registerPromptAuditRoutes(admin *gin.RouterGroup, h *handler.Handlers) { promptAudit := admin.Group("/prompt-audit") { diff --git a/backend/internal/server/routes/gateway.go b/backend/internal/server/routes/gateway.go index 722b04997f18..787a88f1ef62 100644 --- a/backend/internal/server/routes/gateway.go +++ b/backend/internal/server/routes/gateway.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/handler" pkghttputil "github.com/Wei-Shaw/sub2api/internal/pkg/httputil" "github.com/Wei-Shaw/sub2api/internal/pkg/requestmodel" @@ -47,22 +48,15 @@ func RegisterGatewayRoutes( groupModelAllowlist := middleware.GroupModelAllowlist() isOpenAIResponsesCompatibleGatewayPlatform := func(c *gin.Context) bool { - switch getGroupPlatform(c) { - case service.PlatformOpenAI, service.PlatformGrok, - service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, - service.PlatformMiniMax, service.PlatformOpenCodeGo: - // 国产 OpenAI 兼容供应商与 openai/grok 一样经 OpenAI 网关转发。 - return true - default: - return false - } + // openai、grok 与多协议 API Key 供应商经 OpenAI 网关转发(平台清单)。 + return domain.UsesOpenAIGateway(getGroupPlatform(c)) } countTokensHandler := func(c *gin.Context) { - switch getGroupPlatform(c) { - case service.PlatformOpenAI, service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, service.PlatformMiniMax, service.PlatformOpenCodeGo: - h.OpenAIGateway.CountTokens(c) - case service.PlatformGrok: + switch platform := getGroupPlatform(c); { + case platform == service.PlatformGrok: h.OpenAIGateway.GrokCountTokens(c) + case domain.UsesOpenAIGateway(platform): + h.OpenAIGateway.CountTokens(c) default: h.Gateway.CountTokens(c) } diff --git a/backend/internal/server/routes/reasoning_recovery_routes_test.go b/backend/internal/server/routes/reasoning_recovery_routes_test.go deleted file mode 100644 index 1cfa3a46b9fc..000000000000 --- a/backend/internal/server/routes/reasoning_recovery_routes_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package routes - -import ( - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/Wei-Shaw/sub2api/internal/handler" - adminhandler "github.com/Wei-Shaw/sub2api/internal/handler/admin" - servermiddleware "github.com/Wei-Shaw/sub2api/internal/server/middleware" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" -) - -// Both reasoning recovery routes sit behind admin authentication, and neither -// asks for a TOTP step-up: an admin request reaches the handler, which here has -// no service and answers 503. -func TestReasoningRecoveryRoutesRequireAdminAuthenticationWithoutStepUp(t *testing.T) { - gin.SetMode(gin.TestMode) - router := gin.New() - handlers := &handler.Handlers{Admin: &handler.AdminHandlers{ - ReasoningRecovery: adminhandler.NewReasoningRecoveryHandler(nil), - }} - adminAuth := servermiddleware.AdminAuthMiddleware(func(c *gin.Context) { - switch c.GetHeader("Authorization") { - case "": - servermiddleware.AbortWithError(c, http.StatusUnauthorized, "UNAUTHORIZED", "Authorization required") - case "Bearer admin-token": - c.Next() - default: - servermiddleware.AbortWithError(c, http.StatusForbidden, "FORBIDDEN", "Admin access required") - } - }) - auditLog := servermiddleware.AuditLogMiddleware(func(c *gin.Context) { c.Next() }) - stepUp := servermiddleware.StepUpAuthMiddleware(func(c *gin.Context) { - servermiddleware.AbortWithError(c, http.StatusPreconditionRequired, "STEP_UP_REQUIRED", "TOTP step-up required") - }) - RegisterAdminRoutes(router.Group("/api/v1"), handlers, adminAuth, auditLog, stepUp, nil, nil) - - for _, route := range []struct{ method, body string }{ - {http.MethodGet, ""}, - {http.MethodPut, `{"enabled":false}`}, - } { - for _, testCase := range []struct { - name string - auth string - wantStatus int - }{ - {name: "unauthenticated", wantStatus: http.StatusUnauthorized}, - {name: "non-admin", auth: "Bearer user-token", wantStatus: http.StatusForbidden}, - {name: "admin", auth: "Bearer admin-token", wantStatus: http.StatusServiceUnavailable}, - } { - t.Run(route.method+" "+testCase.name, func(t *testing.T) { - recorder := httptest.NewRecorder() - request := httptest.NewRequest(route.method, "/api/v1/admin/reasoning-recovery", strings.NewReader(route.body)) - if testCase.auth != "" { - request.Header.Set("Authorization", testCase.auth) - } - router.ServeHTTP(recorder, request) - require.Equal(t, testCase.wantStatus, recorder.Code) - }) - } - } -} diff --git a/backend/internal/service/account.go b/backend/internal/service/account.go index e11f9d808e69..a1cc0a0ff224 100644 --- a/backend/internal/service/account.go +++ b/backend/internal/service/account.go @@ -335,7 +335,7 @@ func (a *Account) IsCNProvider() bool { // openai/grok 原生走 OpenAI 网关;国产供应商同为 OpenAI Chat Completions // 兼容上游,也经 OpenAI 网关转发。OpenCode 同样经 OpenAI 网关按模型分流。 func (a *Account) IsOpenAICompatible() bool { - return a != nil && (a.Platform == PlatformOpenAI || a.Platform == PlatformGrok || a.IsCNProvider() || a.IsOpenCodeGo()) + return a != nil && domain.UsesOpenAIGateway(a.Platform) } func (a *Account) GeminiOAuthType() string { @@ -1416,7 +1416,7 @@ func (a *Account) IsOpenAIApiKey() bool { // 适用 openai、国产 OpenAI 兼容供应商(kimi/zhipu/deepseek)与 OpenCode Go; // grok 走 GetGrokBaseURL,此处对 grok 返回 "" 以保持原有行为。 func (a *Account) GetOpenAIBaseURL() string { - if !a.IsOpenAI() && !a.IsCNProvider() && !a.IsOpenCodeGo() { + if !a.IsOpenAI() && !a.IsMultiProtocolAPIKey() { return "" } if a.IsMultiProtocolAPIKey() && a.IsAdaptiveAPIProtocol() { @@ -1431,27 +1431,11 @@ func (a *Account) GetOpenAIBaseURL() string { return baseURL } } - // 平台默认 base_url:CN 供应商按 account_mode 选择 payg / coding 默认值。 - switch a.Platform { - case PlatformKimi: - if a.GetAccountMode() == AccountModeCoding { - return DefaultKimiCodingBaseURL - } - return DefaultKimiPayGBaseURL - case PlatformZhipu: - if a.GetAccountMode() == AccountModeCoding { - return DefaultZhipuCodingBaseURL - } - return DefaultZhipuPayGBaseURL - case PlatformDeepseek: - return DefaultDeepseekBaseURL - case PlatformMiniMax: - return DefaultMiniMaxBaseURL - case PlatformOpenCodeGo: - return a.openCodeDefaultChatBaseURL() - default: - return "https://api.openai.com" + // 平台默认 base_url:多协议供应商按 account_mode 查 provider profile。 + if baseURL := a.defaultProviderBaseURL(APIProtocolChatCompletions); baseURL != "" { + return baseURL } + return "https://api.openai.com" } // GetAccountMode 返回国产供应商账号的接入模式(payg / coding);非国产供应商或未设置时 @@ -1492,7 +1476,8 @@ func (a *Account) GetAPIProtocol() string { case APIProtocolChatCompletions: return APIProtocolChatCompletions } - if a.IsOpenCodeGo() { + // 按模型分流的供应商(多模型聚合平台)未显式配置时默认 adaptive。 + if a.routesByModel() { return APIProtocolAdaptive } return APIProtocolChatCompletions @@ -1502,15 +1487,7 @@ func (a *Account) GetAPIProtocol() string { // DeepSeek 官方为 /responses(无 /v1);Kimi 按量付费与 Coding Plan 均为 // /v1/responses(moonshot.cn / kimi.com/coding);MiniMax 为 /v1/responses。 func (a *Account) SupportsNativeCNResponses() bool { - if a == nil { - return false - } - switch a.Platform { - case PlatformDeepseek, PlatformKimi, PlatformMiniMax, PlatformOpenCodeGo: - return true - default: - return false - } + return a.providerSupportsProtocol(APIProtocolResponses) } // UsesNativeCNResponses 报告当前账号是否应按原生 Responses 协议转发 @@ -1551,48 +1528,7 @@ func (a *Account) GetCNProtocolBaseURL(protocol string) string { } } } - return a.defaultCNProtocolBaseURL(protocol) -} - -func (a *Account) defaultCNProtocolBaseURL(protocol string) string { - switch protocol { - case APIProtocolAnthropic: - switch a.Platform { - case PlatformKimi: - if a.GetAccountMode() == AccountModeCoding { - return DefaultKimiCodingAnthropicBaseURL - } - return DefaultKimiPayGAnthropicBaseURL - case PlatformZhipu: - return DefaultZhipuAnthropicBaseURL - case PlatformDeepseek: - return DefaultDeepseekAnthropicBaseURL - case PlatformMiniMax: - return DefaultMiniMaxAnthropicBaseURL - case PlatformOpenCodeGo: - return a.openCodeDefaultAnthropicBaseURL() - } - case APIProtocolChatCompletions, APIProtocolResponses: - switch a.Platform { - case PlatformKimi: - if a.GetAccountMode() == AccountModeCoding { - return DefaultKimiCodingBaseURL - } - return DefaultKimiPayGBaseURL - case PlatformZhipu: - if a.GetAccountMode() == AccountModeCoding { - return DefaultZhipuCodingBaseURL - } - return DefaultZhipuPayGBaseURL - case PlatformDeepseek: - return DefaultDeepseekBaseURL - case PlatformMiniMax: - return DefaultMiniMaxBaseURL - case PlatformOpenCodeGo: - return a.openCodeDefaultChatBaseURL() - } - } - return "" + return a.defaultProviderBaseURL(protocol) } // IsAnthropicProtocol 报告账号是否以原生 Anthropic 协议接入上游 @@ -1616,23 +1552,7 @@ func (a *Account) GetAnthropicProtocolBaseURL() string { return baseURL } } - switch a.Platform { - case PlatformKimi: - if a.GetAccountMode() == AccountModeCoding { - return DefaultKimiCodingAnthropicBaseURL - } - return DefaultKimiPayGAnthropicBaseURL - case PlatformZhipu: - return DefaultZhipuAnthropicBaseURL - case PlatformDeepseek: - return DefaultDeepseekAnthropicBaseURL - case PlatformMiniMax: - return DefaultMiniMaxAnthropicBaseURL - case PlatformOpenCodeGo: - return a.openCodeDefaultAnthropicBaseURL() - default: - return "" - } + return a.defaultProviderBaseURL(APIProtocolAnthropic) } // GetOpenAIFormatBaseURL 返回供 OpenAI 格式端点(/v1/models、/v1/chat/completions @@ -1644,26 +1564,10 @@ func (a *Account) GetOpenAIFormatBaseURL() string { if a == nil || !a.IsAnthropicProtocol() { return a.GetOpenAIBaseURL() } - switch a.Platform { - case PlatformKimi: - if a.GetAccountMode() == AccountModeCoding { - return DefaultKimiCodingBaseURL - } - return DefaultKimiPayGBaseURL - case PlatformZhipu: - if a.GetAccountMode() == AccountModeCoding { - return DefaultZhipuCodingBaseURL - } - return DefaultZhipuPayGBaseURL - case PlatformDeepseek: - return DefaultDeepseekBaseURL - case PlatformMiniMax: - return DefaultMiniMaxBaseURL - case PlatformOpenCodeGo: - return a.openCodeDefaultChatBaseURL() - default: - return a.GetOpenAIBaseURL() + if baseURL := a.defaultProviderBaseURL(APIProtocolChatCompletions); baseURL != "" { + return baseURL } + return a.GetOpenAIBaseURL() } // GetCNAPIKey 返回国产 OpenAI 兼容供应商账号的 api_key 凭据(kimi/zhipu/deepseek)。 @@ -1675,7 +1579,8 @@ func (a *Account) GetCNAPIKey() string { return a.GetCredential("api_key") } -// GetCodingPlanProvider 根据 base_url 识别 Coding Plan 供应商(kimi / zhipu / minimax), +// GetCodingPlanProvider 根据 base_url 识别 Coding Plan 供应商(kimi / zhipu / minimax; +// OpenCode Go 订阅与官方主机上的 Command Code、Cline 账号按平台识别), // 用于路由到对应的额度查询端点。非 coding 模式或无法识别时返回空串。 // 只认官方域名:自定义中转不得把第三方 Key 发往厂商官方额度端点。 func (a *Account) GetCodingPlanProvider() string { @@ -1685,6 +1590,18 @@ func (a *Account) GetCodingPlanProvider() string { if a.IsOpenCodeGoPlan() { return PlatformOpenCodeGo } + if a.IsCommandCode() { + if a.commandCodeUsageSupported() { + return PlatformCommandCode + } + return "" + } + if a.IsCline() { + if a.clineAccountAPISupported() { + return PlatformCline + } + return "" + } if a.GetAccountMode() != AccountModeCoding { return "" } @@ -2166,17 +2083,6 @@ func (a *Account) IsOveragesEnabled() bool { // supportsOpenAIReasoningRecovery reports whether the bounded HTTP // reasoning-signature recovery applies to this kind of account, including // passthrough accounts. Whether it runs is the global switch. -func (a *Account) supportsOpenAIReasoningRecovery() bool { - if a == nil || !a.IsOpenAI() { - return false - } - switch a.Type { - case AccountTypeAPIKey, AccountTypeOAuth, AccountTypeSetupToken: - return true - default: - return false - } -} // IsOpenAIPassthroughEnabled 返回 OpenAI 账号是否启用"自动透传(仅替换认证)"。 // diff --git a/backend/internal/service/account_header_override.go b/backend/internal/service/account_header_override.go index 7f05e253ac98..e807cac15df8 100644 --- a/backend/internal/service/account_header_override.go +++ b/backend/internal/service/account_header_override.go @@ -77,10 +77,10 @@ func (a *Account) IsHeaderOverrideEligible() bool { if a == nil { return false } - switch a.Platform { - case PlatformAnthropic, PlatformOpenAI, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo: + switch { + case a.Platform == PlatformAnthropic, a.Platform == PlatformOpenAI, IsMultiProtocolAPIKeyProvider(a.Platform): return a.Type == AccountTypeAPIKey - case PlatformGrok: + case a.Platform == PlatformGrok: return a.Type == AccountTypeAPIKey || a.Type == AccountTypeOAuth default: return false diff --git a/backend/internal/service/account_scheduling_threshold_eval.go b/backend/internal/service/account_scheduling_threshold_eval.go index 7957a20ea061..010d9944cc3e 100644 --- a/backend/internal/service/account_scheduling_threshold_eval.go +++ b/backend/internal/service/account_scheduling_threshold_eval.go @@ -58,7 +58,7 @@ func EvaluateAccountSchedulingThreshold(account *Account, thresholds map[string] winner = pickLatestResetSchedulingCandidate(anthropicThresholdCandidates(account), threshold, now) case PlatformGrok: winner = pickLatestResetSchedulingCandidate(grokThresholdCandidates(account), threshold, now) - case PlatformKimi, PlatformZhipu, PlatformMiniMax, PlatformOpenCodeGo: + case PlatformKimi, PlatformZhipu, PlatformMiniMax, PlatformOpenCodeGo, PlatformCommandCode: winner = pickLatestResetSchedulingCandidate(cnProviderThresholdCandidates(account, decision.Platform), threshold, now) default: return decision @@ -337,25 +337,33 @@ func grokThresholdCandidates(account *Account) []*accountSchedulingThresholdCand } } -// cnProviderThresholdCandidates 读取国产供应商 Coding Plan 账号的 5h / weekly 滚动窗口 -// 用量快照(由 CNProviderQuotaService 写入 account.Extra,键形如 -// _5h_used_percent / _weekly_reset_at)。payg 账号无此快照, +// cnProviderThresholdCandidates 读取国产供应商 Coding Plan、OpenCode Go 与 Command Code +// 账号的 5h / weekly(及月度)窗口用量快照(由 CNProviderQuotaService 写入 account.Extra, +// 键形如 _5h_used_percent / _weekly_reset_at)。payg 账号无此快照, // 候选为空 → 不触发阈值停调(余额型走余额检测)。与 openai 的快照驱动停调一致: // 仅当用量超阈值且窗口尚未重置时才停调。 func cnProviderThresholdCandidates(account *Account, provider string) []*accountSchedulingThresholdCandidate { if account == nil || len(account.Extra) == 0 { return nil } + if provider == PlatformCommandCode && !commandCodeWindowsBinding(account.Extra) { + return nil + } candidates := []*accountSchedulingThresholdCandidate{ cnThresholdCandidate(account.Extra, provider, "5h"), cnThresholdCandidate(account.Extra, provider, "weekly"), } - if provider == PlatformOpenCodeGo { + if cnQuotaHasMonthlyWindow(provider) { candidates = append(candidates, cnThresholdCandidate(account.Extra, provider, "monthly")) } return candidates } +// cnQuotaHasMonthlyWindow 报告供应商的额度快照是否含月度窗口(OpenCode Go、Command Code)。 +func cnQuotaHasMonthlyWindow(provider string) bool { + return provider == PlatformOpenCodeGo || provider == PlatformCommandCode +} + func cnThresholdCandidate(extra map[string]any, provider, window string) *accountSchedulingThresholdCandidate { var usedKey, resetKey string switch window { diff --git a/backend/internal/service/account_service.go b/backend/internal/service/account_service.go index 379c847c9560..96302be9a26a 100644 --- a/backend/internal/service/account_service.go +++ b/backend/internal/service/account_service.go @@ -532,10 +532,12 @@ func (s *AccountService) TestCredentials(ctx context.Context, id int64) error { case PlatformTypeSafe: // TypeSafe credentials are API keys; inference failures drive health and cooldown state. return nil - case PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo: - // 国产 OpenAI 兼容供应商与 OpenCode:凭证为 API Key,实际可用性经余额/额度探测与转发路径验证。 - return nil default: + if IsMultiProtocolAPIKeyProvider(account.Platform) { + // 多协议 API Key 供应商(国产厂商与聚合平台):凭证为 API Key,实际可用性 + // 经余额/额度探测与转发路径验证。 + return nil + } return fmt.Errorf("unsupported platform: %s", account.Platform) } } diff --git a/backend/internal/service/account_test_logging.go b/backend/internal/service/account_test_logging.go new file mode 100644 index 000000000000..f1f5501d5745 --- /dev/null +++ b/backend/internal/service/account_test_logging.go @@ -0,0 +1,48 @@ +package service + +import ( + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/Wei-Shaw/sub2api/internal/util/logredact" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" +) + +const accountTestBackgroundKey = "account_test_background" + +// Bind attribution before lookup so missing accounts are identifiable too. The +// request logger retains HTTP request IDs or the caller's background correlation. +func initAccountTestLogger(c *gin.Context, accountID int64, modelID, mode string) { + source := "http" + if c.GetBool(accountTestBackgroundKey) { + source = "background" + } + ctx := c.Request.Context() + l := logger.FromContext(ctx).With( + zap.String("component", "service.account_test"), + zap.Int64("account_id", accountID), + zap.String("test_id", uuid.NewString()), + zap.String("test_source", source), + zap.String("requested_model", modelID), + zap.String("test_mode", normalizeAccountTestMode(mode)), + ) + c.Request = c.Request.WithContext(logger.IntoContext(ctx, l)) +} + +func bindAccountTestPlatform(c *gin.Context, account *Account) { + ctx := c.Request.Context() + l := logger.FromContext(ctx).With( + zap.String("platform", account.Platform), + zap.String("account_type", account.Type), + ) + c.Request = c.Request.WithContext(logger.IntoContext(ctx, l)) +} + +func logAccountTestError(c *gin.Context, message string) { + // Redact known credential fields before bounding text; truncating first can + // cut a quoted credential in half and prevent the redactor from matching it. + message = logredact.RedactText(message, "api_key", "authorization") + logger.FromContext(c.Request.Context()).Warn("account_test.failed", + zap.String("error", truncateString(message, 2048)), + ) +} diff --git a/backend/internal/service/account_test_logging_test.go b/backend/internal/service/account_test_logging_test.go new file mode 100644 index 000000000000..8defd03563d9 --- /dev/null +++ b/backend/internal/service/account_test_logging_test.go @@ -0,0 +1,93 @@ +//go:build unit + +package service + +import ( + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +type accountTestLoggingRepo struct { + AccountRepository + account *Account +} + +func (r *accountTestLoggingRepo) GetByID(context.Context, int64) (*Account, error) { + if r.account == nil { + return nil, errors.New("missing") + } + return r.account, nil +} + +func TestAccountTestFailureLogAttribution(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, background := range []bool{false, true} { + for _, missing := range []bool{false, true} { + core, logs := observer.New(zap.WarnLevel) + ctx := logger.IntoContext(context.Background(), zap.New(core).With(zap.String("request_id", "req-test"))) + repo := &accountTestLoggingRepo{} + if !missing { + repo.account = &Account{ID: 77, Platform: PlatformAnthropic, Type: AccountTypeAPIKey} + } + svc := &AccountTestService{accountRepo: repo} + if background { + result, err := svc.RunTestBackground(ctx, 77, "test-model") + require.NoError(t, err) + require.Equal(t, "failed", result.Status) + require.NotEmpty(t, result.ErrorMessage) + } else { + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest("POST", "/accounts/77/test", nil).WithContext(ctx) + err := svc.TestAccountConnection(c, 77, "test-model", "private prompt", "") + require.Error(t, err) + require.Contains(t, rec.Body.String(), `"type":"error"`) + } + entries := logs.FilterMessage("account_test.failed").All() + require.Len(t, entries, 1) + fields := entries[0].ContextMap() + require.Equal(t, int64(77), fields["account_id"]) + require.Equal(t, "req-test", fields["request_id"]) + require.Equal(t, "test-model", fields["requested_model"]) + require.NotEmpty(t, fields["test_id"]) + if background { + require.Equal(t, "background", fields["test_source"]) + } else { + require.Equal(t, "http", fields["test_source"]) + } + if !missing { + require.Equal(t, PlatformAnthropic, fields["platform"]) + } + require.NotContains(t, fields, "prompt") + } + } +} + +func TestAccountTestErrorLogRedaction(t *testing.T) { + core, logs := observer.New(zap.WarnLevel) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest("POST", "/test", nil).WithContext(logger.IntoContext(context.Background(), zap.New(core))) + initAccountTestLogger(c, 1, "", "") + message := `API returned 403: {"api_key":"secret-key","access_token":"secret-token","message":"denied"}` + err := (&AccountTestService{}).sendErrorAndEnd(c, message) + require.EqualError(t, err, message) + entry := logs.FilterMessage("account_test.failed").All()[0] + logged, ok := entry.ContextMap()["error"].(string) + require.True(t, ok) + require.NotContains(t, logged, "secret-key") + require.NotContains(t, logged, "secret-token") + require.Contains(t, logged, "denied") + logAccountTestError(c, strings.Repeat("x", 10000)) + last, ok := logs.All()[1].ContextMap()["error"].(string) + require.True(t, ok) + require.Less(t, len(last), 2100) +} diff --git a/backend/internal/service/account_test_reasoning.go b/backend/internal/service/account_test_reasoning.go index 9fe7acf40f73..5918360155d3 100644 --- a/backend/internal/service/account_test_reasoning.go +++ b/backend/internal/service/account_test_reasoning.go @@ -36,7 +36,8 @@ func AccountTestReasoningOptions(account *Account, model string) (efforts []stri return nil, "" } metadata, known := account.GetUpstreamModelMetadata(model) - if account.Platform == PlatformAnthropic { + newMultiProtocol := account.Type == AccountTypeAPIKey && (account.Platform == PlatformCommandCode || account.Platform == PlatformCline) + if account.Platform == PlatformAnthropic || (newMultiProtocol && account.routesByModel() && account.resolveModelRoutedProtocol(model) == APIProtocolAnthropic) || (newMultiProtocol && account.RoutesProtocolByInbound() && account.GetAPIProtocol() == APIProtocolAnthropic) { // Claude effort levels are a property of the model; the account's // upstream metadata can only rule the model out. if known && metadata.Reasoning != nil && !*metadata.Reasoning { @@ -87,16 +88,17 @@ func accountTestGrokUpstreamModel(model string) string { } func accountTestSupportsReasoningWire(account *Account, model string) bool { + newMultiProtocol := account.Type == AccountTypeAPIKey && (account.Platform == PlatformCommandCode || account.Platform == PlatformCline) switch { - case account.IsOpenCodeGo(): - protocol := openCodeGoNativeProtocol(account, model) - return protocol == APIProtocolResponses || protocol == APIProtocolChatCompletions - case account.IsCNProvider(): + case account.routesByModel(): + protocol := account.resolveModelRoutedProtocol(model) + return protocol == APIProtocolResponses || protocol == APIProtocolChatCompletions || (newMultiProtocol && protocol == APIProtocolAnthropic) + case account.RoutesProtocolByInbound(): protocol := account.GetAPIProtocol() // The adaptive test verifies all native endpoints, including Messages, // whose effort contract differs. Never silently omit a chosen effort on // one leg of that test. - return protocol == APIProtocolResponses || protocol == APIProtocolChatCompletions + return protocol == APIProtocolResponses || protocol == APIProtocolChatCompletions || (newMultiProtocol && protocol == APIProtocolAnthropic) case account.IsOpenAI(): return true case account.Platform == PlatformGrok: diff --git a/backend/internal/service/account_test_reasoning_test.go b/backend/internal/service/account_test_reasoning_test.go index 771133572681..edfeb906ff0d 100644 --- a/backend/internal/service/account_test_reasoning_test.go +++ b/backend/internal/service/account_test_reasoning_test.go @@ -43,6 +43,27 @@ func TestAccountTestReasoningFinalPayload(t *testing.T) { require.NotContains(t, defaultPayload, "reasoning") } +func TestAccountTestReasoningNewPlatformsUseDeclaredWireCapabilities(t *testing.T) { + for _, platform := range []string{PlatformCommandCode, PlatformCline} { + t.Run(platform, func(t *testing.T) { + account := &Account{Platform: platform, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_protocol": APIProtocolAnthropic, "model_mapping": map[string]any{"friendly": "claude-sonnet-5-5"}}} + levels, _ := AccountTestReasoningOptions(account, "friendly") + require.Contains(t, levels, "high") + require.NotContains(t, levels, "ultra") + require.NoError(t, ValidateAccountTestReasoning(account, "friendly", "default", "high")) + context, _ := gin.CreateTestContext(httptest.NewRecorder()) + context.Set(accountTestReasoningContextKey, "high") + payload, err := createTestPayload("claude-sonnet-5-5", "fixture") + require.NoError(t, err) + body, err := accountTestClaudeBody(context, payload, "claude-sonnet-5-5") + require.NoError(t, err) + require.Equal(t, "high", gjson.GetBytes(body, "output_config.effort").String()) + levels, _ = AccountTestReasoningOptions(account, "unknown-model") + require.Empty(t, levels) + }) + } +} + func TestAccountTestReasoningDoesNotAdvertiseUnsupportedProtocol(t *testing.T) { // Anthropic accounts transmit output_config.effort; see the Claude tests below. for _, platform := range []string{PlatformGemini, PlatformDeepseek} { diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index a0b47c38d852..64b65c839d5e 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -434,6 +434,7 @@ func (s *AccountTestService) TestAccountConnection(c *gin.Context, accountID int return s.sendErrorAndEnd(c, err.Error()) } c.Set(accountTestPromptContextKey, prompt) + initAccountTestLogger(c, accountID, modelID, mode) ctx := c.Request.Context() testOpts := firstAccountTestOptions(opts) @@ -447,6 +448,8 @@ func (s *AccountTestService) TestAccountConnection(c *gin.Context, accountID int return s.sendErrorAndEnd(c, err.Error()) } c.Set(accountTestReasoningContextKey, testOpts.ReasoningEffort) + bindAccountTestPlatform(c, account) + ctx = c.Request.Context() // Synthetic UI load-test accounts exercise the real SSE parsing and modal // interactions, but intentionally do not send their placeholder credentials @@ -470,10 +473,8 @@ func (s *AccountTestService) TestAccountConnection(c *gin.Context, accountID int ObserveQuotaAccount(ctx, account.ID) defer finishObservation() } - if account.IsOpenCodeGo() { - return s.testOpenCodeGoConnection(c, account, modelID, prompt) - } - if account.IsCNProvider() { + // 按入站协议分流的多协议供应商(国产厂商等):按账号协议选测试路径。 + if account.RoutesProtocolByInbound() { switch account.GetAPIProtocol() { case APIProtocolAdaptive: return s.testCNProviderAdaptiveConnection(c, account, modelID, prompt) @@ -502,6 +503,10 @@ func (s *AccountTestService) TestAccountConnection(c *gin.Context, accountID int return s.routeAntigravityTest(c, account, modelID, prompt) } + // 按模型分流的多模型聚合平台(OpenCode、Command Code 等)。 + if account.routesByModel() { + return s.testModelRoutedAccountConnection(c, account, modelID, prompt) + } if account.IsTypeSafe() { return s.testTypeSafeAccountConnection(c, account, prompt) } @@ -509,8 +514,59 @@ func (s *AccountTestService) TestAccountConnection(c *gin.Context, accountID int return s.testClaudeAccountConnection(c, account, modelID) } +// testModelRoutedAccountConnection probes the native endpoint for the selected +// model on providers that route by model (see ProviderRoutingByModel). Adaptive +// accounts (the default) follow the provider's protocol rules, e.g. OpenCode Go: +// grok/gpt/muse-spark → Responses, minimax/qwen → Anthropic, everything else +// (including deepseek-v4-flash) → Chat Completions. A pinned api_protocol +// overrides that catalog. Falling through to the generic Claude tester used +// credentials.base_url + /v1/messages?beta=true, which 404s as HTML on +// https://opencode.ai/zen/go/v1/v1/messages. +func (s *AccountTestService) testModelRoutedAccountConnection(c *gin.Context, account *Account, modelID string, prompt string) error { + testModelID := strings.TrimSpace(modelID) + if testModelID == "" { + testModelID = account.providerDefaultTestModel() + } + if testModelID == "" { + testModelID = openai.DefaultTestModel + } + testModelID = account.GetMappedModel(testModelID) + if account.IsOpenCodeGo() && IsOpenCodeUnsupportedModel(testModelID) { + return fmt.Errorf("model %q is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)", testModelID) + } + // 与网关同一判定(含上游模型目录);测试没有入站协议,取模型的首选协议。 + protocol := account.resolveModelRoutedProtocol(testModelID) + if s.openaiGatewayService != nil { + protocol = s.openaiGatewayService.resolveUpstreamProtocolFor(c.Request.Context(), account, "", testModelID) + } + switch protocol { + case APIProtocolAnthropic: + return s.testCNProviderAnthropicConnection(c, account, testModelID) + case APIProtocolResponses: + return s.testModelRoutedResponsesConnection(c, account, testModelID) + default: + return s.testCNProviderChatCompletionsConnection(c, account, testModelID, prompt) + } +} + +func (s *AccountTestService) testModelRoutedResponsesConnection(c *gin.Context, account *Account, testModelID string) error { + authToken := strings.TrimSpace(account.GetOpenAIProtocolAPIKey()) + if authToken == "" { + return s.sendErrorAndEnd(c, "No API key available") + } + c.Writer.Header().Set("Content-Type", "text/event-stream") + c.Writer.Header().Set("Cache-Control", "no-cache") + c.Writer.Header().Set("Connection", "keep-alive") + c.Writer.Header().Set("X-Accel-Buffering", "no") + c.Writer.Flush() + s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID}) + return s.testCNProviderAdaptiveResponsesConnection(c, account, testModelID, authToken) +} func (s *AccountTestService) testCNProviderChatCompletionsConnection(c *gin.Context, account *Account, modelID string, prompt string) error { testModelID := strings.TrimSpace(modelID) + if testModelID == "" { + testModelID = account.providerDefaultTestModel() + } if testModelID == "" { testModelID = openai.DefaultTestModel } @@ -2230,6 +2286,7 @@ func (s *AccountTestService) testOpenAIChatCompletionsConnection( // 账号级请求头覆写:测试请求与真实转发保持一致的最终头 account.ApplyHeaderOverrides(req.Header) + applyOpenCodeSessionHeader(c, account, apiURL, req.Header, payloadBytes) proxyURL := "" if account.ProxyID != nil && account.Proxy != nil { @@ -3227,7 +3284,7 @@ func accountTestWireHeaders(header http.Header) map[string]string { // sendErrorAndEnd sends an error event and ends the stream func (s *AccountTestService) sendErrorAndEnd(c *gin.Context, errorMsg string) error { - log.Printf("Account test error: %s", errorMsg) + logAccountTestError(c, errorMsg) s.sendEvent(c, TestEvent{Type: "error", Error: errorMsg}) return fmt.Errorf("%s", errorMsg) } @@ -3259,6 +3316,7 @@ func (s *AccountTestService) RunTestBackgroundDetailed(ctx context.Context, acco ginCtx.Request = (&http.Request{}).WithContext(ctx) timingWriter := &accountTestTimingWriter{ResponseWriter: ginCtx.Writer, startedAt: startedAt} ginCtx.Writer = timingWriter + ginCtx.Set(accountTestBackgroundKey, true) testErr := s.TestAccountConnection(ginCtx, accountID, modelID, "", AccountTestModeDefault) diff --git a/backend/internal/service/account_test_service_cn_adaptive.go b/backend/internal/service/account_test_service_cn_adaptive.go index 079ac6857ccd..4388842ef832 100644 --- a/backend/internal/service/account_test_service_cn_adaptive.go +++ b/backend/internal/service/account_test_service_cn_adaptive.go @@ -17,14 +17,19 @@ import ( const accountTestSuppressCompletionContextKey = "account_test_suppress_completion" // testCNProviderAdaptiveConnection verifies every native endpoint used by an -// adaptive CN-provider account. Zhipu uses Chat Completions plus Anthropic; -// DeepSeek and Kimi additionally use their native Responses endpoints. +// adaptive account of a provider that routes by inbound protocol. Zhipu uses +// Chat Completions plus Anthropic; DeepSeek and Kimi additionally use their +// native Responses endpoints. Endpoints the provider profile does not offer are +// skipped. func (s *AccountTestService) testCNProviderAdaptiveConnection(c *gin.Context, account *Account, modelID string, prompt string) error { - testModelID := strings.TrimSpace(modelID) - if testModelID == "" { - testModelID = openai.DefaultTestModel + requestedModelID := strings.TrimSpace(modelID) + if requestedModelID == "" { + requestedModelID = account.providerDefaultTestModel() } - testModelID = account.GetMappedModel(testModelID) + if requestedModelID == "" { + requestedModelID = openai.DefaultTestModel + } + testModelID := account.GetMappedModel(requestedModelID) authToken := strings.TrimSpace(account.GetOpenAIProtocolAPIKey()) if authToken == "" { @@ -35,12 +40,14 @@ func (s *AccountTestService) testCNProviderAdaptiveConnection(c *gin.Context, ac // completion events until every native adaptive endpoint has passed. c.Set(accountTestSuppressCompletionContextKey, true) defer c.Set(accountTestSuppressCompletionContextKey, false) - if err := s.testCNProviderChatCompletionsConnection(c, account, modelID, prompt); err != nil { + if err := s.testCNProviderChatCompletionsConnection(c, account, requestedModelID, prompt); err != nil { return err } - if err := s.testCNProviderAdaptiveAnthropicConnection(c, account, testModelID, authToken); err != nil { - return err + if account.providerSupportsProtocol(APIProtocolAnthropic) { + if err := s.testCNProviderAdaptiveAnthropicConnection(c, account, testModelID, authToken); err != nil { + return err + } } if account.SupportsNativeCNResponses() { @@ -121,6 +128,7 @@ func (s *AccountTestService) testCNProviderAdaptiveResponsesConnection(c *gin.Co apiURL := buildOpenAIResponsesURLForPlatform(account.Platform, baseURL) payload := createOpenAITestPayload(testModelID, false, accountTestUserPrompt(c)) + applyAccountTestReasoning(c, payload, false) // DeepSeek / Kimi native Responses endpoints are stateless and do not need // the OpenAI probe's synthetic instructions. delete(payload, "instructions") @@ -195,10 +203,10 @@ func (s *AccountTestService) testCNProviderAnthropicConnection(c *gin.Context, a if err != nil { return s.sendErrorAndEnd(c, fmt.Sprintf("Invalid Anthropic base URL: %s", err.Error())) } - if hint := cnAnthropicBaseURLMisconfigHint(baseURL); hint != "" { + if hint := cnAnthropicBaseURLMisconfigHint(baseURL, account.routesByModel()); hint != "" { return s.sendErrorAndEnd(c, hint) } - apiURL := strings.TrimRight(baseURL, "/") + "/v1/messages" + apiURL := nativeAnthropicMessagesURL(account, baseURL) c.Writer.Header().Set("Content-Type", "text/event-stream") c.Writer.Header().Set("Cache-Control", "no-cache") @@ -253,7 +261,9 @@ func (s *AccountTestService) testCNProviderAnthropicConnection(c *gin.Context, a // endpoint (paas path, version segment, or chat/completions / responses // suffix). The naive {base}/v1/messages join would 404 (e.g. // .../api/paas/v4/v1/messages) with no hint about the actual misconfiguration. -func cnAnthropicBaseURLMisconfigHint(baseURL string) string { +// versionAware is set for providers whose join is version-aware (see +// nativeAnthropicMessagesURL), where a trailing version segment is valid. +func cnAnthropicBaseURLMisconfigHint(baseURL string, versionAware bool) string { parsed, err := url.Parse(strings.TrimSpace(baseURL)) if err != nil || parsed.Scheme == "" || parsed.Host == "" { return "" @@ -265,7 +275,7 @@ func cnAnthropicBaseURLMisconfigHint(baseURL string) string { openAICompatShaped := strings.Contains(path, "/paas/") || strings.HasSuffix(path, "/chat/completions") || strings.HasSuffix(path, "/responses") || - openAIBaseURLHasVersionSuffix(path) + (!versionAware && openAIBaseURLHasVersionSuffix(path)) if !openAICompatShaped { return "" } diff --git a/backend/internal/service/account_test_service_opencode_go.go b/backend/internal/service/account_test_service_opencode_go.go deleted file mode 100644 index 159db6d11908..000000000000 --- a/backend/internal/service/account_test_service_opencode_go.go +++ /dev/null @@ -1,107 +0,0 @@ -package service - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - "github.com/gin-gonic/gin" -) - -// testOpenCodeGoConnection probes exactly one native OpenCode Go endpoint, -// matching the model's adaptive protocol (or the account's pinned protocol). -// OpenCode Go is a multi-protocol API-key gateway, so it must not fall through -// to the generic Claude account test path. -func (s *AccountTestService) testOpenCodeGoConnection(c *gin.Context, account *Account, modelID, prompt string) error { - ctx := c.Request.Context() - testModelID := strings.TrimSpace(modelID) - if testModelID == "" { - testModelID = DefaultOpenCodeGoTestModel - } - testModelID = account.GetMappedModel(testModelID) - protocol := openCodeGoNativeProtocol(account, testModelID) - baseURL := account.GetCNProtocolBaseURL(protocol) - if strings.TrimSpace(baseURL) == "" { - return s.sendErrorAndEnd(c, "No OpenCode Go base URL configured") - } - normalizedBaseURL, err := s.validateUpstreamBaseURL(baseURL) - if err != nil { - return s.sendErrorAndEnd(c, fmt.Sprintf("Invalid OpenCode Go base URL: %s", err.Error())) - } - - c.Writer.Header().Set("Content-Type", "text/event-stream") - c.Writer.Header().Set("Cache-Control", "no-cache") - c.Writer.Header().Set("Connection", "keep-alive") - c.Writer.Header().Set("X-Accel-Buffering", "no") - c.Writer.Flush() - s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID}) - - var apiURL string - var payloadBytes []byte - switch protocol { - case APIProtocolAnthropic: - apiURL = buildOpenAIEndpointURL(normalizedBaseURL, "/v1/messages") - payload, payloadErr := createTestPayload(testModelID, prompt) - if payloadErr != nil { - return s.sendErrorAndEnd(c, "Failed to create OpenCode Go Anthropic test payload") - } - payloadBytes, err = json.Marshal(payload) - case APIProtocolResponses: - apiURL = buildOpenAIResponsesURL(normalizedBaseURL) - payload := createOpenAITestPayload(testModelID, false, prompt) - applyAccountTestReasoning(c, payload, false) - payloadBytes, err = json.Marshal(payload) - default: - apiURL = buildOpenAIChatCompletionsURL(normalizedBaseURL) - payload := createOpenAIChatCompletionsTestPayload(testModelID, prompt) - payloadBytes = accountTestChatBody(c, payload, testModelID) - protocol = APIProtocolChatCompletions - } - if err != nil { - return s.sendErrorAndEnd(c, "Failed to create OpenCode Go test payload") - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(payloadBytes)) - if err != nil { - return s.sendErrorAndEnd(c, "Failed to create OpenCode Go request") - } - req = req.WithContext(WithHTTPUpstreamProfile(req.Context(), HTTPUpstreamProfileOpenAI)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "text/event-stream") - authToken := strings.TrimSpace(account.GetOpenAIProtocolAPIKey()) - if authToken == "" { - return s.sendErrorAndEnd(c, "No API key available") - } - if protocol == APIProtocolAnthropic { - req.Header.Set("anthropic-version", "2023-06-01") - setAnthropicAPIKeyAuthHeader(req.Header, account, authToken, normalizedBaseURL) - } else { - req.Header.Set("Authorization", "Bearer "+authToken) - } - account.ApplyHeaderOverrides(req.Header) - applyOpenCodeSessionHeader(c, account, apiURL, req.Header, payloadBytes) - - proxyURL := "" - if account.ProxyID != nil && account.Proxy != nil { - proxyURL = account.Proxy.URL() - } - resp, err := s.doOpenAIAccountTestUpstream(req, proxyURL, account, true) - if err != nil { - return s.sendErrorAndEnd(c, fmt.Sprintf("OpenCode Go request failed: %s", err.Error())) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return s.sendErrorAndEnd(c, fmt.Sprintf("OpenCode Go API returned %d: %s", resp.StatusCode, string(body))) - } - if protocol == APIProtocolAnthropic { - return s.processClaudeStream(c, resp.Body) - } - if protocol == APIProtocolChatCompletions { - return s.processOpenAIChatCompletionsStream(c, account, resp.Body) - } - return s.processOpenAIStream(c, account, resp.Body) -} diff --git a/backend/internal/service/account_test_stream_contract_test.go b/backend/internal/service/account_test_stream_contract_test.go index 0e95c010f47e..5d21e9a47f1d 100644 --- a/backend/internal/service/account_test_stream_contract_test.go +++ b/backend/internal/service/account_test_stream_contract_test.go @@ -166,5 +166,5 @@ func TestAccountConnectionBackgroundResultKeepsUpstreamError(t *testing.T) { Body: io.NopCloser(strings.NewReader(`{"error":{"message":"This request requires more credits"}}`))}) result, err := svc.RunTestBackgroundDetailed(context.Background(), account.ID, "deepseek-v4-flash") require.NoError(t, err) - require.Equal(t, `OpenCode Go API returned 402: {"error":{"message":"This request requires more credits"}}`, result.ErrorMessage) + require.Equal(t, `Chat Completions API (/v1/chat/completions) returned 402: {"error":{"message":"This request requires more credits"}}`, result.ErrorMessage) } diff --git a/backend/internal/service/admin_account.go b/backend/internal/service/admin_account.go index 813e47a2b6e8..e1f34d0ad535 100644 --- a/backend/internal/service/admin_account.go +++ b/backend/internal/service/admin_account.go @@ -337,7 +337,7 @@ func (s *adminServiceImpl) DuplicateAccount(ctx context.Context, id int64, actor if err := NormalizeHeaderOverrideCredentials(input.Credentials); err != nil { return nil, err } - if err := NormalizeOpenCodeGoProtocolRulesCredentials(input.Credentials); err != nil { + if err := NormalizeProtocolRulesCredentials(input.Credentials); err != nil { return nil, err } duplicate, err := buildAccountForCreate(input, accountExtra) @@ -557,7 +557,7 @@ func (s *adminServiceImpl) CreateAccount(ctx context.Context, input *CreateAccou if err := NormalizeHeaderOverrideCredentials(input.Credentials); err != nil { return nil, err } - if err := NormalizeOpenCodeGoProtocolRulesCredentials(input.Credentials); err != nil { + if err := NormalizeProtocolRulesCredentials(input.Credentials); err != nil { return nil, err } // Never persist ephemeral SSO/password secrets after OAuth conversion. @@ -694,7 +694,7 @@ func (s *adminServiceImpl) UpdateAccount(ctx context.Context, id int64, input *U if err := NormalizeHeaderOverrideCredentials(account.Credentials); err != nil { return nil, err } - if err := NormalizeOpenCodeGoProtocolRulesCredentials(account.Credentials); err != nil { + if err := NormalizeProtocolRulesCredentials(account.Credentials); err != nil { return nil, err } // Strip SSO/password residue that must never sit next to OAuth tokens. @@ -1182,7 +1182,7 @@ func (s *adminServiceImpl) BulkUpdateAccounts(ctx context.Context, input *BulkUp if err := NormalizeHeaderOverrideCredentials(input.Credentials); err != nil { return nil, err } - if err := NormalizeOpenCodeGoProtocolRulesCredentials(input.Credentials); err != nil { + if err := NormalizeProtocolRulesCredentials(input.Credentials); err != nil { return nil, err } // Bulk may mix platforms; always drop ephemeral SSO/password keys (cookie diff --git a/backend/internal/service/admin_group.go b/backend/internal/service/admin_group.go index 103c9edc25bc..41d16bb6c9c7 100644 --- a/backend/internal/service/admin_group.go +++ b/backend/internal/service/admin_group.go @@ -10,6 +10,7 @@ import ( dbent "github.com/Wei-Shaw/sub2api/ent" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/antigravity" "github.com/Wei-Shaw/sub2api/internal/pkg/claude" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" @@ -319,10 +320,13 @@ func defaultAllowImageGenerationForPlatform(platform string) bool { func compositeDefaultModelsListCandidateIDs() []string { seen := make(map[string]struct{}) ids := make([]string, 0) - // TypeSafe stays out of the static composite candidates (jev-latest only works - // through /v1/systemone); groups with TypeSafe accounts still get it from the - // account model mappings collected by GetGroupModelsListCandidates. - for _, platform := range []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo} { + for _, platform := range domain.CompositePrecedencePlatformIDs() { + // TypeSafe stays out of the static composite candidates (jev-latest only works + // through /v1/systemone); groups with TypeSafe accounts still get it from the + // account model mappings collected by GetGroupModelsListCandidates. + if platform == PlatformTypeSafe { + continue + } for _, id := range defaultModelsListCandidateIDs(platform) { if _, ok := seen[id]; ok { continue diff --git a/backend/internal/service/anthropic_apikey_auth.go b/backend/internal/service/anthropic_apikey_auth.go index 8bd8c4ba8f13..a7f3963071f9 100644 --- a/backend/internal/service/anthropic_apikey_auth.go +++ b/backend/internal/service/anthropic_apikey_auth.go @@ -21,7 +21,7 @@ func (a *Account) GetAnthropicAPIKeyAuthScheme() string { if a == nil || a.Type != AccountTypeAPIKey { return AnthropicAPIKeyAuthSchemeXAPIKey } - if a.Platform != PlatformAnthropic && !a.IsCNProvider() { + if a.Platform != PlatformAnthropic && !a.RoutesProtocolByInbound() { return AnthropicAPIKeyAuthSchemeXAPIKey } diff --git a/backend/internal/service/anthropic_buffered_negative_index_test.go b/backend/internal/service/anthropic_buffered_negative_index_test.go new file mode 100644 index 000000000000..c5f4a8511e41 --- /dev/null +++ b/backend/internal/service/anthropic_buffered_negative_index_test.go @@ -0,0 +1,97 @@ +package service + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// anthropicSSEWithNegativeBlockIndex is a buffered Anthropic stream in which a +// malformed upstream sends content_block_delta with a negative index. The +// buffered aggregators only bounded the index from above, so indexing +// finalResp.Content with it panicked. +func anthropicSSEWithNegativeBlockIndex() string { + return strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_neg","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4.5","usage":{"input_tokens":10}}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":-1,"delta":{"type":"text_delta","text":"IGNORED"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") +} + +func TestAnthropicBufferedAggregators_IgnoreNegativeBlockIndex(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + run func(resp *http.Response, c *gin.Context) error + }{ + { + name: "gateway chat completions", + run: func(resp *http.Response, c *gin.Context) error { + _, err := (&GatewayService{}).handleCCBufferedFromAnthropic(resp, c, "gpt-5", "claude-sonnet-4.5", nil, time.Now()) + return err + }, + }, + { + name: "gateway responses", + run: func(resp *http.Response, c *gin.Context) error { + _, err := (&GatewayService{}).handleResponsesBufferedStreamingResponse(resp, c, "claude-sonnet-4.5", "claude-sonnet-4.5", nil, time.Now(), apicompat.ResponsesClientToolMapping{}) + return err + }, + }, + { + name: "openai native anthropic chat completions", + run: func(resp *http.Response, c *gin.Context) error { + _, err := newNativeAnthropicHangTestService(5).handleCCBufferedFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, time.Now()) + return err + }, + }, + { + name: "openai native anthropic responses", + run: func(resp *http.Response, c *gin.Context) error { + _, err := newNativeAnthropicHangTestService(5).handleResponsesBufferedFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, time.Now(), apicompat.ResponsesClientToolMapping{}) + return err + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/", nil) + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(anthropicSSEWithNegativeBlockIndex())), + } + + var err error + require.NotPanics(t, func() { err = tt.run(resp, c) }) + require.NoError(t, err) + require.Contains(t, rec.Body.String(), "Hello") + require.NotContains(t, rec.Body.String(), "IGNORED") + }) + } +} diff --git a/backend/internal/service/auth_service.go b/backend/internal/service/auth_service.go index f2a76ccb9dd3..0774923e6c70 100644 --- a/backend/internal/service/auth_service.go +++ b/backend/internal/service/auth_service.go @@ -1937,9 +1937,9 @@ func (s *AuthService) snapshotPlatformQuotaDefaults(ctx context.Context, userID return nil } // 平台配额快照是 best-effort(fail-open):必须脱离调用方事务执行。 - // 否则某平台违反 user_platform_quotas 的 CHECK 约束(如尚未进约束的新平台)会让 - // 整个调用方事务被 Postgres 标记 aborted,把"无关紧要的默认配额快照"放大成 - // "整笔注册失败"(OAuth pending 路径曾因此 500 → 清 cookie → 404)。 + // 否则写入时的任何数据库错误(历史上是新平台尚未加入 user_platform_quotas 的 + // CHECK 约束)都会让整个调用方事务被 Postgres 标记 aborted,把"无关紧要的默认 + // 配额快照"放大成"整笔注册失败"(OAuth pending 路径曾因此 500 → 清 cookie → 404)。 ctx = dbent.WithoutTx(ctx) // 仅为至少配置了一档限额的平台建行:user_platform_quotas 中不存在的行等价于不限额, // 三档全空的记录不携带任何可执行的限额。 diff --git a/backend/internal/service/channel_monitor_checker.go b/backend/internal/service/channel_monitor_checker.go index a310288bf977..ddb5e78317d5 100644 --- a/backend/internal/service/channel_monitor_checker.go +++ b/backend/internal/service/channel_monitor_checker.go @@ -297,7 +297,7 @@ func callProvider(ctx context.Context, provider, endpoint, apiKey, model, prompt return "", "", 0, err } headers := mergeHeaders(adapter.buildHeaders(apiKey), opts) - full := joinURL(endpoint, adapter.buildPath(model)) + full := joinURL(endpoint, monitorRequestPath(provider, endpoint, adapter, model)) respBytes, status, err := postRawJSON(ctx, full, body, headers) if err != nil { return "", "", status, err @@ -556,6 +556,37 @@ func postRawJSON(ctx context.Context, fullURL string, payload []byte, headers ma return respBody, resp.StatusCode, nil } +// monitorRequestPath 返回探测请求路径。智谱按 endpoint 区分: +// - 已带 /paas/v4(含 Coding Plan 的 /api/coding/paas/v4):只追加 /chat/completions +// - 官方域名根地址:/api/paas/v4/chat/completions +// - 中转站 / 本站网关:只暴露 OpenAI 兼容的 /v1/chat/completions,与 Kimi / DeepSeek 一致 +func monitorRequestPath(provider, endpoint string, adapter providerAdapter, model string) string { + if provider != MonitorProviderZhipu { + return adapter.buildPath(model) + } + u, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil { + return providerOpenAIPath + } + if strings.Contains(u.EscapedPath(), "/paas/v4") { + return "/chat/completions" + } + if isZhipuOfficialHost(u) { + return adapter.buildPath(model) + } + return providerOpenAIPath +} + +func isZhipuOfficialHost(u *url.URL) bool { + host := strings.ToLower(u.Hostname()) + for _, official := range []string{"bigmodel.cn", "z.ai"} { + if host == official || strings.HasSuffix(host, "."+official) { + return true + } + } + return false +} + // joinURL 保留 base 的上游路径前缀,并避免重复追加已有的 API 路径前缀。 // 使用 EscapedPath 匹配完整路径段,避免把 hostname 或编码斜杠当作路径。 func joinURL(base, path string) string { diff --git a/backend/internal/service/channel_monitor_endpoint_test.go b/backend/internal/service/channel_monitor_endpoint_test.go index 91482dd84f08..7bb95aaa1937 100644 --- a/backend/internal/service/channel_monitor_endpoint_test.go +++ b/backend/internal/service/channel_monitor_endpoint_test.go @@ -84,6 +84,9 @@ func TestCallProvider_BasePath(t *testing.T) { {"openai responses version", MonitorProviderOpenAI, MonitorAPIModeResponses, "/relay/v1", "/relay/v1/responses"}, {"gemini version", MonitorProviderGemini, "", "/relay/v1beta", "/relay/v1beta/models/test-model:generateContent"}, {"zhipu version", MonitorProviderZhipu, "", "/api/paas/v4", "/api/paas/v4/chat/completions"}, + {"zhipu coding plan", MonitorProviderZhipu, "", "/api/coding/paas/v4", "/api/coding/paas/v4/chat/completions"}, + {"zhipu relay origin", MonitorProviderZhipu, "", "", "/v1/chat/completions"}, + {"zhipu relay version", MonitorProviderZhipu, "", "/relay/v1", "/relay/v1/chat/completions"}, } { t.Run(tc.name, func(t *testing.T) { _, _, status, err := callProvider(context.Background(), tc.provider, @@ -94,3 +97,19 @@ func TestCallProvider_BasePath(t *testing.T) { }) } } + +func TestMonitorRequestPath_Zhipu(t *testing.T) { + adapter := providerAdapters[MonitorProviderZhipu] + for endpoint, want := range map[string]string{ + "https://open.bigmodel.cn": providerZhipuPath, + "https://api.z.ai": providerZhipuPath, + "https://open.bigmodel.cn/api/coding/paas/v4": "/chat/completions", + "https://birdapi.up.railway.app": providerOpenAIPath, + "https://notbigmodel.cn": providerOpenAIPath, + "https://relay.example.com/v1": providerOpenAIPath, + } { + require.Equal(t, want, monitorRequestPath(MonitorProviderZhipu, endpoint, adapter, "glm-5.3-flash"), endpoint) + } + require.Equal(t, providerOpenAIPath, + monitorRequestPath(MonitorProviderDeepseek, "https://birdapi.up.railway.app", providerAdapters[MonitorProviderDeepseek], "m")) +} diff --git a/backend/internal/service/channel_monitor_v2.go b/backend/internal/service/channel_monitor_v2.go index cb2251de7a7f..d4aed72dcbd2 100644 --- a/backend/internal/service/channel_monitor_v2.go +++ b/backend/internal/service/channel_monitor_v2.go @@ -963,6 +963,9 @@ func ChannelMonitorV2HealthForWithThresholds(metrics ChannelMonitorV2Metric, thr Overall: "unknown", ErrorRate: "unknown", TTFT: "unknown", Cache: "unknown", MinimumSample: thresholds.MinimumSample, Thresholds: thresholds, } + if metrics.RequestCount < result.MinimumSample { + return result + } type scored struct { score float64 diff --git a/backend/internal/service/channel_monitor_v2_test.go b/backend/internal/service/channel_monitor_v2_test.go index d26f43eef272..c30f01cb7438 100644 --- a/backend/internal/service/channel_monitor_v2_test.go +++ b/backend/internal/service/channel_monitor_v2_test.go @@ -252,6 +252,81 @@ func TestChannelMonitorV2HealthBlendsErrorTTFTAndCache(t *testing.T) { require.Nil(t, health.Score) } +func TestChannelMonitorV2HealthRequiresRequestSamples(t *testing.T) { + for _, requestCount := range []int64{0, 8, 9, 49} { + t.Run(fmt.Sprintf("requests_%d", requestCount), func(t *testing.T) { + thresholds := DefaultChannelMonitorV2HealthThresholds() + thresholds.WarningCacheRate = 0.85 + thresholds.CriticalCacheRate = 0.60 + metrics := ChannelMonitorV2Metric{SuccessRequests: requestCount, RequestCount: requestCount} + if requestCount > 0 { + metrics.CacheRateDenominator = 448 + } + health := ChannelMonitorV2HealthForWithThresholds(metrics, thresholds) + require.Equal(t, ChannelMonitorV2Health{ + Overall: "unknown", ErrorRate: "unknown", TTFT: "unknown", Cache: "unknown", + MinimumSample: 50, Thresholds: NormalizeChannelMonitorV2HealthThresholds(thresholds), + }, health) + }) + } +} + +func TestChannelMonitorV2HealthMinimumSampleBoundaries(t *testing.T) { + for _, test := range []struct { + name string + minimumSample int64 + wantMinimum int64 + }{ + {name: "configured", minimumSample: 20, wantMinimum: 20}, + {name: "configured_one", minimumSample: 1, wantMinimum: 1}, + {name: "normalized_zero", minimumSample: 0, wantMinimum: 50}, + {name: "normalized_negative", minimumSample: -1, wantMinimum: 50}, + {name: "normalized_maximum", minimumSample: 10001, wantMinimum: 10000}, + } { + t.Run(test.name, func(t *testing.T) { + thresholds := DefaultChannelMonitorV2HealthThresholds() + thresholds.MinimumSample = test.minimumSample + thresholds.WarningCacheRate = 0.85 + thresholds.CriticalCacheRate = 0.60 + p50 := int64(1000) + metrics := ChannelMonitorV2Metric{ + SuccessRequests: test.wantMinimum - 1, RequestCount: test.wantMinimum - 1, + CacheRateDenominator: 20000, + TTFT: ChannelMonitorV2Latency{SampleCount: 20000, P50Ms: &p50}, + } + health := ChannelMonitorV2HealthForWithThresholds(metrics, thresholds) + require.Equal(t, ChannelMonitorV2Health{ + Overall: "unknown", ErrorRate: "unknown", TTFT: "unknown", Cache: "unknown", + MinimumSample: test.wantMinimum, Thresholds: NormalizeChannelMonitorV2HealthThresholds(thresholds), + }, health) + + metrics.RequestCount = test.wantMinimum + metrics.SuccessRequests = test.wantMinimum + health = ChannelMonitorV2HealthForWithThresholds(metrics, thresholds) + require.Equal(t, test.wantMinimum, health.MinimumSample) + require.Equal(t, "healthy", health.ErrorRate) + require.Equal(t, "healthy", health.TTFT) + require.Equal(t, "critical", health.Cache) + require.NotNil(t, health.ErrorRateScore) + require.NotNil(t, health.TTFTScore) + require.NotNil(t, health.CacheScore) + require.NotNil(t, health.Score) + require.InDelta(t, 80.0, *health.Score, 0.01) + require.Equal(t, "healthy", health.Overall) + }) + } +} + +func TestChannelMonitorV2HealthSufficientHighErrorIsCritical(t *testing.T) { + health := ChannelMonitorV2HealthFor(ChannelMonitorV2Metric{ + RequestCount: 50, ErrorRequests: 50, ErrorRate: 1, + }) + require.Equal(t, "critical", health.ErrorRate) + require.Equal(t, "critical", health.Overall) + require.NotNil(t, health.Score) + require.Zero(t, *health.Score) +} + func TestChannelMonitorV2HealthLeavesMissingTTFTUnknown(t *testing.T) { health := ChannelMonitorV2HealthFor(ChannelMonitorV2Metric{ RequestCount: 200, @@ -452,6 +527,9 @@ func TestSnapshotRedactsPublicConfigPolicyFields(t *testing.T) { Metrics: ChannelMonitorV2Metric{RequestCount: 100, ErrorRate: 0.1, SuccessRate: 0.9, RPM: 5}, }, } + repo.snap.Health = ChannelMonitorV2HealthFor(repo.snap.Metrics) + wantHealth := repo.snap.Health + require.NotNil(t, wantHealth.Score) svc := NewChannelMonitorV2Service(repo) snap, err := svc.Snapshot(context.Background(), ChannelMonitorV2Filter{}, false) require.NoError(t, err) @@ -461,6 +539,7 @@ func TestSnapshotRedactsPublicConfigPolicyFields(t *testing.T) { require.Empty(t, snap.Config.Platforms[0].Models) require.Equal(t, 300, snap.Config.RefreshIntervalSeconds) require.Zero(t, snap.Metrics.RequestCount) + require.Equal(t, wantHealth, snap.Health) require.InDelta(t, 0.1, snap.Metrics.ErrorRate, 0.0001) require.Zero(t, snap.Metrics.RPM) } diff --git a/backend/internal/service/channel_service.go b/backend/internal/service/channel_service.go index b2c605cd80d9..85d640930103 100644 --- a/backend/internal/service/channel_service.go +++ b/backend/internal/service/channel_service.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/pagination" @@ -368,7 +369,7 @@ func isPlatformPricingMatch(groupPlatform, pricingPlatform string) bool { // fallback used before a request target has been resolved. func matchingPlatforms(groupPlatform string) []string { if groupPlatform == PlatformComposite { - return []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe} + return domain.CompositePrecedencePlatformIDs() } return []string{groupPlatform} } diff --git a/backend/internal/service/channel_service_test.go b/backend/internal/service/channel_service_test.go index 62ba8e0b9a1b..cb854fe00abf 100644 --- a/backend/internal/service/channel_service_test.go +++ b/backend/internal/service/channel_service_test.go @@ -2127,7 +2127,7 @@ func TestMatchingPlatforms(t *testing.T) { {"anthropic returns itself", PlatformAnthropic, []string{PlatformAnthropic}}, {"gemini returns itself", PlatformGemini, []string{PlatformGemini}}, {"openai returns itself", PlatformOpenAI, []string{PlatformOpenAI}}, - {"composite returns concrete platforms", PlatformComposite, []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe}}, + {"composite returns concrete platforms", PlatformComposite, []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, PlatformCommandCode, PlatformCline}}, } for _, tt := range tests { diff --git a/backend/internal/service/cline.go b/backend/internal/service/cline.go new file mode 100644 index 000000000000..75d54c0ad4c5 --- /dev/null +++ b/backend/internal/service/cline.go @@ -0,0 +1,85 @@ +package service + +import ( + "net/url" + "strings" +) + +// Cline API(https://docs.cline.bot/api/overview):OpenAI 兼容的多模型聚合端点, +// 基址 https://api.cline.bot/api/v1,只提供 Chat Completions(Cline 自己的客户端同样 +// 只走该端点),模型名带厂商前缀(如 anthropic/claude-sonnet-4-6)。Responses 与 +// Anthropic 入站由网关转换为 Chat Completions。 +// +// 同一个 API Key 下有两个互不影响的额度来源(下称"钱包"),由模型名决定扣哪一个: +// - 积分:cline-pass/* 与 cline-free/* 以外的模型按量扣账户积分,余额可查; +// - ClinePass 订阅:cline-pass/* 模型,受 5 小时 / 7 天 / 30 天三档滚动用量上限限制, +// 各档已用百分比与重置时间可查(/api/v1/users/me/plan/usage-limits); +// - cline-free/* 免费模型不扣任何一边,只有逐模型的每日额度。 +// +// 一边用完只影响该边的模型,所以超限冷却按钱包落在模型级冷却键上(见 +// clineWalletRateLimitKey),不停整个账号,也不需要管理员声明账号类型。 + +const ( + // DefaultClineTestModel 是连接测试的默认模型(按量积分,价格低、长期在售)。 + DefaultClineTestModel = "deepseek/deepseek-v4-flash" + // DefaultClinePassTestModel 是订阅了 ClinePass 的账号连接测试的默认模型。 + DefaultClinePassTestModel = "cline-pass/glm-5.3-flash" + + clineAPIHost = "api.cline.bot" + + clinePassModelPrefix = "cline-pass/" + clineFreeModelPrefix = "cline-free/" + + // clinePassRateLimitKey / clineCreditsRateLimitKey 是两个钱包的模型级冷却键: + // 命中后该钱包的全部模型都不再调度到该账号,另一边不受影响。 + clinePassRateLimitKey = "cline:pass" + clineCreditsRateLimitKey = "cline:credits" +) + +// IsCline 报告账号是否为 Cline 平台账号。 +func (a *Account) IsCline() bool { + return a != nil && a.Platform == PlatformCline +} + +// clineWalletRateLimitKey 返回模型所属钱包的冷却键;免费模型不属于任何钱包,返回空串。 +func clineWalletRateLimitKey(model string) string { + model = strings.ToLower(strings.TrimSpace(model)) + switch { + case model == "": + return "" + case strings.HasPrefix(model, clinePassModelPrefix): + return clinePassRateLimitKey + case strings.HasPrefix(model, clineFreeModelPrefix): + return "" + default: + return clineCreditsRateLimitKey + } +} + +// isOfficialClineHost 报告 URL 是否指向官方 Cline API 主机。 +func isOfficialClineHost(target string) bool { + parsed, err := url.Parse(strings.TrimSpace(target)) + if err != nil { + return false + } + return strings.EqualFold(parsed.Scheme, "https") && strings.EqualFold(parsed.Hostname(), clineAPIHost) +} + +// clineAccountAPISupported 报告账号能否查询积分余额与 ClinePass 用量:API Key 账号且 +// 推理端点指向官方主机。自定义中转的 Key 不发往官方账号接口。 +func (a *Account) clineAccountAPISupported() bool { + return a.IsCline() && a.Type == AccountTypeAPIKey && isOfficialClineHost(a.GetOpenAIBaseURL()) +} + +// clinePassSubscribed 报告最近一次用量探测是否显示账号订阅了 ClinePass(有窗口快照)。 +func (a *Account) clinePassSubscribed() bool { + if a == nil || a.Extra == nil { + return false + } + for _, suffix := range []string{cnExtraSuffix5hUsed, cnExtraSuffixWeeklyUsed, cnExtraSuffixMonthlyUsed} { + if value, ok := a.Extra[cnExtraKey(PlatformCline, suffix)]; ok && value != nil { + return true + } + } + return false +} diff --git a/backend/internal/service/cline_test.go b/backend/internal/service/cline_test.go new file mode 100644 index 000000000000..c55dad595bef --- /dev/null +++ b/backend/internal/service/cline_test.go @@ -0,0 +1,648 @@ +//go:build unit + +package service + +import ( + "context" + "errors" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// clineTestAccount 只带 api_key:端点与测试模型全部来自 provider profile。 +func clineTestAccount(id int64) *Account { + return &Account{ + ID: id, + Name: "cline", + Platform: PlatformCline, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Credentials: map[string]any{"api_key": "cline_test_key", "api_protocol": APIProtocolAdaptive}, + } +} + +// withClinePassSnapshot 模拟用量探测写入的 ClinePass 窗口快照。 +func withClinePassSnapshot(account *Account, extra map[string]any) *Account { + account.Extra = map[string]any{"cline_5h_used_percent": 10.0, "cline_weekly_used_percent": 20.0, "cline_monthly_used_percent": 30.0} + for key, value := range extra { + account.Extra[key] = value + } + return account +} + +func TestClineProfileSingleModeChatCompletionsOnly(t *testing.T) { + profile := LookupProviderProfile(PlatformCline) + require.NotNil(t, profile) + require.Len(t, profile.Modes, 1) + + account := clineTestAccount(1) + require.True(t, account.IsMultiProtocolAPIKey()) + require.True(t, account.RoutesProtocolByInbound()) + require.False(t, account.routesByModel()) + require.Equal(t, APIProtocolAdaptive, account.GetAPIProtocol()) + require.Equal(t, DefaultClineBaseURL, account.GetOpenAIBaseURL()) + require.False(t, account.SupportsNativeCNResponses()) + require.False(t, account.providerSupportsProtocol(APIProtocolAnthropic)) + for _, inbound := range []string{APIProtocolChatCompletions, APIProtocolResponses, APIProtocolAnthropic} { + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, inbound, "anthropic/claude-sonnet-4-6", nil), inbound) + } + + require.False(t, account.clinePassSubscribed()) + require.Equal(t, DefaultClineTestModel, account.providerDefaultTestModel()) + subscribed := withClinePassSnapshot(clineTestAccount(2), nil) + require.True(t, subscribed.clinePassSubscribed()) + require.Equal(t, DefaultClinePassTestModel, subscribed.providerDefaultTestModel()) + // 取消订阅后探测把窗口写成 nil。 + cancelled := withClinePassSnapshot(clineTestAccount(3), map[string]any{ + "cline_5h_used_percent": nil, "cline_weekly_used_percent": nil, "cline_monthly_used_percent": nil, + }) + require.False(t, cancelled.clinePassSubscribed()) +} + +// 经真实入口:三种入站都转换 / 直通到 Chat Completions,模型名带厂商前缀原样发往上游。 +func TestClineGatewayForwardsToChatCompletions(t *testing.T) { + for _, ingress := range routingMatrixIngresses() { + upstream := &httpUpstreamRecorder{err: errors.New("stop after capture")} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + body := routingMatrixCase{ingress: ingress, model: "anthropic/claude-sonnet-4-6"}.body() + _ = ingress.forward(svc, adaptiveProtocolTestContext(ingress.path, body), clineTestAccount(3), body) + + require.NotEmpty(t, upstream.requests, ingress.name) + require.Equal(t, "https://api.cline.bot/api/v1/chat/completions", upstream.requests[len(upstream.requests)-1].URL.String(), ingress.name) + require.Equal(t, "Bearer cline_test_key", upstream.requests[len(upstream.requests)-1].Header.Get("Authorization"), ingress.name) + require.True(t, gjson.GetBytes(upstream.lastBody, "messages").Exists(), ingress.name) + require.Equal(t, "anthropic/claude-sonnet-4-6", gjson.GetBytes(upstream.lastBody, "model").String(), ingress.name) + } +} + +// 连接测试未指定模型时:订阅了 ClinePass 用订阅模型(不消耗积分),否则用按量模型; +// ClinePass 钱包冷却中也用按量模型;自适应与固定 Chat Completions 一致。 +func TestAccountTestService_ClineDefaultTestModel(t *testing.T) { + cases := []struct { + subscribed bool + passCooling bool + apiProtocol string + want string + }{ + {false, false, "", DefaultClineTestModel}, + {true, false, "", DefaultClinePassTestModel}, + {false, false, APIProtocolChatCompletions, DefaultClineTestModel}, + {true, false, APIProtocolChatCompletions, DefaultClinePassTestModel}, + {true, true, "", DefaultClineTestModel}, + } + for i, tc := range cases { + account := clineTestAccount(int64(600 + i)) + if tc.subscribed { + withClinePassSnapshot(account, nil) + } + if tc.passCooling { + now := time.Now() + setAccountModelRateLimitSnapshot(account, clinePassRateLimitKey, now.Add(time.Hour), clinePassLimitReason, now) + } + if tc.apiProtocol != "" { + account.Credentials["api_protocol"] = tc.apiProtocol + } + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNChatTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "", "hi", AccountTestModeDefault) + + require.NoError(t, err, tc) + require.Len(t, upstream.requests, 1, tc) + require.Equal(t, "https://api.cline.bot/api/v1/chat/completions", upstream.requests[0].URL.String(), tc) + require.Equal(t, tc.want, gjson.GetBytes(upstream.lastBody, "model").String(), tc) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`, tc) + } +} + +// 响应体样例取自 Cline 客户端的判定文案与官方错误文档。 +const ( + clineInsufficientCreditsBody = `{"error":{"code":"insufficient_credits","message":"Insufficient credits","details":{"current_balance":0}}}` + clineSpendLimitBody = `{"error":{"code":"SPEND_LIMIT_EXCEEDED","message":"Organization spend limit exceeded"}}` + clineInferenceCapBody = `{"error":{"code":"INFERENCE_CAP_ERROR","message":"Inference cap reached"}}` + clinePass5hBody = `{"error":{"code":429,"message":"You have reached your 5-hour ClinePass limit. Please try again later."}}` + clinePassWeeklyBody = `{"error":{"code":429,"message":"You have reached your weekly ClinePass limit. Please try again later."}}` + clinePassMonthlyBody = `{"error":{"code":429,"message":"You have reached your monthly ClinePass limit. Please try again later."}}` + clineNotSubscribedBody = `{"error":{"code":403,"message":"The user is not subscribed to required model plan"}}` + clineOrgPassBody = `{"error":{"code":403,"message":"Organization accounts cannot use individual model inference subscriptions"}}` + clineFreeModelLimitBody = `{"error":{"code":429,"message":"Free limit reached on model cline-free/kimi-k3. Try again in 3h 20m"}}` + clineRateLimitBody = `{"error":{"code":429,"message":"Rate limit exceeded"}}` +) + +func TestClassifyClineError(t *testing.T) { + cases := map[string]clineErrorKind{ + clineInsufficientCreditsBody: clineCreditsExhausted, + clineSpendLimitBody: clineSpendLimit, + clineInferenceCapBody: clineInferenceCap, + clinePass5hBody: clinePassLimit, + clinePassWeeklyBody: clinePassLimit, + clineNotSubscribedBody: clinePassUnavailable, + clineOrgPassBody: clinePassUnavailable, + clineFreeModelLimitBody: clineFreeModelLimit, + clineRateLimitBody: clineErrorNone, + `{"error":{"message":"You have reached your limit. Please try again later."}}`: clineErrorNone, + } + for body, want := range cases { + kind, _ := parseClineError([]byte(body)) + require.Equal(t, want, kind, body) + } + + require.Equal(t, "5h", clinePassWindow([]byte(clinePass5hBody))) + require.Equal(t, "weekly", clinePassWindow([]byte(clinePassWeeklyBody))) + require.Equal(t, "monthly", clinePassWindow([]byte(clinePassMonthlyBody))) + require.Equal(t, 3*time.Hour+20*time.Minute, clineFreeModelResetAfter([]byte(clineFreeModelLimitBody))) + require.Zero(t, clineFreeModelResetAfter([]byte(`Free limit reached on model x. Try again in 3 weeks`))) +} + +// ClinePass 超限冷却到快照中该窗口的重置时间;没有可用快照时按窗口估计。 +func TestClinePassLimitCooldown(t *testing.T) { + now := time.Now() + require.Equal(t, clineLimitRecheck, clinePassLimitCooldown(clineTestAccount(1), "5h", now)) + require.Equal(t, clinePassWeeklyRecheck, clinePassLimitCooldown(clineTestAccount(1), "weekly", now)) + require.Equal(t, clinePassMonthlyRecheck, clinePassLimitCooldown(clineTestAccount(1), "monthly", now)) + + account := withClinePassSnapshot(clineTestAccount(2), map[string]any{ + "cline_weekly_reset_at": now.Add(50 * time.Hour).UTC().Format(time.RFC3339), + "cline_5h_reset_at": now.Add(-time.Minute).UTC().Format(time.RFC3339), + }) + require.InDelta(t, (50 * time.Hour).Seconds(), clinePassLimitCooldown(account, "weekly", now).Seconds(), 1) + require.Equal(t, clineLimitRecheck, clinePassLimitCooldown(account, "5h", now), "past reset falls back to the estimate") +} + +// cline-pass/* 与其余付费模型各自落在钱包冷却键上,免费模型不属于任何钱包;调度按 +// 请求模型检查所属钱包。 +func TestClineWalletRateLimitKeys(t *testing.T) { + require.Equal(t, clinePassRateLimitKey, clineWalletRateLimitKey("cline-pass/glm-5.3")) + require.Equal(t, clinePassRateLimitKey, clineWalletRateLimitKey(" Cline-Pass/GLM-5.3 ")) + require.Equal(t, clineCreditsRateLimitKey, clineWalletRateLimitKey("deepseek/deepseek-v4-flash")) + require.Empty(t, clineWalletRateLimitKey("cline-free/kimi-k3")) + require.Empty(t, clineWalletRateLimitKey("")) + + account := clineTestAccount(1) + setAccountModelRateLimitSnapshot(account, clinePassRateLimitKey, time.Now().Add(time.Hour), clinePassLimitReason, time.Now()) + ctx := context.Background() + require.True(t, account.isModelRateLimitedWithContext(ctx, "cline-pass/glm-5.3")) + require.False(t, account.isModelRateLimitedWithContext(ctx, "deepseek/deepseek-v4-flash")) + require.False(t, account.isModelRateLimitedWithContext(ctx, "cline-free/kimi-k3")) + + setAccountModelRateLimitSnapshot(account, clineCreditsRateLimitKey, time.Now().Add(time.Hour), clineCreditsReason, time.Now()) + require.True(t, account.isModelRateLimitedWithContext(ctx, "deepseek/deepseek-v4-flash")) + require.False(t, account.isModelRateLimitedWithContext(ctx, "cline-free/kimi-k3")) + require.Equal(t, clineCreditsReason, account.modelRateLimitReason(clineCreditsRateLimitKey)) + + // 其他平台不受 Cline 钱包键影响。 + other := &Account{Platform: PlatformDeepseek, Extra: account.Extra} + require.False(t, other.isModelRateLimitedWithContext(ctx, "cline-pass/glm-5.3")) +} + +// 已识别的错误只冷却请求模型所属的钱包(或免费模型本身),不停整个账号、不置 error。 +func TestHandleUpstreamError_ClineCoolsDownWallet(t *testing.T) { + paid := "deepseek/deepseek-v4-flash" + pass := "cline-pass/glm-5.3" + now := time.Now() + snapshot := withClinePassSnapshot(clineTestAccount(1), map[string]any{ + "cline_5h_reset_at": now.Add(2 * time.Hour).UTC().Format(time.RFC3339), + }) + cases := []struct { + name string + account *Account + model string + status int + body string + scope string + wait time.Duration + reason string + }{ + {"pass weekly", clineTestAccount(10), pass, http.StatusTooManyRequests, clinePassWeeklyBody, clinePassRateLimitKey, clinePassWeeklyRecheck, clinePassLimitReason}, + {"pass 5h from snapshot", snapshot, pass, http.StatusTooManyRequests, clinePass5hBody, clinePassRateLimitKey, 2 * time.Hour, clinePassLimitReason}, + {"not subscribed", clineTestAccount(11), pass, http.StatusForbidden, clineNotSubscribedBody, clinePassRateLimitKey, clineLimitRecheck, clinePassUnavailableReason}, + {"credits", clineTestAccount(12), paid, http.StatusPaymentRequired, clineInsufficientCreditsBody, clineCreditsRateLimitKey, cnBalanceCheckCooldown(nil), clineCreditsReason}, + {"unrecognized 402", clineTestAccount(13), paid, http.StatusPaymentRequired, `{"error":{"message":"Payment required"}}`, clineCreditsRateLimitKey, cnBalanceCheckCooldown(nil), clineCreditsReason}, + {"spend limit", clineTestAccount(14), paid, http.StatusTooManyRequests, clineSpendLimitBody, clineCreditsRateLimitKey, clineLimitRecheck, clineSpendLimitReason}, + {"inference cap on paid model", clineTestAccount(15), paid, http.StatusTooManyRequests, clineInferenceCapBody, clineCreditsRateLimitKey, clineLimitRecheck, clineInferenceCapReason}, + {"inference cap on pass model", clineTestAccount(16), pass, http.StatusTooManyRequests, clineInferenceCapBody, clinePassRateLimitKey, clineLimitRecheck, clineInferenceCapReason}, + {"unknown model uses the error's wallet", clineTestAccount(17), "", http.StatusForbidden, clineOrgPassBody, clinePassRateLimitKey, clineLimitRecheck, clinePassUnavailableReason}, + {"free model", clineTestAccount(18), "cline-free/kimi-k3", http.StatusTooManyRequests, clineFreeModelLimitBody, "cline-free/kimi-k3", 3*time.Hour + 20*time.Minute, clineFreeModelLimitReason}, + } + for _, tc := range cases { + ctx := context.Background() + if tc.model != "" { + ctx = withTempUnschedulableModel(ctx, []string{tc.model}) + } + repo := &commandCodeRateLimitRepo{} + + shouldDisable := NewRateLimitService(repo, nil, &config.Config{}, nil, nil).HandleUpstreamError(ctx, tc.account, + tc.status, http.Header{}, []byte(tc.body)) + + require.False(t, shouldDisable, tc.name) + require.Equal(t, 0, repo.tempCalls, tc.name) + require.Equal(t, 0, repo.setErrorCalls, tc.name) + require.Equal(t, 0, repo.rateLimitedCalls, tc.name) + require.Len(t, repo.modelLimits, 1, tc.name) + require.WithinDuration(t, time.Now().Add(tc.wait), repo.modelLimits[tc.scope], time.Minute, tc.name) + require.True(t, strings.HasPrefix(repo.reasons[tc.scope], tc.reason), tc.name+": "+repo.reasons[tc.scope]) + require.True(t, tc.account.isRateLimitActiveForKey(tc.scope), tc.name) + } + + // 普通频率限制交回默认 429 逻辑。 + plain := &commandCodeRateLimitRepo{} + NewRateLimitService(plain, nil, &config.Config{}, nil, nil).HandleUpstreamError(withTempUnschedulableModel(context.Background(), []string{paid}), + clineTestAccount(30), http.StatusTooManyRequests, http.Header{}, []byte(clineRateLimitBody)) + require.Empty(t, plain.modelLimits) + require.Equal(t, 0, plain.tempCalls) +} + +// clineModelLimitFailRepo 让钱包冷却写入失败。 +type clineModelLimitFailRepo struct { + commandCodeRateLimitRepo +} + +func (r *clineModelLimitFailRepo) SetModelRateLimit(context.Context, int64, string, time.Time, ...string) error { + return errors.New("db unavailable") +} + +// 积分冷却写入失败时 402 也不落入默认分支永久置 error:探测只覆盖激活账号,置 error 后 +// 充值也无法自动恢复。 +func TestHandleUpstreamError_Cline402KeepsAccountWhenCooldownWriteFails(t *testing.T) { + ctx := withTempUnschedulableModel(context.Background(), []string{"deepseek/deepseek-v4-flash"}) + for _, body := range []string{clineInsufficientCreditsBody, `{"error":{"message":"Payment required"}}`} { + repo := &clineModelLimitFailRepo{} + + shouldDisable := NewRateLimitService(repo, nil, &config.Config{}, nil, nil).HandleUpstreamError(ctx, clineTestAccount(19), + http.StatusPaymentRequired, http.Header{}, []byte(body)) + + require.False(t, shouldDisable, body) + require.Equal(t, 0, repo.setErrorCalls, body) + require.Equal(t, 0, repo.tempCalls, body) + } +} + +// clineAccountRepo 为余额 / 用量探测与周期检测提供 Cline 账号,并记录写入。 +type clineAccountRepo struct { + AccountRepository + accounts []Account + mu sync.Mutex + extraWrites map[int64][]map[string]any + modelLimits map[int64]map[string]time.Time + reasons map[int64]map[string]string + paused []int64 +} + +func (r *clineAccountRepo) ListByPlatform(_ context.Context, platform string) ([]Account, error) { + if platform == PlatformCline { + return r.accounts, nil + } + return nil, nil +} + +func (r *clineAccountRepo) GetByID(_ context.Context, id int64) (*Account, error) { + for i := range r.accounts { + if r.accounts[i].ID == id { + return &r.accounts[i], nil + } + } + return nil, errors.New("not found") +} + +func (r *clineAccountRepo) UpdateExtra(_ context.Context, id int64, updates map[string]any) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.extraWrites == nil { + r.extraWrites = map[int64][]map[string]any{} + } + r.extraWrites[id] = append(r.extraWrites[id], updates) + return nil +} + +func (r *clineAccountRepo) SetModelRateLimit(_ context.Context, id int64, scope string, resetAt time.Time, reason ...string) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.modelLimits == nil { + r.modelLimits = map[int64]map[string]time.Time{} + r.reasons = map[int64]map[string]string{} + } + if r.modelLimits[id] == nil { + r.modelLimits[id] = map[string]time.Time{} + r.reasons[id] = map[string]string{} + } + r.modelLimits[id][scope] = resetAt + if len(reason) > 0 { + r.reasons[id][scope] = reason[0] + } + return nil +} + +func (r *clineAccountRepo) SetTempUnschedulable(_ context.Context, id int64, _ time.Time, _ string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.paused = append(r.paused, id) + return nil +} + +const ( + clineUsageLimitsPath = "/api/v1/users/me/plan/usage-limits" + clineUsageLimitsBody = `{"data":{"limits":[{"type":"five_hour","percentUsed":12.5,"resetsAt":"2099-01-01T05:00:00.123456Z"},{"type":"weekly","percentUsed":40},{"type":"monthly","percentUsed":0}]},"success":true}` +) + +func newClineAccountUpstream(usageLimits commandCodeAlphaResponse, balance string) *commandCodeAlphaUpstream { + return &commandCodeAlphaUpstream{responses: map[string]commandCodeAlphaResponse{ + clineUsageLimitsPath: usageLimits, + "/api/v1/users/me": {status: http.StatusOK, body: `{"success":true,"data":{"id":"u-1","email":"a@example.com"}}`}, + "/api/v1/users/u-1/balance": {status: http.StatusOK, body: balance}, + }} +} + +// balance 是百万分之一积分(1 积分 = 1 美元),按 Cline 的展示精度保留 4 位小数。 +func TestClineBalanceProbeConvertsMicrocredits(t *testing.T) { + for _, tc := range []struct { + balance string + want float64 + }{ + {`{"balance":12340000,"userId":"u-1"}`, 12.34}, + {`{"success":true,"data":{"balance":"500000","userId":"u-1"}}`, 0.5}, + {`{"balance":1234567,"userId":"u-1"}`, 1.2346}, + } { + account := clineTestAccount(40) + upstream := newClineAccountUpstream(commandCodeAlphaResponse{}, tc.balance) + repo := &cnBalanceProbeRepo{account: account} + svc := NewCNProviderBalanceService(repo, nil, upstream, &config.Config{}) + + result, err := svc.QueryBalance(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, result.Success) + require.True(t, result.Persisted) + require.InDelta(t, tc.want, result.Balance, 1e-9) + require.Equal(t, "USD", result.Currency) + require.Len(t, upstream.requests, 2) + require.Equal(t, "Bearer cline_test_key", upstream.requests[0].Header.Get("Authorization")) + require.InDelta(t, tc.want, repo.extraWrites[0]["cline_balance"], 1e-9) + } +} + +func TestClineBalanceProbeFailures(t *testing.T) { + account := clineTestAccount(41) + upstream := &commandCodeAlphaUpstream{responses: map[string]commandCodeAlphaResponse{ + "/api/v1/users/me": {status: http.StatusUnauthorized, body: `{"error":"Unauthorized"}`}, + }} + repo := &cnBalanceProbeRepo{account: account} + result, err := NewCNProviderBalanceService(repo, nil, upstream, &config.Config{}).QueryBalance(context.Background(), account.ID) + require.NoError(t, err) + require.False(t, result.Success) + require.Equal(t, http.StatusUnauthorized, result.StatusCode) + require.Contains(t, result.Error, "HTTP 401") + require.Empty(t, repo.extraWrites) + + envelope := &commandCodeAlphaUpstream{responses: map[string]commandCodeAlphaResponse{ + "/api/v1/users/me": {status: http.StatusOK, body: `{"success":false,"error":"token expired"}`}, + }} + result, err = NewCNProviderBalanceService(&cnBalanceProbeRepo{account: account}, nil, envelope, &config.Config{}).QueryBalance(context.Background(), account.ID) + require.NoError(t, err) + require.False(t, result.Success) + require.Contains(t, result.Error, "token expired") +} + +// 余额与用量只查官方主机上的 API Key 账号:自定义中转的 Key 不发往官方账号接口。 +func TestClineAccountAPIRequiresOfficialHost(t *testing.T) { + require.NoError(t, validatePayGAccount(clineTestAccount(50))) + require.NoError(t, validateCodingPlanAccount(clineTestAccount(50))) + require.Equal(t, PlatformCline, clineTestAccount(50).GetCodingPlanProvider()) + + relay := clineTestAccount(52) + relay.Credentials["base_url"] = "https://relay.example.com/v1" + relay.Credentials["api_protocol"] = APIProtocolChatCompletions + require.Error(t, validatePayGAccount(relay)) + require.Error(t, validateCodingPlanAccount(relay)) + require.Empty(t, relay.GetCodingPlanProvider()) +} + +func clineUsageTestConfig() *config.Config { + cfg := &config.Config{} + cfg.Gateway.CNProviders.BalanceThreshold = 0.5 + return cfg +} + +// 一次探测刷新三档窗口与积分余额;两个钱包都可用时不写冷却。 +func TestClineUsageProbeWritesWindowsAndBalance(t *testing.T) { + account := clineTestAccount(60) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: clineUsageLimitsBody}, `{"success":true,"data":{"balance":12340000}}`) + + result, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, result.Success) + require.True(t, result.CredentialValid) + require.True(t, result.Persisted) + require.Equal(t, "ClinePass", result.PlanLevel) + require.Equal(t, []CNQuotaTier{ + {Window: "5h", UsedPercent: 12.5, ResetAt: "2099-01-01T05:00:00Z"}, + {Window: "weekly", UsedPercent: 40}, + {Window: "monthly", UsedPercent: 0}, + }, result.Tiers) + require.NotNil(t, result.Balance) + require.InDelta(t, 12.34, result.Balance.Balance, 1e-9) + require.Equal(t, "Bearer cline_test_key", upstream.requests[0].Header.Get("Authorization")) + + written := repo.extraWrites[60][0] + require.Equal(t, 12.5, written["cline_5h_used_percent"]) + require.Equal(t, "2099-01-01T05:00:00Z", written["cline_5h_reset_at"]) + require.Equal(t, 40.0, written["cline_weekly_used_percent"]) + require.Contains(t, written, "cline_weekly_reset_at") + require.Nil(t, written["cline_weekly_reset_at"], "a window that has not started has no reset time") + require.InDelta(t, 12.34, written["cline_balance"], 1e-9) + require.Empty(t, repo.modelLimits) +} + +// 窗口用满冷却 ClinePass 到重置时间;积分低于阈值冷却积分钱包;都不停整个账号。 +func TestClineUsageProbeCoolsDownExhaustedWallets(t *testing.T) { + resetAt := time.Now().Add(72 * time.Hour).UTC().Truncate(time.Second) + body := `{"data":{"limits":[{"type":"five_hour","percentUsed":30},{"type":"weekly","percentUsed":100,"resetsAt":"` + + resetAt.Format(time.RFC3339) + `"}]},"success":true}` + account := clineTestAccount(61) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: body}, `{"success":true,"data":{"balance":200000}}`) + + _, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, repo.modelLimits[61][clinePassRateLimitKey].Equal(resetAt)) + require.Equal(t, clinePassLimitReason+": weekly window used up", repo.reasons[61][clinePassRateLimitKey]) + require.WithinDuration(t, time.Now().Add(cnBalanceCheckCooldown(nil)), repo.modelLimits[61][clineCreditsRateLimitKey], time.Minute) + require.True(t, strings.HasPrefix(repo.reasons[61][clineCreditsRateLimitKey], clineCreditsLowReason), repo.reasons[61][clineCreditsRateLimitKey]) + require.Empty(t, repo.paused) +} + +// 没有订阅(404 或空列表):清掉旧窗口快照,ClinePass 模型不调度到该账号。 +func TestClineUsageProbeWithoutSubscription(t *testing.T) { + for _, usage := range []commandCodeAlphaResponse{ + {status: http.StatusNotFound, body: `{"success":false,"error":"no plan history found for user"}`}, + {status: http.StatusOK, body: `{"success":true,"data":{"limits":[]}}`}, + } { + account := withClinePassSnapshot(clineTestAccount(62), nil) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(usage, `{"success":true,"data":{"balance":5000000}}`) + + result, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, result.Success) + require.Empty(t, result.Tiers) + require.Empty(t, result.PlanLevel) + written := repo.extraWrites[62][0] + for _, key := range []string{"cline_5h_used_percent", "cline_weekly_used_percent", "cline_monthly_used_percent"} { + require.Contains(t, written, key) + require.Nil(t, written[key]) + } + require.True(t, strings.HasPrefix(repo.reasons[62][clinePassRateLimitKey], clinePassUnavailableReason)) + require.NotContains(t, repo.modelLimits[62], clineCreditsRateLimitKey) + } +} + +// 数据显示钱包恢复可用时解除冷却;组织花费上限无从探测,不提前解除。 +func TestClineUsageProbeClearsRecoveredWallets(t *testing.T) { + for _, tc := range []struct { + creditsReason string + cleared bool + }{ + {clineCreditsReason + ": Insufficient credits", true}, + {clineCreditsLowReason + ": balance 0.1 USD below threshold 0.50", true}, + {clineSpendLimitReason + ": Organization spend limit exceeded", false}, + } { + account := clineTestAccount(63) + now := time.Now() + setAccountModelRateLimitSnapshot(account, clinePassRateLimitKey, now.Add(6*time.Hour), clinePassLimitReason, now) + setAccountModelRateLimitSnapshot(account, clineCreditsRateLimitKey, now.Add(time.Hour), tc.creditsReason, now) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: clineUsageLimitsBody}, `{"success":true,"data":{"balance":5000000}}`) + + _, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.False(t, repo.modelLimits[63][clinePassRateLimitKey].After(time.Now()), tc.creditsReason) + _, touched := repo.modelLimits[63][clineCreditsRateLimitKey] + require.Equal(t, tc.cleared, touched, tc.creditsReason) + if tc.cleared { + require.False(t, repo.modelLimits[63][clineCreditsRateLimitKey].After(time.Now()), tc.creditsReason) + } + } +} + +// ClinePass 同样只解除探测能证明已恢复的冷却(窗口用满、未订阅):组织账户、所需订阅 +// 档位与推理额度上限反映不到用量接口里,提前解除会让每个检测周期都重放一次失败请求。 +func TestClineUsageProbeClearsOnlyProvablePassCooldowns(t *testing.T) { + ctx := withTempUnschedulableModel(context.Background(), []string{"cline-pass/glm-5.3"}) + balance := `{"success":true,"data":{"balance":5000000}}` + for _, tc := range []struct { + status int + body string + cleared bool + }{ + {http.StatusTooManyRequests, clinePassWeeklyBody, true}, + {http.StatusForbidden, clineNotSubscribedBody, false}, + {http.StatusForbidden, clineOrgPassBody, false}, + {http.StatusTooManyRequests, clineInferenceCapBody, false}, + } { + account := clineTestAccount(65) + NewRateLimitService(&commandCodeRateLimitRepo{}, nil, &config.Config{}, nil, nil).HandleUpstreamError(ctx, account, + tc.status, http.Header{}, []byte(tc.body)) + require.True(t, account.isRateLimitActiveForKey(clinePassRateLimitKey), tc.body) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: clineUsageLimitsBody}, balance) + + _, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + _, touched := repo.modelLimits[65][clinePassRateLimitKey] + require.Equal(t, tc.cleared, touched, tc.body) + } + + // 探测发现未订阅设的冷却,订阅后的下一次探测解除。 + repo := &clineAccountRepo{accounts: []Account{*clineTestAccount(66)}} + unsubscribed := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusNotFound, + body: `{"success":false,"error":"no plan history found for user"}`}, balance) + _, err := NewCNProviderQuotaService(repo, nil, unsubscribed, clineUsageTestConfig()).QueryUsage(context.Background(), 66) + require.NoError(t, err) + require.True(t, repo.modelLimits[66][clinePassRateLimitKey].After(time.Now())) + + subscribed := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: clineUsageLimitsBody}, balance) + _, err = NewCNProviderQuotaService(repo, nil, subscribed, clineUsageTestConfig()).QueryUsage(context.Background(), 66) + require.NoError(t, err) + require.False(t, repo.modelLimits[66][clinePassRateLimitKey].After(time.Now())) +} + +// 用量接口鉴权失败:探测失败但不写快照、不动冷却。 +func TestClineUsageProbeAuthFailure(t *testing.T) { + account := clineTestAccount(64) + repo := &clineAccountRepo{accounts: []Account{*account}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusUnauthorized, body: `{"error":"Unauthorized"}`}, `{}`) + + result, err := NewCNProviderQuotaService(repo, nil, upstream, clineUsageTestConfig()).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.False(t, result.Success) + require.False(t, result.CredentialValid) + require.Equal(t, http.StatusUnauthorized, result.StatusCode) + require.Empty(t, repo.extraWrites) + require.Empty(t, repo.modelLimits) +} + +// 周期检测探测官方主机上的全部激活账号(含手动停调的,钱包冷却与调度开关无关), +// 只冷却钱包,不停整个账号。 +func TestCNProviderBalanceCheckRunOnceProbesClineUsage(t *testing.T) { + official := *clineTestAccount(1) + manualOff := *clineTestAccount(2) + manualOff.Schedulable = false + relay := *clineTestAccount(3) + relay.Credentials["base_url"] = "https://relay.example.com/v1" + relay.Credentials["api_protocol"] = APIProtocolChatCompletions + + repo := &clineAccountRepo{accounts: []Account{official, manualOff, relay}} + upstream := newClineAccountUpstream(commandCodeAlphaResponse{status: http.StatusOK, body: clineUsageLimitsBody}, `{"success":true,"data":{"balance":200000}}`) + cfg := clineUsageTestConfig() + svc := &CNProviderBalanceCheckService{ + accountRepo: repo, + balanceService: NewCNProviderBalanceService(repo, nil, upstream, cfg), + quotaService: NewCNProviderQuotaService(repo, nil, upstream, cfg), + cfg: cfg, + } + + svc.runOnce() + + require.Len(t, upstream.requests, 6, "usage-limits + me + balance for each official account") + require.Len(t, repo.extraWrites, 2) + require.Contains(t, repo.modelLimits[1], clineCreditsRateLimitKey) + require.Contains(t, repo.modelLimits[2], clineCreditsRateLimitKey) + require.Empty(t, repo.paused) +} + +// 同一轮检测里每个账号走自己平台登记的检测函数:积分同样低于阈值,Command Code 停整个 +// 账号,Cline 不停(钱包冷却在探测中处理)。 +func TestCNProviderBalanceCheckRunOnceDispatchesAccountProbesByPlatform(t *testing.T) { + commandCode := *commandCodeUsageAccount() + commandCode.ID, commandCode.Schedulable = 1, true + cline := *clineTestAccount(2) + repo := &commandCodeCheckRepo{accounts: []Account{commandCode, cline}} + prober := &commandCodeBalanceProber{balance: 0.2} + svc := &CNProviderBalanceCheckService{accountRepo: repo, quotaService: prober, cfg: clineUsageTestConfig()} + + svc.runOnce() + + require.ElementsMatch(t, []int64{1, 2}, prober.probed) + require.Equal(t, []int64{1}, repo.paused) +} + +// 官方主机不做上游计费探测:中转站的计费接口在官方 API 上不存在,只会把 Key 发往官方主机。 +func TestClineOfficialHostSkipsUpstreamBillingProbe(t *testing.T) { + require.True(t, upstreamBillingProbeTargetIsOfficialAPI(DefaultClineBaseURL)) + require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://relay.example.com/v1")) +} diff --git a/backend/internal/service/cline_usage.go b/backend/internal/service/cline_usage.go new file mode 100644 index 000000000000..dd796994bdad --- /dev/null +++ b/backend/internal/service/cline_usage.go @@ -0,0 +1,368 @@ +package service + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" + "github.com/tidwall/gjson" +) + +// Cline 积分余额与 ClinePass 用量来自账号接口(Cline 控制台与客户端同源),账号 API Key +// 可直接访问(Bearer ;实测带 workos: 前缀反而 401): +// +// GET /api/v1/users/me -> {id, email, ...} +// GET /api/v1/users/{id}/balance -> {balance, userId} +// GET /api/v1/users/me/plan/usage-limits -> {limits:[{type, percentUsed, resetsAt?}]} +// +// 响应包在 {success, error, data} 信封里。balance 单位为百万分之一积分(1 积分 = +// 1 美元;Cline CLI 的 normalizeCreditBalance 除以 1e6,扩展先除 100 再除 10000), +// 按 Cline 的展示精度保留 4 位小数。usage-limits 的 type 为 five_hour / weekly / +// monthly(套餐里对应最近 5 小时 / 7 天 / 30 天的滚动用量上限),percentUsed 为 0-100, +// 窗口未开始时没有 resetsAt;未订阅时返回 404(no plan history found for user)或空列表。 + +const clineAPIBase = "https://" + clineAPIHost + +var errClineInvalidResponse = errors.New("invalid cline account response") + +// clineHTTPError 是账号接口的非 2xx 响应。 +type clineHTTPError struct { + status int + body string +} + +func (e *clineHTTPError) Error() string { + return fmt.Sprintf("HTTP %d: %s", e.status, e.body) +} + +type clineAccountClient struct { + cfg *config.Config + httpUpstream HTTPUpstream + account *Account + apiKey string + proxyURL string + // errPrefix 是应用错误码前缀(CN_BALANCE / CN_QUOTA),与调用方服务的错误码一致。 + errPrefix string +} + +func newClineAccountClient(cfg *config.Config, httpUpstream HTTPUpstream, account *Account, proxyURL, errPrefix string) *clineAccountClient { + return &clineAccountClient{ + cfg: cfg, + httpUpstream: httpUpstream, + account: account, + apiKey: strings.TrimSpace(account.GetCNAPIKey()), + proxyURL: proxyURL, + errPrefix: errPrefix, + } +} + +// get 请求一个账号接口,返回拆掉 {success, error, data} 信封后的 JSON。 +func (c *clineAccountClient) get(ctx context.Context, path string) (gjson.Result, error) { + validatedURL, err := cnValidateProbeURL(c.cfg, clineAPIBase+path) + if err != nil { + return gjson.Result{}, infraerrors.New(http.StatusForbidden, c.errPrefix+"_URL_REJECTED", err.Error()) + } + callCtx, cancel := context.WithTimeout(ctx, cnQuotaUpstreamTimeout) + defer cancel() + req, err := http.NewRequestWithContext(callCtx, http.MethodGet, validatedURL, nil) + if err != nil { + return gjson.Result{}, infraerrors.Newf(http.StatusInternalServerError, c.errPrefix+"_REQUEST_BUILD_FAILED", "build request: %v", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Accept", "application/json") + c.account.ApplyHeaderOverrides(req.Header) + + resp, err := c.httpUpstream.Do(req, c.proxyURL, c.account.ID, maxInt(c.account.Concurrency, 1)) + if err != nil { + return gjson.Result{}, fmt.Errorf("upstream request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(resp.Body, cnQuotaMaxBodyBytes)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return gjson.Result{}, &clineHTTPError{status: resp.StatusCode, body: truncate(strings.TrimSpace(string(body)), 240)} + } + if !gjson.ValidBytes(body) { + return gjson.Result{}, fmt.Errorf("%w: %s", errClineInvalidResponse, path) + } + parsed := gjson.ParseBytes(body) + if success := parsed.Get("success"); success.Exists() { + if !success.Bool() { + return gjson.Result{}, &clineHTTPError{status: resp.StatusCode, body: truncate(parsed.Get("error").String(), 240)} + } + return parsed.Get("data"), nil + } + return parsed, nil +} + +// balance 查询积分余额(美元)。 +func (c *clineAccountClient) balance(ctx context.Context) (float64, error) { + me, err := c.get(ctx, "/api/v1/users/me") + if err != nil { + return 0, err + } + userID := strings.TrimSpace(me.Get("id").String()) + if userID == "" { + return 0, fmt.Errorf("%w: missing user id", errClineInvalidResponse) + } + balance, err := c.get(ctx, "/api/v1/users/"+url.PathEscape(userID)+"/balance") + if err != nil { + return 0, err + } + micro := balance.Get("balance") + if !micro.Exists() || (micro.Type != gjson.Number && micro.Type != gjson.String) { + return 0, fmt.Errorf("%w: missing balance", errClineInvalidResponse) + } + return math.Round(micro.Float()/100) / 10000, nil +} + +// passLimits 查询 ClinePass 三档窗口;subscribed=false 表示没有订阅。 +func (c *clineAccountClient) passLimits(ctx context.Context) (tiers []CNQuotaTier, subscribed bool, err error) { + data, err := c.get(ctx, "/api/v1/users/me/plan/usage-limits") + if err != nil { + var httpErr *clineHTTPError + if errors.As(err, &httpErr) && httpErr.status == http.StatusNotFound { + return nil, false, nil + } + return nil, false, err + } + limits := data.Get("limits") + if !limits.IsArray() { + return nil, false, fmt.Errorf("%w: missing limits", errClineInvalidResponse) + } + windows := map[string]string{"five_hour": "5h", "weekly": "weekly", "monthly": "monthly"} + for _, item := range limits.Array() { + window, ok := windows[strings.ToLower(strings.TrimSpace(item.Get("type").String()))] + used := item.Get("percentUsed") + if !ok || used.Type != gjson.Number { + continue + } + tier := CNQuotaTier{Window: window, UsedPercent: used.Float()} + if resetAt, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(item.Get("resetsAt").String())); err == nil { + tier.ResetAt = resetAt.UTC().Format(time.RFC3339) + } + tiers = append(tiers, tier) + } + return tiers, len(tiers) > 0, nil +} + +// clineFetchFailure 把取数错误转换为探测失败的状态码与文案;ok=false 表示应作为错误返回。 +func clineFetchFailure(err error) (status int, message string, ok bool) { + var httpErr *clineHTTPError + switch { + case errors.As(err, &httpErr): + return httpErr.status, fmt.Sprintf("API error (HTTP %d): %s", httpErr.status, httpErr.body), true + case errors.Is(err, errClineInvalidResponse): + return http.StatusOK, "Invalid response", true + default: + return 0, "", false + } +} + +func clineApplicationError(err error, errPrefix string) error { + if isApplicationError(err) { + return err + } + return infraerrors.Newf(http.StatusBadGateway, errPrefix+"_REQUEST_FAILED", "%v", err) +} + +func clineBalanceResult(usd float64, now time.Time) *CNProviderBalanceResult { + return &CNProviderBalanceResult{ + Provider: PlatformCline, + Success: true, + Balance: usd, + Currency: "USD", + Balances: []CNProviderBalanceEntry{{Currency: "USD", Balance: usd}}, + Available: true, + StatusCode: http.StatusOK, + FetchedAt: now.Unix(), + } +} + +// queryClineBalance 只查积分余额(美元)并落余额快照。 +func (s *CNProviderBalanceService) queryClineBalance(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { + client := newClineAccountClient(s.cfg, s.httpUpstream, account, s.resolveProxyURL(ctx, account), "CN_BALANCE") + if client.apiKey == "" { + return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_APIKEY", "account api_key is empty") + } + now := time.Now().UTC() + usd, err := client.balance(ctx) + if err != nil { + status, message, ok := clineFetchFailure(err) + if !ok { + return nil, clineApplicationError(err, "CN_BALANCE") + } + return &CNProviderBalanceResult{Provider: PlatformCline, StatusCode: status, FetchedAt: now.Unix(), Available: true, + Error: "Balance query failed: " + message}, nil + } + result := clineBalanceResult(usd, now) + if err := s.accountRepo.UpdateExtra(ctx, account.ID, cnBalanceExtraUpdates(PlatformCline, result, now)); err != nil { + slog.Warn("cn_balance_persist_failed", "account_id", account.ID, "provider", PlatformCline, "error", err) + } else { + result.Persisted = true + } + return result, nil +} + +// queryClineUsage 查询 ClinePass 窗口与积分余额,落快照,并按结果设置或解除两个钱包的 +// 冷却(见 applyClineWallets)。余额查询失败不影响窗口结果。 +func (s *CNProviderQuotaService) queryClineUsage(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { + client := newClineAccountClient(s.cfg, s.httpUpstream, account, s.resolveProxyURL(ctx, account), "CN_QUOTA") + if client.apiKey == "" { + return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NO_APIKEY", "account api_key is empty") + } + now := time.Now().UTC() + result := &CNProviderQuotaProbeResult{ + Provider: PlatformCline, + Source: "cline_usage_limits", + FetchedAt: now.Unix(), + } + tiers, subscribed, err := client.passLimits(ctx) + if err != nil { + status, message, ok := clineFetchFailure(err) + if !ok { + return nil, clineApplicationError(err, "CN_QUOTA") + } + result.StatusCode = status + result.Error = message + result.CredentialValid = status != http.StatusUnauthorized && status != http.StatusForbidden + return result, nil + } + result.StatusCode = http.StatusOK + result.Success = true + result.CredentialValid = true + result.Tiers = tiers + if subscribed { + result.PlanLevel = "ClinePass" + } + + updates := cnQuotaExtraUpdates(PlatformCline, tiers, now) + present := make(map[string]CNQuotaTier, len(tiers)) + for _, tier := range tiers { + present[tier.Window] = tier + } + // 缺席的窗口写 nil:取消订阅后不沿用旧快照(clinePassSubscribed 据此判断);窗口尚未 + // 开始时没有重置时间,同样清掉上一次的值。 + for _, window := range []struct{ name, used, reset string }{ + {"5h", cnExtraSuffix5hUsed, cnExtraSuffix5hReset}, + {"weekly", cnExtraSuffixWeeklyUsed, cnExtraSuffixWeeklyReset}, + {"monthly", cnExtraSuffixMonthlyUsed, cnExtraSuffixMonthlyReset}, + } { + tier, ok := present[window.name] + if !ok { + updates[cnExtraKey(PlatformCline, window.used)] = nil + } + if tier.ResetAt == "" { + updates[cnExtraKey(PlatformCline, window.reset)] = nil + } + } + if usd, err := client.balance(ctx); err != nil { + slog.Warn("cline_balance_probe_failed", "account_id", account.ID, "error", err) + } else { + result.Balance = clineBalanceResult(usd, now) + for key, value := range cnBalanceExtraUpdates(PlatformCline, result.Balance, now) { + updates[key] = value + } + } + if err := s.accountRepo.UpdateExtra(ctx, account.ID, updates); err != nil { + slog.Warn("cn_quota_persist_failed", "account_id", account.ID, "provider", PlatformCline, "error", err) + } else { + result.Persisted = true + if result.Balance != nil { + result.Balance.Persisted = true + } + } + s.applyClineWallets(ctx, account, subscribed, tiers, result.Balance, now) + return result, nil +} + +const ( + clineCreditsLowReason = "cline_credits_low" + // clinePassNoSubscriptionReason 是探测发现未订阅时的原因;与请求侧的未订阅 / 组织账户 + // 共用 clinePassUnavailableReason 前缀,单独区分出探测能证明已恢复的这一种。 + clinePassNoSubscriptionReason = clinePassUnavailableReason + ": no ClinePass subscription" +) + +// applyClineWallets 按探测结果设置或解除两个钱包的冷却: +// - ClinePass:未订阅,或有窗口用满(≥100%)时冷却到该窗口的重置时间;可用时只解除 +// 窗口用满与未订阅的冷却(组织账户、所需订阅档位与推理额度上限反映不到用量接口里, +// 仍等到期后由请求重新确认); +// - 积分:余额低于阈值时冷却,恢复后只解除积分类原因的冷却(组织花费上限无从探测, +// 仍等到期后由请求重新确认)。 +// +// 没有重置时间时冷却 2 个检测周期,由下一次探测续期或解除。 +func (s *CNProviderQuotaService) applyClineWallets(ctx context.Context, account *Account, subscribed bool, tiers []CNQuotaTier, balance *CNProviderBalanceResult, now time.Time) { + fallback := now.Add(cnBalanceCheckCooldown(s.cfg)) + + switch until, reason := clinePassWalletState(subscribed, tiers, now, fallback); { + case until != nil: + s.setClineWalletLimit(ctx, account, clinePassRateLimitKey, *until, reason, now) + case account.isRateLimitActiveForKey(clinePassRateLimitKey): + reason := account.modelRateLimitReason(clinePassRateLimitKey) + if strings.HasPrefix(reason, clinePassLimitReason) || strings.HasPrefix(reason, clinePassNoSubscriptionReason) { + s.setClineWalletLimit(ctx, account, clinePassRateLimitKey, now, "", now) + } + } + + if balance == nil { + return + } + threshold := 0.0 + if s.cfg != nil { + threshold = s.cfg.Gateway.CNProviders.BalanceThreshold + } + switch low := balance.Balance <= 0 || balance.Balance < threshold; { + case low: + s.setClineWalletLimit(ctx, account, clineCreditsRateLimitKey, fallback, + fmt.Sprintf("%s: balance %.4g USD below threshold %.2f", clineCreditsLowReason, balance.Balance, threshold), now) + case account.isRateLimitActiveForKey(clineCreditsRateLimitKey): + reason := account.modelRateLimitReason(clineCreditsRateLimitKey) + if strings.HasPrefix(reason, clineCreditsReason) || strings.HasPrefix(reason, clineCreditsLowReason) { + s.setClineWalletLimit(ctx, account, clineCreditsRateLimitKey, now, "", now) + } + } +} + +// clinePassWalletState 返回 ClinePass 钱包应冷却到的时间;nil 表示可用。 +func clinePassWalletState(subscribed bool, tiers []CNQuotaTier, now, fallback time.Time) (*time.Time, string) { + if !subscribed { + return &fallback, clinePassNoSubscriptionReason + } + var until *time.Time + var exhausted []string + for _, tier := range tiers { + if tier.UsedPercent < 100 { + continue + } + exhausted = append(exhausted, tier.Window) + resetAt := fallback + if parsed := parseSchedulingResetAt(tier.ResetAt); parsed != nil && parsed.After(now) && parsed.Sub(now) <= clinePassMaxCooldown { + resetAt = *parsed + } + if until == nil || resetAt.After(*until) { + until = &resetAt + } + } + if until == nil { + return nil, "" + } + return until, clinePassLimitReason + ": " + strings.Join(exhausted, ", ") + " window used up" +} + +// setClineWalletLimit 写钱包冷却;until 不晚于 now 时即解除。 +func (s *CNProviderQuotaService) setClineWalletLimit(ctx context.Context, account *Account, scope string, until time.Time, reason string, now time.Time) { + setAccountModelRateLimitSnapshot(account, scope, until, reason, now) + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, scope, until, reason); err != nil { + slog.Warn("cline_wallet_limit_set_failed", "account_id", account.ID, "scope", scope, "error", err) + return + } + slog.Info("cline_wallet_limit", "account_id", account.ID, "scope", scope, "until", until.UTC(), "cleared", !until.After(now)) +} diff --git a/backend/internal/service/cn_provider_balance_check_service.go b/backend/internal/service/cn_provider_balance_check_service.go index 5eb36c2a12eb..3a51f7536344 100644 --- a/backend/internal/service/cn_provider_balance_check_service.go +++ b/backend/internal/service/cn_provider_balance_check_service.go @@ -27,6 +27,8 @@ const cnQuotaProbeConcurrency = 4 // 克隆自 AccountExpiryService 的 Start/Stop/runOnce + ticker 骨架。 // 余额探测仅覆盖有公开余额端点的 kimi / deepseek;智谱无余额端点,仅靠响应式 429/402。 // 额度探测覆盖 kimi / zhipu 的 coding plan 账号(deepseek 无 coding 套餐)。 +// Command Code 账号一次探测同时刷新窗口快照与积分余额,积分低于阈值时同样临时停调。 +// Cline 账号同样一次探测刷新 ClinePass 窗口与积分,只冷却用完的一边(见 cline_usage.go)。 type CNProviderBalanceCheckService struct { accountRepo AccountRepository balanceService *CNProviderBalanceService @@ -154,7 +156,8 @@ func (s *CNProviderBalanceCheckService) runOnce() { } // 预算按工作量放大:4 并发 × 15s/批 + payg 每账号 5s,下限 30s 上限 300s。 - batches := (len(quotaTargets) + cnQuotaProbeConcurrency - 1) / cnQuotaProbeConcurrency + accountTargets := s.collectAccountProbeTargets() + batches := (len(quotaTargets) + len(accountTargets) + cnQuotaProbeConcurrency - 1) / cnQuotaProbeConcurrency timeout := 30*time.Second + time.Duration(batches)*15*time.Second + time.Duration(len(paygTargets))*5*time.Second if timeout > 300*time.Second { timeout = 300 * time.Second @@ -188,11 +191,106 @@ func (s *CNProviderBalanceCheckService) runOnce() { wg.Wait() } + if len(accountTargets) > 0 { + sem := make(chan struct{}, cnQuotaProbeConcurrency) + var wg sync.WaitGroup + var mu sync.Mutex + for _, target := range accountTargets { + wg.Add(1) + go func(t accountProbeTarget) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + switch t.check(ctx, t.account, threshold) { + case cnBalancePaused: + mu.Lock() + paused++ + mu.Unlock() + case cnBalanceCleared: + mu.Lock() + cleared++ + mu.Unlock() + } + }(target) + } + wg.Wait() + } + if paused > 0 || cleared > 0 { log.Printf("[CNBalance] paused=%d cleared=%d (threshold=%.2f)", paused, cleared, threshold) } } +// accountProbeCheck 是走账号接口探测的平台的周期检测函数。 +type accountProbeCheck func(ctx context.Context, account *Account, threshold float64) cnBalanceCheckOutcome + +// accountProbeTarget 是一个待检测账号与其平台的检测函数。 +type accountProbeTarget struct { + account *Account + check accountProbeCheck +} + +// collectAccountProbeTargets 选出走账号接口探测的平台上、官方主机上激活的账号,并配上 +// 该平台的检测函数。一次探测同时刷新窗口快照与积分余额: +// - Command Code:手动可调度的账号再按积分余额停调 / 恢复(窗口快照供阈值停调读取); +// - Cline:探测中按结果设置或解除两个钱包的冷却,不停整个账号。 +func (s *CNProviderBalanceCheckService) collectAccountProbeTargets() []accountProbeTarget { + if s.quotaService == nil { + return nil + } + var targets []accountProbeTarget + for _, p := range []struct { + platform string + supported func(*Account) bool + check accountProbeCheck + }{ + {PlatformCommandCode, (*Account).commandCodeUsageSupported, s.checkCommandCode}, + {PlatformCline, (*Account).clineAccountAPISupported, s.checkCline}, + } { + accounts, err := s.accountRepo.ListByPlatform(context.Background(), p.platform) + if err != nil { + log.Printf("[CNBalance] list %s accounts failed: %v", p.platform, err) + continue + } + for i := range accounts { + if account := &accounts[i]; account.IsActive() && p.supported(account) { + targets = append(targets, accountProbeTarget{account: account, check: p.check}) + } + } + } + return targets +} + +// checkCommandCode 探测 Command Code 窗口与积分;手动可调度的账号再按积分余额停调 / 恢复。 +func (s *CNProviderBalanceCheckService) checkCommandCode(ctx context.Context, account *Account, threshold float64) cnBalanceCheckOutcome { + result, err := s.quotaService.QueryUsage(ctx, account.ID) + if err != nil { + log.Printf("[CNBalance] command code usage account %d failed: %v", account.ID, err) + return cnBalanceNoChange + } + if result == nil || !result.Success || result.Balance == nil { + if result != nil && result.Error != "" { + log.Printf("[CNBalance] command code usage account %d error: %s", account.ID, result.Error) + } + return cnBalanceNoChange + } + if !account.Schedulable { + return cnBalanceNoChange + } + return s.applyBalanceResult(ctx, account, result.Balance, threshold) +} + +// checkCline 探测 Cline 窗口与积分;钱包冷却在探测中按结果设置或解除,不停整个账号。 +func (s *CNProviderBalanceCheckService) checkCline(ctx context.Context, account *Account, _ float64) cnBalanceCheckOutcome { + result, err := s.quotaService.QueryUsage(ctx, account.ID) + if err != nil { + log.Printf("[CNBalance] cline usage account %d failed: %v", account.ID, err) + } else if result != nil && !result.Success && result.Error != "" { + log.Printf("[CNBalance] cline usage account %d error: %s", account.ID, result.Error) + } + return cnBalanceNoChange +} + // probeQuota 探测单个 coding plan 账号的滚动窗口用量并落 extra 快照。 // 不在此处做停调/恢复决策:调度阈值评估读取快照统一判定(含暂停账号的续停)。 func (s *CNProviderBalanceCheckService) probeQuota(ctx context.Context, accountID int64, platform string) { @@ -224,7 +322,11 @@ func (s *CNProviderBalanceCheckService) checkOne(ctx context.Context, account *A if err != nil || result == nil || !result.Success { return cnBalanceNoChange } + return s.applyBalanceResult(ctx, account, result, threshold) +} +// applyBalanceResult 按余额探测结果停调或恢复账号。 +func (s *CNProviderBalanceCheckService) applyBalanceResult(ctx context.Context, account *Account, result *CNProviderBalanceResult, threshold float64) cnBalanceCheckOutcome { // 双币种(deepseek CNY+USD)任一币种余额达标即可继续调度;仅当全部低于 // 阈值(或不可用)才停调。 low := !result.Available || allCNBalancesBelowThreshold(result, threshold) @@ -286,15 +388,5 @@ func allCNBalancesBelowThreshold(result *CNProviderBalanceResult, threshold floa // cooldown 返回临时停调持续时长(= 2× 检测周期),与响应式 402/429 路径一致。 func (s *CNProviderBalanceCheckService) cooldown() time.Duration { - minutes := 10 - if s.cfg != nil { - if cfgMin := s.cfg.Gateway.CNProviders.BalanceCheckIntervalMinutes; cfgMin > 0 { - minutes = cfgMin - } - } - cooldown := time.Duration(minutes) * time.Minute * 2 - if cooldown < time.Minute { - cooldown = 10 * time.Minute - } - return cooldown + return cnBalanceCheckCooldown(s.cfg) } diff --git a/backend/internal/service/cn_provider_balance_service.go b/backend/internal/service/cn_provider_balance_service.go index c47468307798..87e8a24ceac2 100644 --- a/backend/internal/service/cn_provider_balance_service.go +++ b/backend/internal/service/cn_provider_balance_service.go @@ -123,6 +123,12 @@ func (s *CNProviderBalanceService) QueryBalanceForAccount(ctx context.Context, a } func (s *CNProviderBalanceService) queryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { + if account.IsCommandCode() { + return s.queryCommandCodeBalance(ctx, account) + } + if account.IsCline() { + return s.queryClineBalance(ctx, account) + } provider := account.Platform if provider != PlatformKimi && provider != PlatformDeepseek { return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_ENDPOINT", "account provider has no balance endpoint") @@ -219,28 +225,33 @@ func (s *CNProviderBalanceService) queryBalanceForAccount(ctx context.Context, a result.Available = available result.Success = true - balanceUpdates := make([]any, 0, len(entries)) - for _, entry := range entries { + updates := cnBalanceExtraUpdates(provider, result, now) + if err := s.accountRepo.UpdateExtra(ctx, account.ID, updates); err != nil { + slog.Warn("cn_balance_persist_failed", "account_id", account.ID, "provider", provider, "error", err) + } else { + result.Persisted = true + } + return result, nil +} + +// cnBalanceExtraUpdates 构造 provider 维度的余额快照更新。 +func cnBalanceExtraUpdates(provider string, result *CNProviderBalanceResult, now time.Time) map[string]any { + balanceUpdates := make([]any, 0, len(result.Balances)) + for _, entry := range result.Balances { balanceUpdates = append(balanceUpdates, map[string]any{ "currency": entry.Currency, "balance": entry.Balance, }) } - updates := map[string]any{ + return map[string]any{ cnExtraKey(provider, cnBalanceExtraSuffixBalance): result.Balance, cnExtraKey(provider, cnBalanceExtraSuffixCurrency): result.Currency, - cnExtraKey(provider, cnBalanceExtraSuffixAvailable): available, + cnExtraKey(provider, cnBalanceExtraSuffixAvailable): result.Available, cnExtraKey(provider, cnBalanceExtraSuffixUpdated): now.Format(time.RFC3339), cnExtraKey(provider, cnBalanceExtraSuffixBalances): balanceUpdates, // 余额探测成功即清除响应式 402/429 写下的 balance_low 标记。 cnExtraKey(provider, cnBalanceExtraSuffixLow): false, } - if err := s.accountRepo.UpdateExtra(ctx, account.ID, updates); err != nil { - slog.Warn("cn_balance_persist_failed", "account_id", account.ID, "provider", provider, "error", err) - } else { - result.Persisted = true - } - return result, nil } // loadPayGAccount 加载 payg 模式的国产供应商账号(余额仅对 payg 有意义;coding 走额度)。 @@ -261,6 +272,20 @@ func validatePayGAccount(account *Account) error { if account == nil { return infraerrors.New(http.StatusNotFound, "CN_BALANCE_ACCOUNT_NOT_FOUND", "account not found") } + // Cline 积分余额只查官方主机。 + if account.IsCline() { + if !account.clineAccountAPISupported() { + return infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_ENDPOINT", "cline balance is only available for api key accounts on the official host") + } + return nil + } + // Command Code 积分余额与套餐窗口同源,只查官方主机。 + if account.IsCommandCode() { + if !account.commandCodeUsageSupported() { + return infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_ENDPOINT", "command code balance is only available for api key accounts on the official host") + } + return nil + } if !account.IsCNProvider() { return infraerrors.New(http.StatusBadRequest, "CN_BALANCE_INVALID_PLATFORM", "account is not a CN provider account") } diff --git a/backend/internal/service/cn_provider_quota_service.go b/backend/internal/service/cn_provider_quota_service.go index 48fe0a680337..def7586fa493 100644 --- a/backend/internal/service/cn_provider_quota_service.go +++ b/backend/internal/service/cn_provider_quota_service.go @@ -62,9 +62,13 @@ type CNProviderQuotaProbeResult struct { FetchedAt int64 `json:"fetched_at"` Persisted bool `json:"persisted"` Error string `json:"error,omitempty"` + + // Balance 为同一次探测得到的余额(Command Code 积分与窗口同源),其余供应商为空。 + Balance *CNProviderBalanceResult `json:"balance,omitempty"` } -// CNProviderQuotaService 探测 Kimi / Zhipu Coding Plan 的滚动窗口用量。 +// CNProviderQuotaService 探测 Kimi / Zhipu / MiniMax Coding Plan、OpenCode Go、 +// Command Code 与 Cline(ClinePass)的滚动窗口用量。 type CNProviderQuotaService struct { accountRepo AccountRepository proxyRepo ProxyRepository @@ -131,6 +135,12 @@ func (s *CNProviderQuotaService) QueryUsageForAccount(ctx context.Context, accou func (s *CNProviderQuotaService) queryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { provider := account.GetCodingPlanProvider() + if provider == PlatformCommandCode { + return s.queryCommandCodeUsage(ctx, account) + } + if provider == PlatformCline { + return s.queryClineUsage(ctx, account) + } if provider != PlatformKimi && provider != PlatformZhipu && provider != PlatformMiniMax && provider != PlatformOpenCodeGo { return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "account is not a kimi/zhipu/minimax coding plan or opencode go account") } @@ -290,6 +300,18 @@ func validateCodingPlanAccount(account *Account) error { if account.IsOpenCodeGoPlan() { return nil } + if account.IsCommandCode() { + if !account.commandCodeUsageSupported() { + return infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "command code usage is only available for api key accounts on the official host") + } + return nil + } + if account.IsCline() { + if !account.clineAccountAPISupported() { + return infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "cline usage is only available for api key accounts on the official host") + } + return nil + } if account.IsOpenCodeGo() { return infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "opencode zen accounts have no subscription quota window") } diff --git a/backend/internal/service/cn_providers_test.go b/backend/internal/service/cn_providers_test.go index 44becd4e5bee..b78dfbebebed 100644 --- a/backend/internal/service/cn_providers_test.go +++ b/backend/internal/service/cn_providers_test.go @@ -668,8 +668,9 @@ func TestAdaptiveProtocolBaseURLs(t *testing.T) { }{ {"kimi payg", PlatformKimi, AccountModePayG, DefaultKimiPayGBaseURL, DefaultKimiPayGAnthropicBaseURL, DefaultKimiPayGBaseURL}, {"kimi coding", PlatformKimi, AccountModeCoding, DefaultKimiCodingBaseURL, DefaultKimiCodingAnthropicBaseURL, DefaultKimiCodingBaseURL}, - {"zhipu payg", PlatformZhipu, AccountModePayG, DefaultZhipuPayGBaseURL, DefaultZhipuAnthropicBaseURL, DefaultZhipuPayGBaseURL}, - {"zhipu coding", PlatformZhipu, AccountModeCoding, DefaultZhipuCodingBaseURL, DefaultZhipuAnthropicBaseURL, DefaultZhipuCodingBaseURL}, + // 智谱没有原生 Responses 端点:profile 不登记 responses 条目,默认基址为空。 + {"zhipu payg", PlatformZhipu, AccountModePayG, DefaultZhipuPayGBaseURL, DefaultZhipuAnthropicBaseURL, ""}, + {"zhipu coding", PlatformZhipu, AccountModeCoding, DefaultZhipuCodingBaseURL, DefaultZhipuAnthropicBaseURL, ""}, {"deepseek", PlatformDeepseek, AccountModePayG, DefaultDeepseekBaseURL, DefaultDeepseekAnthropicBaseURL, DefaultDeepseekBaseURL}, {"minimax payg", PlatformMiniMax, AccountModePayG, DefaultMiniMaxBaseURL, DefaultMiniMaxAnthropicBaseURL, DefaultMiniMaxBaseURL}, {"minimax coding", PlatformMiniMax, AccountModeCoding, DefaultMiniMaxBaseURL, DefaultMiniMaxAnthropicBaseURL, DefaultMiniMaxBaseURL}, diff --git a/backend/internal/service/codex_gateway_borrow_http_test.go b/backend/internal/service/codex_gateway_borrow_http_test.go index 5e286a24edca..b0127c75332a 100644 --- a/backend/internal/service/codex_gateway_borrow_http_test.go +++ b/backend/internal/service/codex_gateway_borrow_http_test.go @@ -112,91 +112,6 @@ func TestCodexGatewayBorrowHTTPPreparationExcludedFromFirstOutputBudgets(t *test require.False(t, svc.isOpenAIAccountModelRuntimeBlocked(account, "gpt-6-astra")) } -func TestCodexGatewayBorrowHTTPRecoveryPreservesCallerContext(t *testing.T) { - for _, lifecycle := range []string{"cancel", "deadline"} { - for _, replacedGuard := range []bool{false, true} { - name := lifecycle + "/prepared_context" - if replacedGuard { - name = lifecycle + "/replaced_guard" - } - t.Run(name, func(t *testing.T) { - type callerValueKey struct{} - values := context.WithValue(context.Background(), callerValueKey{}, "retained") - caller, cancel := context.WithCancel(values) - if lifecycle == "deadline" { - cancel() - caller, cancel = context.WithTimeout(values, 250*time.Millisecond) - } - defer cancel() - transport, _ := detachUpstreamContext(caller) - transport = WithHTTPUpstreamProfile(transport, HTTPUpstreamProfileOpenAI) - var oldGuard *openAIFirstOutputHeaderGuard - if replacedGuard { - transport, oldGuard = newOpenAIFirstOutputHeaderGuard(transport, func() {}, time.Now().Add(time.Minute)) - defer oldGuard.close() - } - account := borrowCoreAccount(2) - var probes, businessCalls atomic.Int32 - borrow := newBorrowCoreTest(t, borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { - if probes.Add(1) == 1 { - time.Sleep(10 * time.Millisecond) - return borrowCoreResponse("gpt-6-astra", "", "synthetic-state"), nil - } - return borrowCoreResponse("gpt-6-astra", "", ""), nil - }), account) - borrowCoreCandidate(borrow, time.Now().Add(time.Minute)) - body := []byte(`{"model":"gpt-6-astra","input":"synthetic recovery"}`) - req, err := http.NewRequestWithContext(transport, http.MethodPost, chatgptCodexURL, bytes.NewReader(body)) - require.NoError(t, err) - req.Header.Set("Authorization", "Bearer synthetic-target") - recovery := &openAIReasoningRecoveryState{ctx: caller, account: account, token: "synthetic-target", enabled: true, retryUsed: true, retryBody: body} - recovery.identity = openAIReasoningRequestIdentity(account, req, "", recovery.token) - defer recovery.Close() - req, _, err = recovery.PrepareRequest(req, body, "") - require.NoError(t, err) - prepared, elapsed, err := borrow.gateway.prepareCodexGatewayBorrowHTTP(caller, req, account, "gpt-6-astra", "") - require.NoError(t, err) - require.Equal(t, int32(2), probes.Load(), "fixture must qualify the recovery request") - require.Positive(t, elapsed) - if oldGuard != nil { - oldGuard.close() - } - adjusted := codexGatewayBorrowHTTPPreparedContext(caller, prepared.Context(), replacedGuard, recovery.RecoveryAttempt()) - if !replacedGuard { - require.Same(t, prepared.Context(), adjusted) - } - callerDeadline, callerHasDeadline := caller.Deadline() - deadline, hasDeadline := adjusted.Deadline() - require.Equal(t, callerHasDeadline, hasDeadline) - require.Equal(t, callerDeadline, deadline) - require.Equal(t, "retained", adjusted.Value(callerValueKey{})) - require.Equal(t, HTTPUpstreamProfileOpenAI, HTTPUpstreamProfileFromContext(adjusted)) - require.True(t, HTTPUpstreamRedirectsDisabled(adjusted)) - require.NotNil(t, adjusted.Value(codexGatewayBorrowHTTPPreparationContextKey{})) - require.NoError(t, adjusted.Err()) - require.NoError(t, context.Cause(adjusted), "replaced header guard must not supply the cancellation cause") - prepared = prepared.WithContext(adjusted) - if lifecycle == "cancel" { - cancel() - } - select { - case <-adjusted.Done(): - case <-time.After(time.Second): - t.Fatal("qualified recovery lost the caller's lifecycle") - } - borrow.gateway.httpUpstream = borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { - businessCalls.Add(1) - return nil, errors.New("expired recovery must not dispatch") - }) - _, err = borrow.gateway.doOpenAICodexUpstream(prepared, account, "", "gpt-6-astra") - require.ErrorIs(t, err, adjusted.Err()) - require.Equal(t, int32(2), probes.Load()) - require.Zero(t, businessCalls.Load()) - }) - } - } -} - func TestCodexGatewayBorrowHTTPFailureUsesAccountNeutralFailover(t *testing.T) { for _, cause := range []error{errors.New("source proxy connection refused: complete diagnostic"), ErrCodexGatewayBorrowBusy, ErrCodexGatewayBorrowUnavailable, ErrCodexGatewayBorrowChanged} { t.Run(cause.Error(), func(t *testing.T) { @@ -276,72 +191,6 @@ func TestCodexGatewayBorrowHTTPPreparedDispatchNeverReprepares(t *testing.T) { } } -func TestCodexGatewayBorrowHTTPPreparedDispatchPreservesFrozenRequest(t *testing.T) { - for _, changedCookie := range []bool{false, true} { - name := "canonical_cookie" - if changedCookie { - name = "changed_cookie" - } - t.Run(name, func(t *testing.T) { - gin.SetMode(gin.TestMode) - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - body := []byte(`{"model":"gpt-6-astra","input":[{"type":"reasoning","encrypted_content":"synthetic-cipher"}]}`) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) - account := borrowCoreAccount(2) - var probes, businessCalls atomic.Int32 - borrow := newBorrowCoreTest(t, borrowCoreProbe(func(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { - if probes.Add(1) == 1 { - return borrowCoreResponse("gpt-6-astra", "", "synthetic-state"), nil - } - return borrowCoreResponse("gpt-6-astra", "", ""), nil - }), account) - borrowCoreCandidate(borrow, time.Now().Add(time.Minute)) - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, chatgptCodexURL, bytes.NewReader(body)) - require.NoError(t, err) - req.Header.Set("Authorization", "Bearer synthetic-target") - req.Header.Set("Cookie", "target=kept") - recovery := &openAIReasoningRecoveryState{ctx: context.Background(), c: c, account: account, enabled: true} - c.Set(openAIReasoningRecoveryContextKey, recovery) - req, _, err = recovery.PrepareRequest(req, body, "") - require.NoError(t, err) - prepared, _, err := borrow.gateway.prepareCodexGatewayBorrowHTTP(context.Background(), req, account, "gpt-6-astra", "") - require.NoError(t, err) - guardCtx, guard := newOpenAIFirstOutputHeaderGuard(prepared.Context(), func() {}, time.Now().Add(time.Minute)) - defer guard.close() - prepared = prepared.WithContext(guardCtx) - if changedCookie { - prepared.Header.Set("Cookie", "target=kept; __oailb=stale") - } - finalized, err := borrow.gateway.applyCodexGatewayBorrowHTTP(prepared, account, "gpt-6-astra", "") - require.NoError(t, err) - if changedCookie { - require.NotSame(t, prepared, finalized) - } else { - require.Same(t, prepared, finalized) - } - recovery.BindDiagnosticRequest(body, finalized) - borrow.gateway.httpUpstream = borrowCoreProbe(func(got *http.Request, _ string, _ int64, _ int, _ *tlsfingerprint.Profile) (*http.Response, error) { - businessCalls.Add(1) - require.Same(t, finalized, got, "cached dispatch must keep the diagnostic's plaintext request") - require.Nil(t, got.GetBody, "diagnostics must not restore transparent POST replay") - diagnostic := buildOpenAIContinuationDiagnostic(c, body, got, body, []byte(`{"error":{"message":"bad request"}}`), "unclassified_bad_request") - require.NotNil(t, diagnostic) - require.Equal(t, "frozen_request", diagnostic.Wire.BodySource) - require.False(t, diagnostic.Wire.InspectionLimited) - return &http.Response{StatusCode: http.StatusBadRequest, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("synthetic rejection"))}, nil - }) - oldCompression := codexRequestZstd.Load() - SetCodexRequestZstdEnabled(false) - defer SetCodexRequestZstdEnabled(oldCompression) - resp, err := borrow.gateway.doOpenAICodexUpstream(finalized, account, "", "gpt-6-astra") - require.NoError(t, err) - _ = resp.Body.Close() - require.Equal(t, int32(2), probes.Load()) - require.Equal(t, int32(1), businessCalls.Load()) - }) - } -} - type borrowHTTPUnreadBody struct{ reads atomic.Int32 } func (r *borrowHTTPUnreadBody) Read([]byte) (int, error) { diff --git a/backend/internal/service/command_code.go b/backend/internal/service/command_code.go new file mode 100644 index 000000000000..41b9d49cf3b7 --- /dev/null +++ b/backend/internal/service/command_code.go @@ -0,0 +1,27 @@ +package service + +// Command Code Provider API(https://commandcode.ai/docs/provider):同一 API Key 下 +// 按模型分流到三种原生端点,基址 https://api.commandcode.ai/provider/v1: +// - Claude 系模型只在 /messages(Anthropic Messages)提供,发往其他端点返回 400; +// - OpenAI 与开源模型提供 /chat/completions,多数也提供 /responses。 +// 每个模型支持的端点由 /models 的 supported_endpoints 给出(公开接口),网关据此在 +// 模型支持入站协议时同协议直通(见 model_protocol_catalog.go);目录就绪前使用下面的 +// 内置规则。模型名可带厂商前缀(如 deepseek/deepseek-v4-flash),原样发往上游。 + +// DefaultCommandCodeTestModel 是管理端连接测试未指定模型时使用的模型(Chat Completions)。 +const DefaultCommandCodeTestModel = "deepseek/deepseek-v4-flash" + +// DefaultCommandCodeProtocolRules 是 Command Code 的内置分流表:Claude 只走 Anthropic; +// GPT 同时支持 Responses 与 Chat Completions(入站同协议直通,其余入站首选 Responses); +// 其余模型只保证 Chat Completions,是否支持 Responses 以上游模型目录为准。 +func DefaultCommandCodeProtocolRules() []ProtocolRule { + return []ProtocolRule{ + {Pattern: "claude-*", Protocol: APIProtocolAnthropic}, + {Pattern: "gpt-*", Protocol: APIProtocolResponses, Protocols: []string{APIProtocolResponses, APIProtocolChatCompletions}}, + } +} + +// IsCommandCode 报告账号是否为 Command Code 平台账号。 +func (a *Account) IsCommandCode() bool { + return a != nil && a.Platform == PlatformCommandCode +} diff --git a/backend/internal/service/command_code_test.go b/backend/internal/service/command_code_test.go new file mode 100644 index 000000000000..aabc96db5b62 --- /dev/null +++ b/backend/internal/service/command_code_test.go @@ -0,0 +1,176 @@ +//go:build unit + +package service + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// commandCodeTestAccount 只带 api_key:端点与分流规则全部来自 provider profile。 +func commandCodeTestAccount(id int64) *Account { + return &Account{ + ID: id, + Name: "cc", + Platform: PlatformCommandCode, + Type: AccountTypeAPIKey, + Status: StatusActive, + Concurrency: 1, + Credentials: map[string]any{"api_key": "user_test_key"}, + } +} + +func TestCommandCodeProfileRoutesByModel(t *testing.T) { + account := commandCodeTestAccount(1) + require.True(t, account.IsMultiProtocolAPIKey()) + require.True(t, account.routesByModel()) + require.False(t, account.RoutesProtocolByInbound()) + require.Equal(t, APIProtocolAdaptive, account.GetAPIProtocol()) + require.Equal(t, DefaultCommandCodeBaseURL, account.GetOpenAIBaseURL()) + require.Equal(t, DefaultCommandCodeAnthropicBaseURL, account.GetCNProtocolBaseURL(APIProtocolAnthropic)) + require.Equal(t, DefaultCommandCodeBaseURL, account.GetCNProtocolBaseURL(APIProtocolResponses)) + + // 未取到模型目录时按内置规则:Claude 只走 Anthropic;GPT 支持 Responses 与 Chat + // Completions(同协议直通,Anthropic 入站走首选 Responses);其余只保证 Chat Completions。 + for _, inbound := range []string{APIProtocolChatCompletions, APIProtocolResponses, APIProtocolAnthropic} { + require.Equal(t, APIProtocolAnthropic, resolveUpstreamProtocol(account, inbound, "claude-sonnet-4-6", nil), inbound) + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, inbound, "deepseek/deepseek-v4-flash", nil), inbound) + } + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(account, APIProtocolResponses, "gpt-5.5", nil)) + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolChatCompletions, "gpt-5.5", nil)) + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(account, APIProtocolAnthropic, "gpt-5.5", nil)) + + // 显式协议优先于内置规则。 + account.Credentials["api_protocol"] = APIProtocolChatCompletions + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolAnthropic, "claude-sonnet-4-6", nil)) +} + +func TestCommandCodeNativeAnthropicTargetURL(t *testing.T) { + svc := &OpenAIGatewayService{} + targetURL, err := svc.nativeAnthropicTargetURL(commandCodeTestAccount(2)) + require.NoError(t, err) + require.Equal(t, "https://api.commandcode.ai/provider/v1/messages", targetURL) +} + +func TestAccountTestService_CommandCodeClaudeUsesAnthropicMessages(t *testing.T) { + account := commandCodeTestAccount(501) + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNAnthropicTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "claude-sonnet-4-6", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://api.commandcode.ai/provider/v1/messages", upstream.requests[0].URL.String()) + require.Equal(t, "user_test_key", upstream.requests[0].Header.Get("x-api-key")) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) +} + +// 连接测试与转发共用端点拼接:按模型分流的平台,Anthropic 基址带 /v1 也合法。 +func TestAccountTestService_CommandCodeAnthropicBaseWithVersion(t *testing.T) { + account := commandCodeTestAccount(504) + account.Credentials["api_base_urls"] = map[string]any{APIProtocolAnthropic: "https://api.commandcode.ai/provider/v1"} + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNAnthropicTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "claude-sonnet-4-6", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://api.commandcode.ai/provider/v1/messages", upstream.requests[0].URL.String()) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) + + target, err := (&OpenAIGatewayService{cfg: rawChatCompletionsTestConfig()}).nativeAnthropicTargetURL(account) + require.NoError(t, err) + require.Equal(t, upstream.requests[0].URL.String(), target) +} + +// 连接测试与网关同一判定:模型目录决定协议时,测试也按目录选择端点。 +func TestAccountTestService_CommandCodeUsesModelCatalog(t *testing.T) { + base := fmt.Sprintf("http://cc-test-%d.example/provider/v1", time.Now().UnixNano()) + account := commandCodeTestAccount(505) + account.Credentials["api_base_urls"] = map[string]any{ + APIProtocolChatCompletions: base, + APIProtocolResponses: base, + } + url := buildOpenAIModelsURL(base) + upstreamModelProtocols.store(modelProtocolCatalogKey(account, url), + map[string][]string{"vendor/responses-only": {APIProtocolResponses}}, nil, time.Now()) + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNResponsesTestResponse()) + svc.openaiGatewayService = &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "vendor/responses-only", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, base+"/responses", upstream.requests[0].URL.String()) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) +} + +func TestAccountTestService_CommandCodeGPTUsesResponses(t *testing.T) { + account := commandCodeTestAccount(502) + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNResponsesTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "gpt-5.5", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://api.commandcode.ai/provider/v1/responses", upstream.requests[0].URL.String()) + require.Equal(t, "Bearer user_test_key", upstream.requests[0].Header.Get("Authorization")) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) +} + +func TestAccountTestService_CommandCodeEmptyModelUsesProfileDefault(t *testing.T) { + account := commandCodeTestAccount(503) + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNChatTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://api.commandcode.ai/provider/v1/chat/completions", upstream.requests[0].URL.String()) + require.Equal(t, DefaultCommandCodeTestModel, gjson.GetBytes(upstream.lastBody, "model").String()) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) +} + +// 仅在 profile 登记、按入站协议分流的新供应商:adaptive 测试走原生端点探测, +// 跳过 profile 未提供的 Anthropic 端点,不再落入 Claude 测试路径。 +func TestAccountTestService_ByInboundProviderSkipsMissingNativeEndpoints(t *testing.T) { + const platform = "test_inbound_only_chat" + providerProfiles[platform] = &ProviderProfile{ + Platform: platform, DefaultMode: AccountModePayG, Routing: ProviderRoutingByInbound, + DefaultTestModel: "inbound-test-model", + Modes: map[string]ProviderEndpoints{AccountModePayG: {BaseURLs: map[string]string{ + APIProtocolChatCompletions: "https://inbound.example.com/v1", + }}}, + } + t.Cleanup(func() { delete(providerProfiles, platform) }) + + account := &Account{ + ID: 601, Name: "inbound", Platform: platform, Type: AccountTypeAPIKey, Status: StatusActive, Concurrency: 1, + Credentials: map[string]any{"api_key": "sk-inbound", "api_protocol": APIProtocolAdaptive}, + } + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNChatTestResponse()) + c, recorder := newTestContext() + + err := svc.TestAccountConnection(c, account.ID, "", "hi", AccountTestModeDefault) + + require.NoError(t, err) + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://inbound.example.com/v1/chat/completions", upstream.requests[0].URL.String()) + require.Equal(t, "inbound-test-model", gjson.GetBytes(upstream.lastBody, "model").String()) + require.Contains(t, recorder.Body.String(), `"type":"test_complete"`) +} + +// 官方主机不做上游计费探测:中转站的计费接口在官方 API 上不存在,只会把 Key 发往官方主机。 +func TestCommandCodeOfficialHostSkipsUpstreamBillingProbe(t *testing.T) { + require.True(t, upstreamBillingProbeTargetIsOfficialAPI(DefaultCommandCodeBaseURL)) + require.True(t, upstreamBillingProbeTargetIsOfficialAPI(DefaultCommandCodeAnthropicBaseURL)) +} diff --git a/backend/internal/service/command_code_usage.go b/backend/internal/service/command_code_usage.go new file mode 100644 index 000000000000..4781b4c9031a --- /dev/null +++ b/backend/internal/service/command_code_usage.go @@ -0,0 +1,511 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" +) + +// Command Code 的积分与用量窗口来自 /alpha 下四个只读接口(官方 CLI /usage 同源, +// 社区插件实测复核),全部使用账号 API Key(Bearer): +// +// GET /alpha/whoami -> {user, org};个人账户 org 为 null +// GET /alpha/billing/credits[?orgId=] -> {credits:{monthlyCredits, purchasedCredits, freeCredits}, +// windowLimits:{limited, fiveHour, weekly}} +// GET /alpha/billing/subscriptions[?orgId=] -> {data:{planId, currentPeriodStart, currentPeriodEnd}} +// GET /alpha/usage/summary[?orgId=][&since=] -> {totalCost, totalMonthlyCredits} +// +// 积分单位为美元。windowLimits 与 credits 平级;fiveHour / weekly 的 used、cap 为积分, +// resetAt 为 epoch 毫秒;limited=false(按量计费)时没有滚动窗口。月度窗口上游只给 +// 剩余量(credits.monthlyCredits),上限按「本周期已用 + 剩余」还原。个人账户不能带 +// orgId 参数(空值返回 400),只有组织账户才带。 +// +// 订阅套餐的滚动窗口只限制套餐内积分:有充值积分时请求跳过窗口检查,此时窗口快照 +// 不参与阈值停调与 429 冷却(见 commandCodeWindowsBinding),已有的窗口冷却在探测到 +// 充值积分后解除(见 clearCommandCodeWindowCooldown)。 + +const ( + commandCodeAPIBase = "https://api.commandcode.ai" + + // commandCodeExtraSuffixPurchasedCredits 记录充值积分,供阈值停调与 429 冷却判断 + // 滚动窗口是否生效。 + commandCodeExtraSuffixPurchasedCredits = "purchased_credits" +) + +var errCommandCodeInvalidResponse = errors.New("invalid command code usage response") + +// commandCodePlanNames 与官方 CLI 的 planId 展示名一致;未收录的原样展示。 +var commandCodePlanNames = map[string]string{ + "individual-go": "Go", + "individual-goat": "GOAT", + "individual-pro": "Pro", + "individual-pro-v1": "Pro", + "individual-provider": "Provider", + "individual-max": "Max", + "individual-ultra": "Ultra", + "teams-pro": "Teams Pro", +} + +// commandCodeUsageSupported 报告账号能否查询 Command Code 用量:API Key 账号且推理 +// 端点指向官方主机。自定义中转的 Key 不发往官方用量接口。 +func (a *Account) commandCodeUsageSupported() bool { + return a.IsCommandCode() && a.Type == AccountTypeAPIKey && isOfficialCommandCodeHost(a.GetOpenAIBaseURL()) +} + +// commandCodeNumber 兼容数字与数字字符串。 +type commandCodeNumber float64 + +func (n *commandCodeNumber) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *n = 0 + return nil + } + if data[0] == '"' { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil { + return err + } + *n = commandCodeNumber(v) + return nil + } + var v float64 + if err := json.Unmarshal(data, &v); err != nil { + return err + } + *n = commandCodeNumber(v) + return nil +} + +type commandCodeWindow struct { + Used commandCodeNumber `json:"used"` + Cap commandCodeNumber `json:"cap"` + ResetAt commandCodeNumber `json:"resetAt"` +} + +type commandCodeCredits struct { + MonthlyCredits commandCodeNumber `json:"monthlyCredits"` + PurchasedCredits commandCodeNumber `json:"purchasedCredits"` + FreeCredits commandCodeNumber `json:"freeCredits"` +} + +type commandCodeCreditsPayload struct { + Credits *commandCodeCredits `json:"credits"` + WindowLimits struct { + Limited bool `json:"limited"` + FiveHour *commandCodeWindow `json:"fiveHour"` + Weekly *commandCodeWindow `json:"weekly"` + } `json:"windowLimits"` +} + +type commandCodeSubscriptionPayload struct { + Data struct { + PlanID string `json:"planId"` + CurrentPeriodStart string `json:"currentPeriodStart"` + CurrentPeriodEnd string `json:"currentPeriodEnd"` + } `json:"data"` +} + +type commandCodeSummaryPayload struct { + TotalCost *commandCodeNumber `json:"totalCost"` + TotalMonthlyCredits *commandCodeNumber `json:"totalMonthlyCredits"` +} + +// commandCodeUsage 是一次取数的汇总;subscription / summary 取数失败时为 nil,并置 +// periodFetchFailed:此时无从判断月度窗口是否存在,不能当作没有月度窗口。 +type commandCodeUsage struct { + credits commandCodeCreditsPayload + subscription *commandCodeSubscriptionPayload + summary *commandCodeSummaryPayload + periodFetchFailed bool +} + +// commandCodeHTTPError 是 /alpha 接口的非 2xx 响应。 +type commandCodeHTTPError struct { + status int + body string +} + +func (e *commandCodeHTTPError) Error() string { + return fmt.Sprintf("HTTP %d: %s", e.status, e.body) +} + +func (e *commandCodeHTTPError) unauthorized() bool { + return e.status == http.StatusUnauthorized || e.status == http.StatusForbidden +} + +func isCommandCodeAuthError(err error) bool { + var httpErr *commandCodeHTTPError + return errors.As(err, &httpErr) && httpErr.unauthorized() +} + +type commandCodeUsageClient struct { + cfg *config.Config + httpUpstream HTTPUpstream + account *Account + apiKey string + proxyURL string +} + +// get 请求一个 /alpha 接口并解码 JSON。 +func (c *commandCodeUsageClient) get(ctx context.Context, path string, query url.Values, out any) error { + target := commandCodeAPIBase + path + if len(query) > 0 { + target += "?" + query.Encode() + } + validatedURL, err := cnValidateProbeURL(c.cfg, target) + if err != nil { + return infraerrors.New(http.StatusForbidden, "CN_QUOTA_URL_REJECTED", err.Error()) + } + callCtx, cancel := context.WithTimeout(ctx, cnQuotaUpstreamTimeout) + defer cancel() + req, err := http.NewRequestWithContext(callCtx, http.MethodGet, validatedURL, nil) + if err != nil { + return infraerrors.Newf(http.StatusInternalServerError, "CN_QUOTA_REQUEST_BUILD_FAILED", "build request: %v", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Accept", "application/json") + c.account.ApplyHeaderOverrides(req.Header) + + resp, err := c.httpUpstream.Do(req, c.proxyURL, c.account.ID, maxInt(c.account.Concurrency, 1)) + if err != nil { + return fmt.Errorf("upstream request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(resp.Body, cnQuotaMaxBodyBytes)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return &commandCodeHTTPError{status: resp.StatusCode, body: truncate(strings.TrimSpace(string(body)), 240)} + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("%w: %s: %v", errCommandCodeInvalidResponse, path, err) + } + return nil +} + +// fetch 取积分与窗口;withPeriod 时再取订阅周期与本周期用量以还原月度窗口。 +// whoami 用来取 orgId,任何失败都整体返回:无从判断是否组织账户时,按个人账户查会把 +// 组织账号查成成员的个人钱包,并据此按余额停调。 +func (c *commandCodeUsageClient) fetch(ctx context.Context, withPeriod bool) (*commandCodeUsage, error) { + var whoami struct { + Org *struct { + ID string `json:"id"` + } `json:"org"` + } + query := url.Values{} + if err := c.get(ctx, "/alpha/whoami", nil, &whoami); err != nil { + return nil, err + } + if whoami.Org != nil && strings.TrimSpace(whoami.Org.ID) != "" { + query.Set("orgId", strings.TrimSpace(whoami.Org.ID)) + } + + usage := &commandCodeUsage{} + if err := c.get(ctx, "/alpha/billing/credits", query, &usage.credits); err != nil { + return nil, err + } + if usage.credits.Credits == nil { + return nil, fmt.Errorf("%w: credits missing", errCommandCodeInvalidResponse) + } + if !withPeriod { + return usage, nil + } + + var subscription commandCodeSubscriptionPayload + if err := c.get(ctx, "/alpha/billing/subscriptions", query, &subscription); err != nil { + if isCommandCodeAuthError(err) { + return nil, err + } + slog.Warn("command_code_subscription_failed", "account_id", c.account.ID, "error", err) + usage.periodFetchFailed = true + return usage, nil + } + usage.subscription = &subscription + periodStart := strings.TrimSpace(subscription.Data.CurrentPeriodStart) + if periodStart == "" { + return usage, nil + } + summaryQuery := url.Values{} + for key, values := range query { + summaryQuery[key] = values + } + summaryQuery.Set("since", periodStart) + var summary commandCodeSummaryPayload + if err := c.get(ctx, "/alpha/usage/summary", summaryQuery, &summary); err != nil { + if isCommandCodeAuthError(err) { + return nil, err + } + slog.Warn("command_code_usage_summary_failed", "account_id", c.account.ID, "error", err) + usage.periodFetchFailed = true + return usage, nil + } + usage.summary = &summary + return usage, nil +} + +func isApplicationError(err error) bool { + var appErr *infraerrors.ApplicationError + return errors.As(err, &appErr) +} + +func commandCodePercent(used, limit float64) float64 { + percent := used / limit * 100 + if percent < 0 { + percent = 0 + } + return math.Round(percent*100) / 100 +} + +// tiers 返回 5h / weekly(限额套餐)与月度窗口。 +func (u *commandCodeUsage) tiers() []CNQuotaTier { + var tiers []CNQuotaTier + limits := u.credits.WindowLimits + if limits.Limited { + for _, item := range []struct { + window string + limit *commandCodeWindow + }{ + {window: "5h", limit: limits.FiveHour}, + {window: "weekly", limit: limits.Weekly}, + } { + if item.limit == nil || item.limit.Cap <= 0 { + continue + } + tiers = append(tiers, CNQuotaTier{ + Window: item.window, + UsedPercent: commandCodePercent(float64(item.limit.Used), float64(item.limit.Cap)), + ResetAt: cnMillisToRFC3339(int64(item.limit.ResetAt)), + }) + } + } + if u.summary != nil && u.subscription != nil { + used := 0.0 + switch { + case u.summary.TotalMonthlyCredits != nil: + used = float64(*u.summary.TotalMonthlyCredits) + case u.summary.TotalCost != nil: + used = float64(*u.summary.TotalCost) + } + if limit := used + float64(u.credits.Credits.MonthlyCredits); limit > 0 { + tiers = append(tiers, CNQuotaTier{ + Window: "monthly", + UsedPercent: commandCodePercent(used, limit), + ResetAt: cnNormalizeResetTime(u.subscription.Data.CurrentPeriodEnd), + }) + } + } + return tiers +} + +func (u *commandCodeUsage) planName() string { + if u.subscription == nil { + return "" + } + planID := strings.TrimSpace(u.subscription.Data.PlanID) + if name, ok := commandCodePlanNames[planID]; ok { + return name + } + return planID +} + +// balance 返回可用积分合计(套餐剩余 + 充值 + 赠送,美元)。 +func (u *commandCodeUsage) balance(now time.Time) *CNProviderBalanceResult { + credits := u.credits.Credits + total := float64(credits.MonthlyCredits + credits.PurchasedCredits + credits.FreeCredits) + total = math.Round(total*10000) / 10000 + return &CNProviderBalanceResult{ + Provider: PlatformCommandCode, + Success: true, + Balance: total, + Currency: "USD", + Balances: []CNProviderBalanceEntry{{Currency: "USD", Balance: total}}, + Available: true, + StatusCode: http.StatusOK, + FetchedAt: now.Unix(), + } +} + +// clearCommandCodeWindowCooldown 在探测到充值积分时解除窗口用满的临时停调:充值积分 +// 不受滚动窗口限制,上游已允许继续请求。只清除 commandCodeUsageLimitReason 前缀的停调, +// 积分不足、花费上限、管理员规则等其他原因的停调不受影响。 +func clearCommandCodeWindowCooldown(ctx context.Context, repo AccountRepository, account *Account, usage *commandCodeUsage, now time.Time) bool { + if usage == nil || usage.credits.Credits == nil || usage.credits.Credits.PurchasedCredits <= 0 { + return false + } + if account.TempUnschedulableUntil == nil || !now.Before(*account.TempUnschedulableUntil) || + !strings.HasPrefix(account.TempUnschedulableReason, commandCodeUsageLimitReason) { + return false + } + if err := repo.ClearTempUnschedulable(ctx, account.ID); err != nil { + slog.Warn("command_code_usage_limit_clear_failed", "account_id", account.ID, "error", err) + return false + } + slog.Info("command_code_usage_limit_cleared", "account_id", account.ID, + "purchased_credits", float64(usage.credits.Credits.PurchasedCredits)) + return true +} + +// commandCodeWindowsBinding 报告账号快照中的滚动 / 月度窗口是否限制调度: +// 有充值积分时上游跳过窗口检查,窗口用满也不会被拒绝。 +func commandCodeWindowsBinding(extra map[string]any) bool { + purchased, ok := cnParseF64(extra[cnExtraKey(PlatformCommandCode, commandCodeExtraSuffixPurchasedCredits)]) + return !ok || purchased <= 0 +} + +// commandCodeUsageExtraUpdates 构造窗口 + 余额快照。缺席的窗口写 nil,避免套餐 +// 变化后沿用旧快照;窗口尚未开始时没有重置时间,同样清掉上一个窗口的值;月度窗口 +// 取数失败时保留旧快照;传 nil tiers 时只写余额。 +func commandCodeUsageExtraUpdates(usage *commandCodeUsage, tiers []CNQuotaTier, balance *CNProviderBalanceResult, now time.Time, withTiers bool) map[string]any { + updates := cnBalanceExtraUpdates(PlatformCommandCode, balance, now) + updates[cnExtraKey(PlatformCommandCode, commandCodeExtraSuffixPurchasedCredits)] = float64(usage.credits.Credits.PurchasedCredits) + if !withTiers { + return updates + } + for key, value := range cnQuotaExtraUpdates(PlatformCommandCode, tiers, now) { + updates[key] = value + } + present := make(map[string]CNQuotaTier, len(tiers)) + for _, tier := range tiers { + present[tier.Window] = tier + } + for _, window := range []struct { + name, used, reset string + }{ + {"5h", cnExtraSuffix5hUsed, cnExtraSuffix5hReset}, + {"weekly", cnExtraSuffixWeeklyUsed, cnExtraSuffixWeeklyReset}, + {"monthly", cnExtraSuffixMonthlyUsed, cnExtraSuffixMonthlyReset}, + } { + tier, ok := present[window.name] + if !ok && window.name == "monthly" && usage.periodFetchFailed { + continue + } + if !ok { + updates[cnExtraKey(PlatformCommandCode, window.used)] = nil + } + if tier.ResetAt == "" { + updates[cnExtraKey(PlatformCommandCode, window.reset)] = nil + } + } + return updates +} + +func newCommandCodeUsageClient(cfg *config.Config, httpUpstream HTTPUpstream, account *Account, proxyURL string) *commandCodeUsageClient { + return &commandCodeUsageClient{ + cfg: cfg, + httpUpstream: httpUpstream, + account: account, + apiKey: strings.TrimSpace(account.GetCNAPIKey()), + proxyURL: proxyURL, + } +} + +// commandCodeFetchFailure 把取数错误转换为探测失败文案;ok=false 表示应作为错误返回。 +func commandCodeFetchFailure(err error) (status int, message string, ok bool) { + var httpErr *commandCodeHTTPError + switch { + case errors.As(err, &httpErr) && httpErr.unauthorized(): + return httpErr.status, fmt.Sprintf("Authentication failed (HTTP %d)", httpErr.status), true + case errors.As(err, &httpErr): + return httpErr.status, fmt.Sprintf("API error (HTTP %d): %s", httpErr.status, httpErr.body), true + case errors.Is(err, errCommandCodeInvalidResponse): + return http.StatusOK, "Invalid usage response", true + default: + return 0, "", false + } +} + +// queryCommandCodeUsage 查询 Command Code 积分与窗口,落窗口与余额快照。 +func (s *CNProviderQuotaService) queryCommandCodeUsage(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { + client := newCommandCodeUsageClient(s.cfg, s.httpUpstream, account, s.resolveProxyURL(ctx, account)) + if client.apiKey == "" { + return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NO_APIKEY", "account api_key is empty") + } + now := time.Now().UTC() + result := &CNProviderQuotaProbeResult{ + Provider: PlatformCommandCode, + Source: "command_code_usage", + FetchedAt: now.Unix(), + } + usage, err := client.fetch(ctx, true) + if err != nil { + status, message, ok := commandCodeFetchFailure(err) + if !ok { + if isApplicationError(err) { + return nil, err + } + return nil, infraerrors.Newf(http.StatusBadGateway, "CN_QUOTA_REQUEST_FAILED", "%v", err) + } + result.StatusCode = status + result.Error = message + return result, nil + } + + tiers := usage.tiers() + balance := usage.balance(now) + result.StatusCode = http.StatusOK + result.Tiers = tiers + result.PlanLevel = usage.planName() + result.Balance = balance + result.Success = true + result.CredentialValid = true + + updates := commandCodeUsageExtraUpdates(usage, tiers, balance, now, true) + if err := s.accountRepo.UpdateExtra(ctx, account.ID, updates); err != nil { + slog.Warn("cn_quota_persist_failed", "account_id", account.ID, "provider", PlatformCommandCode, "error", err) + } else { + result.Persisted = true + balance.Persisted = true + } + clearCommandCodeWindowCooldown(ctx, s.accountRepo, account, usage, now) + return result, nil +} + +// queryCommandCodeBalance 只查积分余额(whoami + credits),落余额快照。 +func (s *CNProviderBalanceService) queryCommandCodeBalance(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { + client := newCommandCodeUsageClient(s.cfg, s.httpUpstream, account, s.resolveProxyURL(ctx, account)) + if client.apiKey == "" { + return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_APIKEY", "account api_key is empty") + } + now := time.Now().UTC() + usage, err := client.fetch(ctx, false) + if err != nil { + status, message, ok := commandCodeFetchFailure(err) + if !ok { + if isApplicationError(err) { + return nil, err + } + return nil, infraerrors.Newf(http.StatusBadGateway, "CN_BALANCE_REQUEST_FAILED", "%v", err) + } + return &CNProviderBalanceResult{ + Provider: PlatformCommandCode, + StatusCode: status, + FetchedAt: now.Unix(), + Available: true, + Error: message, + }, nil + } + result := usage.balance(now) + if err := s.accountRepo.UpdateExtra(ctx, account.ID, commandCodeUsageExtraUpdates(usage, nil, result, now, false)); err != nil { + slog.Warn("cn_balance_persist_failed", "account_id", account.ID, "provider", PlatformCommandCode, "error", err) + } else { + result.Persisted = true + } + clearCommandCodeWindowCooldown(ctx, s.accountRepo, account, usage, now) + return result, nil +} diff --git a/backend/internal/service/command_code_usage_test.go b/backend/internal/service/command_code_usage_test.go new file mode 100644 index 000000000000..fc720111a117 --- /dev/null +++ b/backend/internal/service/command_code_usage_test.go @@ -0,0 +1,694 @@ +//go:build unit + +package service + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/stretchr/testify/require" +) + +// commandCodeAlphaUpstream 按路径返回 /alpha 接口响应,并记录请求。 +type commandCodeAlphaUpstream struct { + mu sync.Mutex + responses map[string]commandCodeAlphaResponse + requests []*http.Request +} + +type commandCodeAlphaResponse struct { + status int + body string +} + +func (u *commandCodeAlphaUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + u.mu.Lock() + u.requests = append(u.requests, req) + u.mu.Unlock() + resp, ok := u.responses[req.URL.Path] + if !ok { + resp = commandCodeAlphaResponse{status: http.StatusNotFound, body: `{"error":"not found"}`} + } + return &http.Response{ + StatusCode: resp.status, + Body: io.NopCloser(strings.NewReader(resp.body)), + Header: make(http.Header), + Request: req, + }, nil +} + +func (u *commandCodeAlphaUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, concurrency int, _ *tlsfingerprint.Profile) (*http.Response, error) { + return u.Do(req, proxyURL, accountID, concurrency) +} + +func (u *commandCodeAlphaUpstream) request(path string) *http.Request { + for _, req := range u.requests { + if req.URL.Path == path { + return req + } + } + return nil +} + +const ( + commandCodeTestCredits = `{ + "credits": {"belowThreshold": false, "monthlyCredits": "34.9", "purchasedCredits": 0, "freeCredits": 0.1}, + "windowLimits": { + "limited": true, + "fiveHour": {"used": 1.5, "cap": 14, "resetAt": 1790000000000}, + "weekly": {"used": 35, "cap": 35, "resetAt": 1790400000000} + } + }` + commandCodeTestSubscription = `{"data": {"planId": "individual-goat", "status": "active", + "currentPeriodStart": "2026-09-01T00:00:00Z", "currentPeriodEnd": "2026-10-01T00:00:00Z"}}` + commandCodeTestSummary = `{"totalCost": 40, "totalMonthlyCredits": 35.1}` +) + +func commandCodeUsageAccount() *Account { + account := commandCodeTestAccount(77) + account.Concurrency = 1 + return account +} + +func newCommandCodeAlphaUpstream(whoami string) *commandCodeAlphaUpstream { + return &commandCodeAlphaUpstream{responses: map[string]commandCodeAlphaResponse{ + "/alpha/whoami": {status: http.StatusOK, body: whoami}, + "/alpha/billing/credits": {status: http.StatusOK, body: commandCodeTestCredits}, + "/alpha/billing/subscriptions": {status: http.StatusOK, body: commandCodeTestSubscription}, + "/alpha/usage/summary": {status: http.StatusOK, body: commandCodeTestSummary}, + }} +} + +func TestCommandCodeQuotaProbeParsesWindowsAndBalance(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"user": {"userName": "alice"}, "org": null}`) + repo := &cnBalanceProbeRepo{account: account} + svc := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}) + + result, err := svc.QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, result.Success) + require.True(t, result.Persisted) + require.Equal(t, PlatformCommandCode, result.Provider) + require.Equal(t, "GOAT", result.PlanLevel) + require.Equal(t, []CNQuotaTier{ + {Window: "5h", UsedPercent: 10.71, ResetAt: time.UnixMilli(1790000000000).UTC().Format(time.RFC3339)}, + {Window: "weekly", UsedPercent: 100, ResetAt: time.UnixMilli(1790400000000).UTC().Format(time.RFC3339)}, + {Window: "monthly", UsedPercent: 50.14, ResetAt: "2026-10-01T00:00:00Z"}, + }, result.Tiers) + require.NotNil(t, result.Balance) + require.InDelta(t, 35.0, result.Balance.Balance, 1e-9) + require.Equal(t, "USD", result.Balance.Currency) + + // 个人账户不带 orgId;本周期用量从订阅周期起点统计。 + for _, req := range upstream.requests { + require.Equal(t, "Bearer user_test_key", req.Header.Get("Authorization")) + require.Equal(t, "api.commandcode.ai", req.URL.Host) + require.False(t, req.URL.Query().Has("orgId"), req.URL.String()) + } + require.Equal(t, "2026-09-01T00:00:00Z", upstream.request("/alpha/usage/summary").URL.Query().Get("since")) + + require.Len(t, repo.extraWrites, 1) + extra := repo.extraWrites[0] + require.Equal(t, 10.71, extra["command_code_5h_used_percent"]) + require.Equal(t, 100.0, extra["command_code_weekly_used_percent"]) + require.Equal(t, 50.14, extra["command_code_monthly_used_percent"]) + require.Equal(t, "2026-10-01T00:00:00Z", extra["command_code_monthly_reset_at"]) + require.InDelta(t, 35.0, extra["command_code_balance"], 1e-9) + require.Equal(t, "USD", extra["command_code_balance_currency"]) + require.Equal(t, 0.0, extra["command_code_purchased_credits"]) + require.Equal(t, false, extra["command_code_balance_low"]) +} + +func TestCommandCodeQuotaProbePassesOrgID(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"user": {"userName": "alice"}, "org": {"id": "9a0f5d0e-6d2c-4f36-9c55-0b8b0c0d1e2f", "login": "team"}}`) + svc := NewCNProviderQuotaService(&cnBalanceProbeRepo{account: account}, nil, upstream, &config.Config{}) + + _, err := svc.QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + for _, path := range []string{"/alpha/billing/credits", "/alpha/billing/subscriptions", "/alpha/usage/summary"} { + require.Equal(t, "9a0f5d0e-6d2c-4f36-9c55-0b8b0c0d1e2f", upstream.request(path).URL.Query().Get("orgId"), path) + } +} + +func TestCommandCodeQuotaProbeUnlimitedPlanClearsRollingWindows(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + upstream.responses["/alpha/billing/credits"] = commandCodeAlphaResponse{status: http.StatusOK, body: `{ + "credits": {"monthlyCredits": 10, "purchasedCredits": 25, "freeCredits": 0}, + "windowLimits": {"limited": false} + }`} + upstream.responses["/alpha/usage/summary"] = commandCodeAlphaResponse{status: http.StatusOK, body: `{"totalCost": 5}`} + repo := &cnBalanceProbeRepo{account: account} + svc := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}) + + result, err := svc.QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.Equal(t, []CNQuotaTier{{Window: "monthly", UsedPercent: 33.33, ResetAt: "2026-10-01T00:00:00Z"}}, result.Tiers) + extra := repo.extraWrites[0] + for _, key := range []string{"command_code_5h_used_percent", "command_code_5h_reset_at", "command_code_weekly_used_percent", "command_code_weekly_reset_at"} { + value, ok := extra[key] + require.True(t, ok, key) + require.Nil(t, value, key) + } + require.Equal(t, 25.0, extra["command_code_purchased_credits"]) + require.InDelta(t, 35.0, extra["command_code_balance"], 1e-9) +} + +// 窗口尚未开始(resetAt 为空)时清掉上一个窗口的重置时间,不与新的用量配对。 +func TestCommandCodeQuotaProbeClearsResetOfUnstartedWindow(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + upstream.responses["/alpha/billing/credits"] = commandCodeAlphaResponse{status: http.StatusOK, body: `{ + "credits": {"monthlyCredits": 34.9, "purchasedCredits": 0, "freeCredits": 0}, + "windowLimits": { + "limited": true, + "fiveHour": {"used": 0, "cap": 14, "resetAt": null}, + "weekly": {"used": 35, "cap": 35, "resetAt": 1790400000000} + } + }`} + repo := &cnBalanceProbeRepo{account: account} + + _, err := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + extra := repo.extraWrites[0] + require.Equal(t, 0.0, extra["command_code_5h_used_percent"]) + value, written := extra["command_code_5h_reset_at"] + require.True(t, written) + require.Nil(t, value) + require.Equal(t, time.UnixMilli(1790400000000).UTC().Format(time.RFC3339), extra["command_code_weekly_reset_at"]) + require.Equal(t, "2026-10-01T00:00:00Z", extra["command_code_monthly_reset_at"]) +} + +func TestCommandCodeQuotaProbeAuthFailureDoesNotPersist(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{}`) + upstream.responses["/alpha/whoami"] = commandCodeAlphaResponse{status: http.StatusUnauthorized, body: `{"error":"unauthorized"}`} + repo := &cnBalanceProbeRepo{account: account} + svc := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}) + + result, err := svc.QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.False(t, result.Success) + require.Equal(t, http.StatusUnauthorized, result.StatusCode) + require.Equal(t, "Authentication failed (HTTP 401)", result.Error) + require.Empty(t, repo.extraWrites) + require.Len(t, upstream.requests, 1) +} + +// 订阅周期或本周期用量取数失败:滚动窗口照常更新,月度快照保留上一次的值(无从判断 +// 月度窗口是否还在);订阅确实没有周期时才清掉月度快照。 +// whoami 失败(非鉴权错误)时无从判断是否组织账户:探测整体失败,不按个人账户去查 +// credits,也不写快照。 +func TestCommandCodeProbeFailsWhenWhoamiFails(t *testing.T) { + for _, whoami := range []commandCodeAlphaResponse{ + {status: http.StatusBadGateway, body: `bad gateway`}, + {status: http.StatusOK, body: `not json`}, + } { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{}`) + upstream.responses["/alpha/whoami"] = whoami + repo := &cnBalanceProbeRepo{account: account} + + result, err := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}).QueryUsage(context.Background(), account.ID) + require.NoError(t, err, whoami.body) + require.False(t, result.Success, whoami.body) + require.NotEmpty(t, result.Error, whoami.body) + + balance, err := NewCNProviderBalanceService(repo, nil, upstream, &config.Config{}).QueryBalance(context.Background(), account.ID) + require.NoError(t, err, whoami.body) + require.False(t, balance.Success, whoami.body) + + require.Nil(t, upstream.request("/alpha/billing/credits"), whoami.body) + require.Empty(t, repo.extraWrites, whoami.body) + } + + // 请求本身失败(网络错误)同样整体失败。 + account := commandCodeUsageAccount() + repo := &cnBalanceProbeRepo{account: account} + _, err := NewCNProviderQuotaService(repo, nil, failingCommandCodeUpstream{}, &config.Config{}).QueryUsage(context.Background(), account.ID) + require.Error(t, err) + require.Empty(t, repo.extraWrites) +} + +// failingCommandCodeUpstream 让每个请求都在网络层失败。 +type failingCommandCodeUpstream struct{} + +func (failingCommandCodeUpstream) Do(*http.Request, string, int64, int) (*http.Response, error) { + return nil, errors.New("dial tcp: i/o timeout") +} + +func (u failingCommandCodeUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, concurrency int, _ *tlsfingerprint.Profile) (*http.Response, error) { + return u.Do(req, proxyURL, accountID, concurrency) +} + +func TestCommandCodeQuotaProbeKeepsWindowsWhenPeriodEndpointsFail(t *testing.T) { + for _, tc := range []struct { + name string + path string + response commandCodeAlphaResponse + planLevel string + keepMonthly bool + }{ + {"subscriptions failed", "/alpha/billing/subscriptions", commandCodeAlphaResponse{status: http.StatusInternalServerError, body: `oops`}, "", true}, + {"summary failed", "/alpha/usage/summary", commandCodeAlphaResponse{status: http.StatusBadGateway, body: `oops`}, "GOAT", true}, + {"no subscription period", "/alpha/billing/subscriptions", commandCodeAlphaResponse{status: http.StatusOK, body: `{"data":{}}`}, "", false}, + } { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + upstream.responses[tc.path] = tc.response + repo := &cnBalanceProbeRepo{account: account} + + result, err := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err, tc.name) + require.True(t, result.Success, tc.name) + require.Len(t, result.Tiers, 2, tc.name) + require.Equal(t, tc.planLevel, result.PlanLevel, tc.name) + if tc.path == "/alpha/billing/subscriptions" { + require.Nil(t, upstream.request("/alpha/usage/summary"), tc.name) + } + extra := repo.extraWrites[0] + require.Equal(t, 10.71, extra["command_code_5h_used_percent"], tc.name) + for _, key := range []string{"command_code_monthly_used_percent", "command_code_monthly_reset_at"} { + value, written := extra[key] + require.Equal(t, !tc.keepMonthly, written, tc.name+": "+key) + require.Nil(t, value, tc.name+": "+key) + } + } +} + +func TestCommandCodeUsageRequiresOfficialHost(t *testing.T) { + account := commandCodeUsageAccount() + require.Equal(t, PlatformCommandCode, account.GetCodingPlanProvider()) + require.NoError(t, validateCodingPlanAccount(account)) + require.NoError(t, validatePayGAccount(account)) + + account.Credentials["base_url"] = "https://relay.example.com/v1" + account.Credentials["api_protocol"] = APIProtocolChatCompletions + require.Empty(t, account.GetCodingPlanProvider()) + require.Error(t, validateCodingPlanAccount(account)) + require.Error(t, validatePayGAccount(account)) +} + +func TestCommandCodeBalanceProbeFetchesCreditsOnly(t *testing.T) { + account := commandCodeUsageAccount() + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + repo := &cnBalanceProbeRepo{account: account} + svc := NewCNProviderBalanceService(repo, nil, upstream, &config.Config{}) + + result, err := svc.QueryBalance(context.Background(), account.ID) + + require.NoError(t, err) + require.True(t, result.Success) + require.True(t, result.Persisted) + require.InDelta(t, 35.0, result.Balance, 1e-9) + require.Len(t, upstream.requests, 2) + require.Len(t, repo.extraWrites, 1) + _, hasWindow := repo.extraWrites[0]["command_code_5h_used_percent"] + require.False(t, hasWindow) + require.InDelta(t, 35.0, repo.extraWrites[0]["command_code_balance"], 1e-9) +} + +func TestCommandCodeWindowsIgnoredWithPurchasedCredits(t *testing.T) { + future := time.Now().Add(2 * time.Hour).UTC().Format(time.RFC3339) + account := commandCodeUsageAccount() + account.Extra = map[string]any{ + "command_code_5h_used_percent": 100.0, + "command_code_5h_reset_at": future, + "command_code_purchased_credits": 0.0, + } + require.NotEmpty(t, cnProviderThresholdCandidates(account, PlatformCommandCode)) + require.NotNil(t, cnProviderQuotaSnapshotReset(account, time.Now())) + + account.Extra["command_code_purchased_credits"] = 12.5 + require.Empty(t, cnProviderThresholdCandidates(account, PlatformCommandCode)) + require.Nil(t, cnProviderQuotaSnapshotReset(account, time.Now())) +} + +// commandCodeCheckRepo 按平台列出账号,并记录周期检测对账号的停调。 +type commandCodeCheckRepo struct { + AccountRepository + accounts []Account + mu sync.Mutex + paused []int64 +} + +func (r *commandCodeCheckRepo) ListByPlatform(_ context.Context, platform string) ([]Account, error) { + var accounts []Account + for _, account := range r.accounts { + if account.Platform == platform { + accounts = append(accounts, account) + } + } + return accounts, nil +} + +func (r *commandCodeCheckRepo) SetTempUnschedulable(_ context.Context, id int64, _ time.Time, _ string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.paused = append(r.paused, id) + return nil +} + +type commandCodeBalanceProber struct { + mu sync.Mutex + probed []int64 + balance float64 +} + +func (p *commandCodeBalanceProber) QueryUsage(_ context.Context, accountID int64) (*CNProviderQuotaProbeResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.probed = append(p.probed, accountID) + return &CNProviderQuotaProbeResult{ + Success: true, + Balance: &CNProviderBalanceResult{Success: true, Balance: p.balance, Currency: "USD", Available: true}, + }, nil +} + +func TestCNProviderBalanceCheckRunOnceProbesCommandCode(t *testing.T) { + official := *commandCodeUsageAccount() + official.ID, official.Schedulable = 1, true + manualOff := *commandCodeUsageAccount() + manualOff.ID, manualOff.Schedulable = 2, false + relay := *commandCodeTestAccount(3) + relay.Schedulable = true + relay.Credentials = map[string]any{"api_key": "k", "api_protocol": APIProtocolChatCompletions, "base_url": "https://relay.example.com/v1"} + inactive := *commandCodeUsageAccount() + inactive.ID, inactive.Status = 4, StatusDisabled + + repo := &commandCodeCheckRepo{accounts: []Account{official, manualOff, relay, inactive}} + prober := &commandCodeBalanceProber{balance: 0.2} + cfg := &config.Config{} + cfg.Gateway.CNProviders.BalanceThreshold = 0.5 + svc := &CNProviderBalanceCheckService{accountRepo: repo, quotaService: prober, cfg: cfg} + + svc.runOnce() + + require.ElementsMatch(t, []int64{1, 2}, prober.probed) + require.Equal(t, []int64{1}, repo.paused) +} + +// 积分不足(402)是可恢复状态:临时停调并标记 balance_low,不能永久置 error。 +func TestHandleUpstreamError_CommandCode402PausesInsteadOfError(t *testing.T) { + repo := &rateLimitAccountRepoStub{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + + shouldDisable := service.HandleUpstreamError(context.Background(), account, http.StatusPaymentRequired, + http.Header{}, []byte(`{"error":{"message":"Insufficient credits"}}`)) + + require.True(t, shouldDisable) + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 1, repo.tempCalls) + require.Contains(t, repo.lastTempReason, cnBalanceLowReasonPrefix) + require.Equal(t, true, repo.lastExtraUpdates["command_code_balance_low"]) +} + +func TestHandleUpstreamError_CommandCode429InsufficientBalancePauses(t *testing.T) { + repo := &rateLimitAccountRepoStub{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + + service.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, + http.Header{}, []byte(`{"error":{"type":"insufficient_credits","message":"no credits left"}}`)) + + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 1, repo.tempCalls) + require.Contains(t, repo.lastTempReason, cnBalanceLowReasonPrefix) + require.Equal(t, 0, repo.rateLimitedCalls) +} + +// commandCodeRateLimitRepo 记录 Command Code 响应式处理写入的临时停调与模型级限流。 +type commandCodeRateLimitRepo struct { + rateLimitAccountRepoStub + tempUntil time.Time + modelLimits map[string]time.Time + reasons map[string]string +} + +func (r *commandCodeRateLimitRepo) SetTempUnschedulable(ctx context.Context, id int64, until time.Time, reason string) error { + r.tempUntil = until + return r.rateLimitAccountRepoStub.SetTempUnschedulable(ctx, id, until, reason) +} + +func (r *commandCodeRateLimitRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, resetAt time.Time, reason ...string) error { + if r.modelLimits == nil { + r.modelLimits = map[string]time.Time{} + r.reasons = map[string]string{} + } + r.modelLimits[scope] = resetAt + if len(reason) > 0 { + r.reasons[scope] = reason[0] + } + return nil +} + +// 窗口用满:按文案中的重置时间写带原因的临时停调(充值后可据原因解除),不依赖额度快照。 +func TestHandleUpstreamError_CommandCodeWindowLimitCoolsDownToMessageReset(t *testing.T) { + for _, status := range []int{http.StatusTooManyRequests, http.StatusForbidden, http.StatusPaymentRequired} { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + + shouldDisable := service.HandleUpstreamError(context.Background(), account, status, http.Header{}, + []byte(`{"error":{"type":"usage_exceeded","message":"You've reached your 5-hour usage limit. Resets in 2h 41m (3:00 PM)."}}`)) + + require.Equal(t, status != http.StatusTooManyRequests, shouldDisable, status) + require.Equal(t, 0, repo.setErrorCalls, status) + require.Equal(t, 0, repo.rateLimitedCalls, status) + require.Equal(t, 1, repo.tempCalls, status) + require.True(t, strings.HasPrefix(repo.lastTempReason, commandCodeUsageLimitReason+": "), repo.lastTempReason) + expected := time.Now().Add(2*time.Hour + 41*time.Minute) + require.WithinDuration(t, expected, repo.tempUntil, time.Minute, status) + } +} + +// 文案没有重置时间时回落额度快照中最早的窗口重置点。 +func TestHandleUpstreamError_CommandCodeWindowLimitFallsBackToSnapshot(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + account.Extra = map[string]any{ + "command_code_5h_used_percent": 100.0, + "command_code_5h_reset_at": time.Now().Add(90 * time.Minute).UTC().Format(time.RFC3339), + "command_code_purchased_credits": 0.0, + } + + service.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, + []byte(`{"error":{"message":"You've reached your weekly usage limit."}}`)) + + require.Equal(t, 1, repo.tempCalls) + require.WithinDuration(t, time.Now().Add(90*time.Minute), repo.tempUntil, time.Minute) +} + +// 普通频率限制很快恢复:即使快照里有窗口重置点,也不冷却到窗口重置。 +func TestHandleUpstreamError_CommandCodeRateLimitUsesDefaultCooldown(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + account.Extra = map[string]any{ + "command_code_5h_used_percent": 97.0, + "command_code_5h_reset_at": time.Now().Add(4 * time.Hour).UTC().Format(time.RFC3339), + "command_code_purchased_credits": 0.0, + } + + service.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, + []byte(`{"error":{"type":"rate_limit","message":"Rate limit exceeded. Please wait a moment and try again."}}`)) + + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 0, repo.tempCalls) + if repo.rateLimitedCalls > 0 { + require.True(t, repo.lastRateLimitedAt.Before(time.Now().Add(30*time.Minute)), "must not cool down to the 5h window reset") + } +} + +const ( + commandCodeSpendLimitBody = `{"error":{"type":"usage_exceeded","message":"You've reached the $10.00/mo spending limit your organization set for GLM-5. It resets in 3 days."}}` + commandCodeMemberSpendLimitBody = `{"error":{"type":"usage_exceeded","message":"You've reached the $50.00/mo spending limit your organization set. It resets in 12 days."}}` +) + +// 组织为某个模型设的花费上限只限当前模型:写模型级限流到文案中的重置时间,不停整个账号。 +func TestHandleUpstreamError_CommandCodeSpendLimitIsModelScoped(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + ctx := withTempUnschedulableModel(context.Background(), []string{"glm-5"}) + + shouldDisable := service.HandleUpstreamError(ctx, account, http.StatusForbidden, http.Header{}, []byte(commandCodeSpendLimitBody)) + + require.False(t, shouldDisable) + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 0, repo.tempCalls) + require.Equal(t, 0, repo.rateLimitedCalls) + require.Contains(t, repo.modelLimits, "glm-5") + require.WithinDuration(t, time.Now().Add(72*time.Hour), repo.modelLimits["glm-5"], time.Minute) + require.Equal(t, commandCodeSpendLimitReason, repo.reasons["glm-5"]) +} + +// 成员总额度耗尽:即使请求模型已知也按账号冷却,但最长一小时(管理员可能调高上限)。 +func TestHandleUpstreamError_CommandCodeMemberSpendLimitIsAccountScoped(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + ctx := withTempUnschedulableModel(context.Background(), []string{"glm-5"}) + + shouldDisable := service.HandleUpstreamError(ctx, account, http.StatusForbidden, http.Header{}, []byte(commandCodeMemberSpendLimitBody)) + + require.True(t, shouldDisable) + require.Empty(t, repo.modelLimits) + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 1, repo.tempCalls) + require.Equal(t, commandCodeSpendLimitReason, repo.lastTempReason) + require.WithinDuration(t, time.Now().Add(commandCodeAccountSpendLimitRecheck), repo.tempUntil, time.Minute) + + // 文案中的重置时间早于一小时时以文案为准。 + soon := &commandCodeRateLimitRepo{} + NewRateLimitService(soon, nil, &config.Config{}, nil, nil).HandleUpstreamError(ctx, account, http.StatusTooManyRequests, http.Header{}, + []byte(`{"error":{"message":"You've reached the $50.00/mo spending limit your organization set. It resets in 20 minutes."}}`)) + require.WithinDuration(t, time.Now().Add(20*time.Minute), soon.tempUntil, time.Minute) +} + +// 不知道模型时按账号冷却(最长一小时),不停账号数天,也不进入 403 累计禁用。 +func TestHandleUpstreamError_CommandCodeSpendLimitWithoutModelShortCooldown(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := commandCodeUsageAccount() + + for i := 0; i < 5; i++ { + service.HandleUpstreamError(context.Background(), account, http.StatusForbidden, http.Header{}, []byte(commandCodeSpendLimitBody)) + } + + require.Equal(t, 0, repo.setErrorCalls) + require.Empty(t, repo.modelLimits) + require.Equal(t, 5, repo.tempCalls) + require.False(t, repo.tempUntil.After(time.Now().Add(commandCodeAccountSpendLimitRecheck))) +} + +func TestCommandCodeSpendLimitScope(t *testing.T) { + require.True(t, commandCodeSpendLimitIsModelScoped([]byte(commandCodeSpendLimitBody))) + require.True(t, commandCodeSpendLimitIsModelScoped([]byte(`spending limit your organization set for gpt-5.5. It resets in 2 days.`))) + require.False(t, commandCodeSpendLimitIsModelScoped([]byte(commandCodeMemberSpendLimitBody))) + require.False(t, commandCodeSpendLimitIsModelScoped([]byte(`You've reached the $50.00/mo spending limit your organization set for you.`))) + require.False(t, commandCodeSpendLimitIsModelScoped([]byte(`spending limit reached`))) +} + +// commandCodeStatefulRepo 把停调写回同一个账号对象,用于验证完整的状态转换。 +type commandCodeStatefulRepo struct { + AccountRepository + account *Account + clearCalls int +} + +func (r *commandCodeStatefulRepo) GetByID(_ context.Context, _ int64) (*Account, error) { + clone := *r.account + return &clone, nil +} + +func (r *commandCodeStatefulRepo) UpdateExtra(_ context.Context, _ int64, _ map[string]any) error { + return nil +} + +func (r *commandCodeStatefulRepo) SetTempUnschedulable(_ context.Context, _ int64, until time.Time, reason string) error { + r.account.TempUnschedulableUntil = &until + r.account.TempUnschedulableReason = reason + return nil +} + +func (r *commandCodeStatefulRepo) ClearTempUnschedulable(_ context.Context, _ int64) error { + r.clearCalls++ + r.account.TempUnschedulableUntil = nil + r.account.TempUnschedulableReason = "" + return nil +} + +func commandCodeCreditsBody(purchased float64) string { + return fmt.Sprintf(`{"credits":{"monthlyCredits":0,"purchasedCredits":%g,"freeCredits":0}, + "windowLimits":{"limited":true,"fiveHour":{"used":14,"cap":14,"resetAt":1790000000000}}}`, purchased) +} + +// 窗口用满进入冷却 → 充值 → 下一次额度探测发现充值积分 → 恢复可调度。 +func TestCommandCodeWindowCooldownClearedAfterTopUp(t *testing.T) { + account := commandCodeUsageAccount() + account.Schedulable = true + repo := &commandCodeStatefulRepo{account: account} + rateLimit := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + + rateLimit.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, + []byte(`{"error":{"message":"You've reached your 5-hour usage limit. Resets in 3h 10m."}}`)) + require.False(t, account.IsSchedulable()) + + // 未充值:探测不解除冷却。 + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + upstream.responses["/alpha/billing/credits"] = commandCodeAlphaResponse{status: http.StatusOK, body: commandCodeCreditsBody(0)} + quota := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}) + _, err := quota.QueryUsage(context.Background(), account.ID) + require.NoError(t, err) + require.Equal(t, 0, repo.clearCalls) + require.False(t, account.IsSchedulable()) + + // 充值后:额度探测与余额探测都能解除窗口冷却。 + upstream.responses["/alpha/billing/credits"] = commandCodeAlphaResponse{status: http.StatusOK, body: commandCodeCreditsBody(20)} + _, err = quota.QueryUsage(context.Background(), account.ID) + require.NoError(t, err) + require.Equal(t, 1, repo.clearCalls) + require.True(t, account.IsSchedulable()) + + rateLimit.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, + []byte(`{"error":{"message":"You've reached your weekly usage limit. Resets in 2 days."}}`)) + require.False(t, account.IsSchedulable()) + _, err = NewCNProviderBalanceService(repo, nil, upstream, &config.Config{}).QueryBalance(context.Background(), account.ID) + require.NoError(t, err) + require.True(t, account.IsSchedulable()) +} + +// 充值积分只解除窗口冷却:积分不足、花费上限等其他原因的停调保持不变。 +func TestCommandCodeTopUpKeepsOtherCooldowns(t *testing.T) { + for _, reason := range []string{cnBalanceLowReason("no credits"), commandCodeSpendLimitReason, "manual rule"} { + until := time.Now().Add(time.Hour) + account := commandCodeUsageAccount() + account.Schedulable = true + account.TempUnschedulableUntil = &until + account.TempUnschedulableReason = reason + repo := &commandCodeStatefulRepo{account: account} + upstream := newCommandCodeAlphaUpstream(`{"org": null}`) + upstream.responses["/alpha/billing/credits"] = commandCodeAlphaResponse{status: http.StatusOK, body: commandCodeCreditsBody(20)} + + _, err := NewCNProviderQuotaService(repo, nil, upstream, &config.Config{}).QueryUsage(context.Background(), account.ID) + + require.NoError(t, err) + require.Equal(t, 0, repo.clearCalls, reason) + require.False(t, account.IsSchedulable(), reason) + } +} + +func TestClassifyCommandCodeUsageError(t *testing.T) { + cases := map[string]commandCodeUsageErrorKind{ + `{"error":{"message":"You've reached your 5-hour usage limit. Resets in 2h 41m (3:00 PM)."}}`: commandCodeUsageWindowLimit, + `{"error":{"message":"You've reached your weekly usage limit."}}`: commandCodeUsageWindowLimit, + commandCodeSpendLimitBody: commandCodeUsageSpendLimit, + `{"error":{"type":"usage_exceeded","message":"Monthly credits spent."}}`: commandCodeUsageCreditsExhausted, + `{"error":{"message":"Insufficient credits"}}`: commandCodeUsageCreditsExhausted, + `{"error":{"type":"rate_limit","message":"Rate limit exceeded. Please wait a moment."}}`: commandCodeUsageErrorNone, + `{"error":{"message":"forbidden"}}`: commandCodeUsageErrorNone, + } + for body, want := range cases { + require.Equal(t, want, classifyCommandCodeUsageError([]byte(body)), body) + } +} diff --git a/backend/internal/service/composite_platform.go b/backend/internal/service/composite_platform.go index c10e335cbebe..b7fcfd3451ae 100644 --- a/backend/internal/service/composite_platform.go +++ b/backend/internal/service/composite_platform.go @@ -4,6 +4,7 @@ import ( "context" "strings" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" ) @@ -204,11 +205,5 @@ func (s *GatewayService) resolveCompositeRouteDecision(ctx context.Context, grou } func isConcreteRequestPlatform(platform string) bool { - switch platform { - case PlatformAnthropic, PlatformOpenAI, PlatformGemini, PlatformAntigravity, PlatformGrok, - PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe: - return true - default: - return false - } + return domain.IsConcretePlatform(platform) } diff --git a/backend/internal/service/composite_platform_test.go b/backend/internal/service/composite_platform_test.go index 28e6231823e2..7f253b84f059 100644 --- a/backend/internal/service/composite_platform_test.go +++ b/backend/internal/service/composite_platform_test.go @@ -224,13 +224,13 @@ func TestCompositeGroupSchedulerHasAllCanonicalPlatformBuckets(t *testing.T) { platforms = append(platforms, platform) } require.ElementsMatch(t, - []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe}, + []string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, PlatformCommandCode, PlatformCline}, platforms, ) } func TestCompositeConcretePlatformsIncludeCNProviders(t *testing.T) { - for _, platform := range []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe} { + for _, platform := range []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, PlatformCommandCode, PlatformCline} { require.True(t, isConcreteRequestPlatform(platform)) require.True(t, canCopyAccountsFromGroupPlatform(PlatformComposite, platform)) } diff --git a/backend/internal/service/composite_route_resolver.go b/backend/internal/service/composite_route_resolver.go index f2e1928ace34..5d73de7de09f 100644 --- a/backend/internal/service/composite_route_resolver.go +++ b/backend/internal/service/composite_route_resolver.go @@ -22,6 +22,52 @@ func (r *CompositeRouteResolver) SetModelOwnershipResolver(resolver CompositeMod } } +// ListExactPublicModels returns enabled, concrete route IDs suitable for a model catalog. +func (r *CompositeRouteResolver) ListExactPublicModels(ctx context.Context, groupID int64, endpoint string, includeSystemOne bool) ([]string, error) { + if r == nil || r.repo == nil || groupID <= 0 { + return nil, nil + } + routes, err := r.repo.ListByGroup(ctx, groupID, false) + if err != nil { + return nil, err + } + models := make([]string, 0, len(routes)) + for _, route := range routes { + if route.Enabled && route.MatchType == CompositeRouteMatchExact && + (endpoint == "" || normalizeCompositeRouteEndpoint(route.Endpoint) == CompositeRouteEndpointAny || normalizeCompositeRouteEndpoint(route.Endpoint) == endpoint) { + if model := strings.TrimSpace(route.PublicModel); model != "" { + models = append(models, model) + } + } + } + if !includeSystemOne { + models = filterSystemOneRouteModels(routes, models, endpoint) + } + return models, nil +} + +func (r *CompositeRouteResolver) FilterCodexModels(ctx context.Context, groupID int64, models []string) ([]string, error) { + if r == nil || r.repo == nil || groupID <= 0 { + return models, nil + } + routes, err := r.repo.ListByGroup(ctx, groupID, false) + if err != nil { + return nil, err + } + return filterSystemOneRouteModels(routes, models, CompositeRouteEndpointResponses), nil +} + +func filterSystemOneRouteModels(routes []CompositeModelRoute, models []string, endpoint string) []string { + filtered := make([]string, 0, len(models)) + for _, model := range models { + route, matched := matchCompositeRoute(routes, model, normalizeCompositeRouteEndpoint(endpoint)) + if !matched || route.TargetPlatform != PlatformTypeSafe { + filtered = append(filtered, model) + } + } + return filtered +} + func (r *CompositeRouteResolver) Resolve(ctx context.Context, groupID int64, model, endpoint string) (CompositeRouteDecision, error) { model = strings.TrimSpace(model) endpoint = normalizeCompositeRouteEndpoint(endpoint) diff --git a/backend/internal/service/composite_route_resolver_test.go b/backend/internal/service/composite_route_resolver_test.go index 474b29387bb2..92827ab659de 100644 --- a/backend/internal/service/composite_route_resolver_test.go +++ b/backend/internal/service/composite_route_resolver_test.go @@ -12,6 +12,85 @@ type compositeRouteRepoStub struct { routes []CompositeModelRoute } +func TestCompositeRouteResolverCodexFilteringMatchesProductionPriority(t *testing.T) { + for _, scenario := range []struct { + name string + routes []CompositeModelRoute + want []string + }{ + { + name: "endpoint beats priority", + routes: []CompositeModelRoute{ + {ID: 1, PublicModel: "alias", MatchType: CompositeRouteMatchExact, Endpoint: CompositeRouteEndpointAny, TargetPlatform: PlatformOpenAI, Priority: 1}, + {ID: 2, PublicModel: "alias", MatchType: CompositeRouteMatchExact, Endpoint: CompositeRouteEndpointResponses, TargetPlatform: PlatformTypeSafe, Priority: 100}, + }, + want: []string{"unrouted"}, + }, + { + name: "Responses OpenAI overrides any TypeSafe", + routes: []CompositeModelRoute{ + {ID: 1, PublicModel: "alias", MatchType: CompositeRouteMatchExact, Endpoint: CompositeRouteEndpointAny, TargetPlatform: PlatformTypeSafe, Priority: 1}, + {ID: 2, PublicModel: "alias", MatchType: CompositeRouteMatchExact, Endpoint: CompositeRouteEndpointResponses, TargetPlatform: PlatformOpenAI, Priority: 100}, + }, + want: []string{"alias", "unrouted"}, + }, + { + name: "longer prefix beats priority", + routes: []CompositeModelRoute{ + {ID: 1, PublicModel: "a", MatchType: CompositeRouteMatchPrefix, TargetPlatform: PlatformOpenAI, Priority: 1}, + {ID: 2, PublicModel: "ali", MatchType: CompositeRouteMatchPrefix, TargetPlatform: PlatformTypeSafe, Priority: 100}, + }, + want: []string{"unrouted"}, + }, + { + name: "exact beats endpoint prefix", + routes: []CompositeModelRoute{ + {ID: 1, PublicModel: "alias", MatchType: CompositeRouteMatchPrefix, Endpoint: CompositeRouteEndpointResponses, TargetPlatform: PlatformTypeSafe}, + {ID: 2, PublicModel: "alias", MatchType: CompositeRouteMatchExact, Endpoint: CompositeRouteEndpointAny, TargetPlatform: PlatformOpenAI}, + }, + want: []string{"alias", "unrouted"}, + }, + { + name: "lower priority wins", + routes: []CompositeModelRoute{ + {ID: 1, PublicModel: "alias", MatchType: CompositeRouteMatchExact, TargetPlatform: PlatformTypeSafe, Priority: 100}, + {ID: 2, PublicModel: "alias", MatchType: CompositeRouteMatchExact, TargetPlatform: PlatformOpenAI, Priority: 1}, + }, + want: []string{"alias", "unrouted"}, + }, + { + name: "lower ID breaks tie", + routes: []CompositeModelRoute{ + {ID: 2, PublicModel: "alias", MatchType: CompositeRouteMatchExact, TargetPlatform: PlatformOpenAI}, + {ID: 1, PublicModel: "alias", MatchType: CompositeRouteMatchExact, TargetPlatform: PlatformTypeSafe}, + }, + want: []string{"unrouted"}, + }, + } { + t.Run(scenario.name, func(t *testing.T) { + for index := range scenario.routes { + scenario.routes[index].Enabled = true + scenario.routes[index].GroupID = 7 + } + resolver := NewCompositeRouteResolver(compositeRouteRepoStub{routes: scenario.routes}) + models, err := resolver.FilterCodexModels(context.Background(), 7, []string{"alias", "unrouted"}) + require.NoError(t, err) + require.Equal(t, scenario.want, models) + decision, err := resolver.Resolve(context.Background(), 7, "alias", CompositeRouteEndpointResponses) + require.NoError(t, err) + require.True(t, decision.Matched) + require.Equal(t, decision.TargetPlatform != PlatformTypeSafe, len(models) == 2) + exactModels, err := resolver.ListExactPublicModels(context.Background(), 7, CompositeRouteEndpointResponses, false) + require.NoError(t, err) + if decision.TargetPlatform == PlatformTypeSafe { + require.Empty(t, exactModels) + } else { + require.Contains(t, exactModels, "alias") + } + }) + } +} + func (s compositeRouteRepoStub) ListByGroup(ctx context.Context, groupID int64, includeDisabled bool) ([]CompositeModelRoute, error) { routes := make([]CompositeModelRoute, 0, len(s.routes)) for _, route := range s.routes { diff --git a/backend/internal/service/domain_constants.go b/backend/internal/service/domain_constants.go index 4ff97cfd4dec..cf39bb03a5f1 100644 --- a/backend/internal/service/domain_constants.go +++ b/backend/internal/service/domain_constants.go @@ -44,13 +44,15 @@ const ( PlatformAntigravity = domain.PlatformAntigravity PlatformGrok = domain.PlatformGrok // 国产 OpenAI 兼容供应商(与 grok 一样经 OpenAI 网关转发)。 - PlatformKimi = domain.PlatformKimi - PlatformZhipu = domain.PlatformZhipu - PlatformDeepseek = domain.PlatformDeepseek - PlatformMiniMax = domain.PlatformMiniMax - PlatformTypeSafe = domain.PlatformTypeSafe - PlatformOpenCodeGo = domain.PlatformOpenCodeGo - PlatformComposite = domain.PlatformComposite + PlatformKimi = domain.PlatformKimi + PlatformZhipu = domain.PlatformZhipu + PlatformDeepseek = domain.PlatformDeepseek + PlatformMiniMax = domain.PlatformMiniMax + PlatformOpenCodeGo = domain.PlatformOpenCodeGo + PlatformTypeSafe = domain.PlatformTypeSafe + PlatformCommandCode = domain.PlatformCommandCode + PlatformCline = domain.PlatformCline + PlatformComposite = domain.PlatformComposite // PlatformKiro is retained for unsupported-platform threshold tests and legacy // account rows. Scheduling-threshold evaluation never pauses kiro accounts. PlatformKiro = "kiro" @@ -86,6 +88,10 @@ const ( DefaultOpenCodeGoBaseURL = "https://opencode.ai/zen/go/v1" // OpenCode Zen:按量付费网关,模型列表为 /zen/v1/models。 DefaultOpenCodeZenBaseURL = "https://opencode.ai/zen/v1" + // Command Code Provider API:Chat Completions / Responses / models 共用 /provider/v1 基址。 + DefaultCommandCodeBaseURL = "https://api.commandcode.ai/provider/v1" + // Cline API:只提供 Chat Completions({base}/chat/completions)与模型列表。 + DefaultClineBaseURL = "https://api.cline.bot/api/v1" ) // 国产供应商 Anthropic 协议端点的默认 base_url(上游路径为 {base}/v1/messages)。 @@ -99,16 +105,14 @@ const ( // OpenCode Go Anthropic 基址不含 /v1:nativeAnthropicTargetURL 会再拼 /v1/messages。 DefaultOpenCodeGoAnthropicBaseURL = "https://opencode.ai/zen/go" DefaultOpenCodeZenAnthropicBaseURL = "https://opencode.ai/zen" + // Command Code 的 Anthropic 端点为 /provider/v1/messages(Claude 系模型只在此端点提供)。 + DefaultCommandCodeAnthropicBaseURL = "https://api.commandcode.ai/provider" ) -// IsCNProvider 报告 platform 是否为国产 OpenAI 兼容供应商(kimi/zhipu/deepseek/minimax)。 +// IsCNProvider 报告 platform 是否为国产 OpenAI 兼容供应商(kimi/zhipu/deepseek/minimax), +// 以平台清单(domain/platforms.go)为准。 func IsCNProvider(platform string) bool { - switch platform { - case PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax: - return true - default: - return false - } + return domain.IsCNProviderPlatform(platform) } // IsOpenCodeGo 报告 platform 是否为 OpenCode Go 订阅网关。 @@ -118,30 +122,19 @@ func IsOpenCodeGo(platform string) bool { // IsMultiProtocolAPIKeyProvider 报告 platform 是否为多协议 API Key 网关 // (国产供应商 + OpenCode):走 OpenAI 网关、支持 adaptive 协议分流。 +// 归属以 provider profile 登记为准(见 provider_profile.go)。 func IsMultiProtocolAPIKeyProvider(platform string) bool { - return IsCNProvider(platform) || platform == PlatformOpenCodeGo + return LookupProviderProfile(platform) != nil } -// AllowedQuotaPlatforms 是允许设置 user × platform quota 的平台列表(单一权威来源)。 -// ent/schema/user_platform_quota.go 的 Validate 函数独立维护(构建期约束), -// 若新增平台需同步修改该 schema。 -var AllowedQuotaPlatforms = []string{ - PlatformAnthropic, - PlatformOpenAI, - PlatformGemini, - PlatformAntigravity, - PlatformGrok, - PlatformKimi, - PlatformZhipu, - PlatformDeepseek, - PlatformMiniMax, - PlatformOpenCodeGo, - PlatformTypeSafe, -} +// AllowedQuotaPlatforms 是允许设置 user × platform quota 的平台列表:全部已登记的 +// 具体平台(domain/platforms.go),ent/schema/user_platform_quota.go 的校验同源。 +var AllowedQuotaPlatforms = domain.ConcretePlatformIDs() // AllowedSchedulingThresholdPlatforms 是允许设置账号自动停调阈值的平台列表。 // openai/anthropic/grok 有原生用量窗口;kimi/zhipu/minimax 的 Coding Plan 同样暴露 // 5h/weekly 滚动窗口,纳入阈值评估。deepseek 为余额型,走余额检测而非阈值。 +// OpenCode Go 与 Command Code 的订阅套餐另有月度窗口。 var AllowedSchedulingThresholdPlatforms = []string{ PlatformOpenAI, PlatformAnthropic, @@ -150,6 +143,7 @@ var AllowedSchedulingThresholdPlatforms = []string{ PlatformZhipu, PlatformMiniMax, PlatformOpenCodeGo, + PlatformCommandCode, } // IsAllowedQuotaPlatform 报告 s 是否为合法的 quota platform 标识。 diff --git a/backend/internal/service/gateway_cache_ttl_order_test.go b/backend/internal/service/gateway_cache_ttl_order_test.go new file mode 100644 index 000000000000..fe319052f5f4 --- /dev/null +++ b/backend/internal/service/gateway_cache_ttl_order_test.go @@ -0,0 +1,156 @@ +package service + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// requireCacheTTLOrderAccepted fails when a ttl=1h breakpoint follows a 5m one +// in Anthropic's processing order (tools, system, messages). Anthropic rejects +// such requests with HTTP 400. +func requireCacheTTLOrderAccepted(t *testing.T, body []byte) { + t.Helper() + _, messagePaths, toolPaths, systemPaths := collectCacheControlPaths(body) + var paths []string + paths = append(paths, toolPaths...) + paths = append(paths, systemPaths...) + paths = append(paths, messagePaths...) + shortTTLPath := "" + for _, path := range paths { + if gjson.GetBytes(body, path+".ttl").String() == cacheTTLTarget1h { + require.Empty(t, shortTTLPath, "1h breakpoint %s follows 5m breakpoint %s in %s", path, shortTTLPath, body) + continue + } + if shortTTLPath == "" { + shortTTLPath = path + } + } +} + +func TestNormalizeCacheControlTTLOrder_RaisesBreakpointsBeforeLast1h(t *testing.T) { + body := []byte(`{"tools":[{"name":"t","input_schema":{"type":"object"},"cache_control":{"type":"ephemeral"}}],` + + `"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"5m"}}],` + + `"messages":[{"role":"user","content":[` + + `{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1h"}},` + + `{"type":"text","text":"b","cache_control":{"type":"ephemeral","ttl":"5m"}}]}]}`) + + out := normalizeCacheControlTTLOrder(body) + + require.Equal(t, "1h", gjson.GetBytes(out, "tools.0.cache_control.ttl").String(), "omitted ttl counts as 5m") + require.Equal(t, "1h", gjson.GetBytes(out, "system.0.cache_control.ttl").String()) + require.Equal(t, "1h", gjson.GetBytes(out, "messages.0.content.0.cache_control.ttl").String()) + require.Equal(t, "5m", gjson.GetBytes(out, "messages.0.content.1.cache_control.ttl").String(), + "a 5m breakpoint after the last 1h one is valid and must keep its ttl") + requireCacheTTLOrderAccepted(t, out) +} + +func TestNormalizeCacheControlTTLOrder_TopLevelCacheControlIsLast(t *testing.T) { + body := []byte(`{"cache_control":{"type":"ephemeral","ttl":"1h"},` + + `"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"5m"}}],` + + `"messages":[{"role":"user","content":"hi"}]}`) + + out := normalizeCacheControlTTLOrder(body) + + require.Equal(t, "1h", gjson.GetBytes(out, "system.0.cache_control.ttl").String()) +} + +func TestNormalizeCacheControlTTLOrder_KeepsValidRequestsUnchanged(t *testing.T) { + cases := map[string]string{ + "no breakpoints": `{"system":"s","messages":[{"role":"user","content":"hi"}]}`, + "all 5m": `{"tools":[{"name":"t","input_schema":{"type":"object"},"cache_control":{"type":"ephemeral"}}],` + + `"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"5m"}}],` + + `"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"5m"}}]}]}`, + "1h before 5m": `{"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + + `"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + require.Equal(t, body, string(normalizeCacheControlTTLOrder([]byte(body)))) + }) + } +} + +// A non-Claude Code client sends one 1h breakpoint. The mimic path adds 5m +// breakpoints to the injected system block and the last tool, which come first +// in Anthropic's processing order. The outbound body must still be accepted. +func TestEnforceCacheControlLimit_MimicPathKeepsClient1hBreakpointValid(t *testing.T) { + cases := map[string]string{ + "message 1h": `{"model":"claude-sonnet-4-6","messages":[{"role":"user","content":[` + + `{"type":"text","text":"Reply OK.","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`, + "system 1h moved into messages": `{"model":"claude-sonnet-4-6",` + + `"system":[{"type":"text","text":"Be terse.","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + + `"messages":[{"role":"user","content":"Reply OK."}]}`, + "tools and message 1h": `{"model":"claude-sonnet-4-6","tools":[{"name":"probe","input_schema":{"type":"object"}}],` + + `"messages":[{"role":"user","content":[` + + `{"type":"text","text":"Reply OK.","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`, + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + body := []byte(raw) + var system any + if sys := gjson.GetBytes(body, "system"); sys.Exists() { + require.NoError(t, json.Unmarshal([]byte(sys.Raw), &system)) + } + body = rewriteSystemForNonClaudeCode(body, system) + body = applyToolsLastCacheBreakpoint(body) + + out := enforceCacheControlLimit(body) + + requireCacheTTLOrderAccepted(t, out) + }) + } +} + +func TestForwardCountTokens_MimicPathKeepsClient1hBreakpointValid(t *testing.T) { + gin.SetMode(gin.TestMode) + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages/count_tokens", nil) + c.Request.Header.Set("User-Agent", "third-party-client/1.0") + + // The mimic path adds a 5m breakpoint to tools[-1], ahead of the client's 1h one. + body := []byte(`{"model":"claude-sonnet-4-6",` + + `"tools":[{"name":"probe","description":"d","input_schema":{"type":"object"}}],` + + `"messages":[{"role":"user","content":[{"type":"text","text":"Reply OK.","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`) + parsed := &ParsedRequest{Body: NewRequestBodyRef(body), Model: "claude-sonnet-4-6"} + + upstream := &anthropicHTTPUpstreamRecorder{ + resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"input_tokens":42}`)), + }, + } + svc := &GatewayService{ + cfg: &config.Config{Gateway: config.GatewayConfig{MaxLineSize: defaultMaxLineSize}}, + httpUpstream: upstream, + rateLimitService: &RateLimitService{}, + } + account := &Account{ + ID: 402, + Name: "count-tokens-ttl-order-test", + Platform: PlatformAnthropic, + Type: AccountTypeSetupToken, + Concurrency: 1, + Credentials: map[string]any{"access_token": "test-oauth-token"}, + Status: StatusActive, + Schedulable: true, + } + + require.NoError(t, svc.ForwardCountTokens(context.Background(), c, account, parsed)) + + require.True(t, gjson.GetBytes(upstream.lastBody, "tools.0.cache_control").Exists(), + "the mimic tools breakpoint should still be present") + requireCacheTTLOrderAccepted(t, upstream.lastBody) +} diff --git a/backend/internal/service/gateway_claude_oauth_body.go b/backend/internal/service/gateway_claude_oauth_body.go index 9fa4c1e134be..085cfefbd92f 100644 --- a/backend/internal/service/gateway_claude_oauth_body.go +++ b/backend/internal/service/gateway_claude_oauth_body.go @@ -1052,9 +1052,15 @@ func collectCacheControlPaths(body []byte) (invalidThinking []cacheControlPath, return invalidThinking, messagePaths, toolPaths, systemPaths } -// enforceCacheControlLimit 强制执行 cache_control 块数量限制(最多 4 个) -// 超限时优先移除工具断点,再移除 messages 断点,最后才移除 system 断点。 +// enforceCacheControlLimit 让出站 cache_control 满足上游约束: +// 先执行块数量限制,再修正 TTL 顺序。 func enforceCacheControlLimit(body []byte) []byte { + return normalizeCacheControlTTLOrder(enforceCacheControlBlockLimit(body)) +} + +// enforceCacheControlBlockLimit 强制执行 cache_control 块数量限制(最多 4 个) +// 超限时优先移除工具断点,再移除 messages 断点,最后才移除 system 断点。 +func enforceCacheControlBlockLimit(body []byte) []byte { if len(body) == 0 { return body } @@ -1137,6 +1143,46 @@ func enforceCacheControlLimit(body []byte) []byte { return body } +// normalizeCacheControlTTLOrder raises earlier ephemeral breakpoints to 1h when +// a later breakpoint uses 1h. Anthropic processes tools, system, then messages, +// and rejects a ttl=1h breakpoint that follows a 5m one; an omitted ttl means 5m. +// The mimic path adds its own 5m breakpoints to tools and system, so a client +// 1h breakpoint in messages would otherwise turn a valid request into a 400. +// +// Raising keeps the client's choice. Tokens before a 1h breakpoint are already +// written at the 1h rate, so the earlier breakpoints add no cache write cost. +// Requests with a valid order are returned unchanged. +func normalizeCacheControlTTLOrder(body []byte) []byte { + _, messagePaths, toolPaths, systemPaths := collectCacheControlPaths(body) + paths := make([]string, 0, len(toolPaths)+len(systemPaths)+len(messagePaths)+1) + paths = append(paths, toolPaths...) + paths = append(paths, systemPaths...) + paths = append(paths, messagePaths...) + // A top-level cache_control applies to the last cacheable block. + if gjson.GetBytes(body, "cache_control").Exists() { + paths = append(paths, "cache_control") + } + + last1h := -1 + for i, path := range paths { + if gjson.GetBytes(body, path+".ttl").String() == cacheTTLTarget1h { + last1h = i + } + } + + out := body + for _, path := range paths[:last1h+1] { + cc := gjson.GetBytes(out, path) + if cc.Get("type").String() != "ephemeral" || cc.Get("ttl").String() == cacheTTLTarget1h { + continue + } + if next, err := sjson.SetBytes(out, path+".ttl", cacheTTLTarget1h); err == nil { + out = next + } + } + return out +} + // injectAnthropicCacheControlTTL1h 将已有 ephemeral cache_control 块的 ttl 强制写为 1h。 // 仅修改已经存在的 cache_control,不新增缓存断点。 func injectAnthropicCacheControlTTL1h(body []byte) []byte { diff --git a/backend/internal/service/gateway_forward_as_chat_completions.go b/backend/internal/service/gateway_forward_as_chat_completions.go index 5fb09ac3a1d6..13af1d147518 100644 --- a/backend/internal/service/gateway_forward_as_chat_completions.go +++ b/backend/internal/service/gateway_forward_as_chat_completions.go @@ -281,7 +281,7 @@ func (s *GatewayService) handleCCBufferedFromAnthropic( } if event.Type == "content_block_delta" && event.Delta != nil && finalResp != nil && event.Index != nil { idx := *event.Index - if idx < len(finalResp.Content) { + if idx >= 0 && idx < len(finalResp.Content) { switch event.Delta.Type { case "text_delta": finalResp.Content[idx].Text += event.Delta.Text @@ -443,10 +443,11 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( if event == nil { return false } - // Drop Anthropic keepalive pings before OpenAI conversion: - // leaking `event: ping` frames crashes OpenAI-stream clients. - // Error events must still forward — they carry upstream failures. if event.Type == "ping" { + if _, err := fmt.Fprint(c.Writer, ": ping\n\n"); err != nil { + return true + } + c.Writer.Flush() return false } if firstChunk { diff --git a/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go b/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go new file mode 100644 index 000000000000..df08d20f78ce --- /dev/null +++ b/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go @@ -0,0 +1,205 @@ +//go:build unit + +package service + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type ccPingResponseWriter struct { + *httptest.ResponseRecorder + flushed chan string + failPing bool + failed bool + writesAfterError int + flushesAfterError int +} + +func (writer *ccPingResponseWriter) Write(data []byte) (int, error) { + if writer.failed { + writer.writesAfterError++ + } + if writer.failPing && string(data) == ": ping\n\n" { + writer.failed = true + return 0, errors.New("heartbeat write failed") + } + return writer.ResponseRecorder.Write(data) +} + +func (writer *ccPingResponseWriter) Flush() { + if writer.failed { + writer.flushesAfterError++ + } + writer.ResponseRecorder.Flush() + if writer.flushed != nil { + writer.flushed <- writer.Body.String() + } +} + +func TestHandleCCStreamingFromAnthropic_PingFlushesImmediately(t *testing.T) { + writer := &ccPingResponseWriter{ + ResponseRecorder: httptest.NewRecorder(), + flushed: make(chan string, 32), + } + context, _ := gin.CreateTestContext(writer) + reader, upstream := io.Pipe() + type outcome struct { + result *ForwardResult + err error + } + finished := make(chan outcome, 1) + go func() { + result, err := (&GatewayService{}).handleCCStreamingFromAnthropic( + &http.Response{Body: reader}, context, "gpt-5", "claude-sonnet-4.5", nil, time.Now(), + ) + finished <- outcome{result: result, err: err} + }() + t.Cleanup(func() { + _ = upstream.Close() + _ = reader.Close() + }) + + awaitFlush := func() string { + t.Helper() + select { + case body := <-writer.flushed: + return body + case <-time.After(5 * time.Second): + t.Fatal("event did not flush before the next upstream event or EOF") + return "" + } + } + var body string + send := func(eventType, payload string) { + t.Helper() + _, err := io.WriteString(upstream, "event: "+eventType+"\ndata: "+payload+"\n\n") + require.NoError(t, err) + body = awaitFlush() + } + ping := func() { + t.Helper() + previous := body + send("ping", `{"type":"ping"}`) + require.Equal(t, previous+": ping\n\n", body) + } + + ping() + ping() + send("message_start", `{"type":"message_start","message":{"id":"msg_ping","role":"assistant","content":[],"usage":{"input_tokens":10}}}`) + send("content_block_start", `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`) + send("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}`) + require.Contains(t, body, `"content":"hello"`) + ping() + send("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" world"}}`) + require.Contains(t, body, `"content":" world"`) + send("message_delta", `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}`) + send("message_stop", `{"type":"message_stop"}`) + ping() + require.NotContains(t, body, "[DONE]") + require.NoError(t, upstream.Close()) + + select { + case got := <-finished: + require.NoError(t, got.err) + require.Equal(t, 10, got.result.Usage.InputTokens) + require.Equal(t, 7, got.result.Usage.OutputTokens) + require.NotNil(t, got.result.FirstTokenMs) + case <-time.After(5 * time.Second): + t.Fatal("stream did not finish after EOF") + } + body = writer.Body.String() + require.NotContains(t, body, "event:") + require.NotContains(t, body, `"type":"ping"`) + require.Equal(t, 4, strings.Count(body, ": ping\n\n")) + require.Equal(t, 1, strings.Count(body, "data: [DONE]\n\n")) + require.True(t, strings.HasSuffix(body, "data: [DONE]\n\n")) + var content strings.Builder + var usage *apicompat.ChatUsage + var finishedChoice bool + for _, frame := range strings.Split(body, "\n\n") { + if frame == "" || frame == ": ping" || frame == "data: [DONE]" { + continue + } + require.True(t, strings.HasPrefix(frame, "data: ")) + var chunk apicompat.ChatCompletionsChunk + require.NoError(t, json.Unmarshal([]byte(strings.TrimPrefix(frame, "data: ")), &chunk)) + require.Equal(t, "chat.completion.chunk", chunk.Object) + for _, choice := range chunk.Choices { + if choice.Delta.Content != nil { + content.WriteString(*choice.Delta.Content) + } + if choice.FinishReason != nil { + require.Equal(t, "stop", *choice.FinishReason) + finishedChoice = true + } + } + if chunk.Usage != nil { + usage = chunk.Usage + } + } + require.Equal(t, "hello world", content.String()) + require.True(t, finishedChoice) + require.NotNil(t, usage) + require.Equal(t, 10, usage.PromptTokens) + require.Equal(t, 7, usage.CompletionTokens) + require.Equal(t, 17, usage.TotalTokens) +} + +func TestHandleCCStreamingFromAnthropic_PingDoesNotSetFirstToken(t *testing.T) { + writer := httptest.NewRecorder() + context, _ := gin.CreateTestContext(writer) + result, err := (&GatewayService{}).handleCCStreamingFromAnthropic( + &http.Response{Body: io.NopCloser(strings.NewReader("event: ping\ndata: {\"type\":\"ping\"}\n\n"))}, + context, "gpt-5", "claude-sonnet-4.5", nil, time.Now(), + ) + require.NoError(t, err) + require.Nil(t, result.FirstTokenMs) + require.True(t, strings.HasPrefix(writer.Body.String(), ": ping\n\n")) +} + +func TestHandleCCStreamingFromAnthropic_PingWriteFailureStopsDownstream(t *testing.T) { + for _, withUsage := range []bool{false, true} { + name := "before_message_start" + if withUsage { + name = "after_message_start" + } + t.Run(name, func(t *testing.T) { + writer := &ccPingResponseWriter{ResponseRecorder: httptest.NewRecorder(), failPing: true} + context, _ := gin.CreateTestContext(writer) + stream := "" + if withUsage { + stream += "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_failure\",\"usage\":{\"input_tokens\":10}}}\n\n" + } + stream += "event: ping\ndata: {\"type\":\"ping\"}\n\n" + + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":7}}\n\n" + + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" + result, err := (&GatewayService{}).handleCCStreamingFromAnthropic( + &http.Response{Body: io.NopCloser(strings.NewReader(stream))}, + context, "gpt-5", "claude-sonnet-4.5", nil, time.Now(), + ) + require.NoError(t, err) + require.True(t, writer.failed) + require.Zero(t, writer.writesAfterError) + require.Zero(t, writer.flushesAfterError) + require.NotContains(t, writer.Body.String(), "[DONE]") + require.Zero(t, result.Usage.OutputTokens) + if withUsage { + require.Equal(t, 10, result.Usage.InputTokens) + } else { + require.Zero(t, result.Usage.InputTokens) + require.Nil(t, result.FirstTokenMs) + } + }) + } +} diff --git a/backend/internal/service/gateway_forward_as_responses.go b/backend/internal/service/gateway_forward_as_responses.go index 96de9cdda764..93a75a500be9 100644 --- a/backend/internal/service/gateway_forward_as_responses.go +++ b/backend/internal/service/gateway_forward_as_responses.go @@ -439,7 +439,7 @@ func (s *GatewayService) handleResponsesBufferedStreamingResponse( } if event.Type == "content_block_delta" && event.Delta != nil && finalResp != nil && event.Index != nil { idx := *event.Index - if idx < len(finalResp.Content) { + if idx >= 0 && idx < len(finalResp.Content) { switch event.Delta.Type { case "text_delta": finalResp.Content[idx].Text += event.Delta.Text diff --git a/backend/internal/service/gateway_service.go b/backend/internal/service/gateway_service.go index 4b64b3477ab5..616019abc0a1 100644 --- a/backend/internal/service/gateway_service.go +++ b/backend/internal/service/gateway_service.go @@ -745,8 +745,6 @@ type UpstreamFailoverError struct { ClientErrorParam string ClientMessage string SuppressAccountHealthPenalty bool - CiphertextAccountMismatch bool // 本账号无法校验请求里的密文且无法就地修复;原样请求可能被签发该密文的组织下的其他账号接受 - RecoveryRetrySpent bool // 本账号的剥离重发已发出并因账号自身的原因失败;换下一个账号,不得在本账号再试 } func (e *UpstreamFailoverError) Error() string { @@ -1601,6 +1599,21 @@ func explicitModelMappingClaims(account Account, model string) bool { return ok && strings.TrimSpace(mapped) != "" } +// GetCompositeRouteModels returns public IDs from enabled exact composite routes. +func (s *GatewayService) GetCompositeRouteModels(ctx context.Context, groupID *int64, endpoint string, includeSystemOne bool) ([]string, error) { + if s == nil || s.compositeResolver == nil || groupID == nil { + return nil, nil + } + return s.compositeResolver.ListExactPublicModels(ctx, *groupID, endpoint, includeSystemOne) +} + +func (s *GatewayService) FilterCompositeCodexModels(ctx context.Context, groupID int64, models []string) ([]string, error) { + if s == nil || s.compositeResolver == nil { + return models, nil + } + return s.compositeResolver.FilterCodexModels(ctx, groupID, models) +} + // GetSchedulablePlatforms returns the concrete platforms that currently have // schedulable accounts in the target group. func (s *GatewayService) GetSchedulablePlatforms(ctx context.Context, groupID *int64) map[string]struct{} { diff --git a/backend/internal/service/gateway_tool_changes_beta_test.go b/backend/internal/service/gateway_tool_changes_beta_test.go new file mode 100644 index 000000000000..a7ffd6a79e28 --- /dev/null +++ b/backend/internal/service/gateway_tool_changes_beta_test.go @@ -0,0 +1,63 @@ +//go:build unit + +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestBuildUpstreamRequestToolChangeBetas(t *testing.T) { + const mid = "mid-conversation-tool-changes-2026-07-01" + const inline = "inline-tools-2026-09-15" + for _, tc := range []struct { + name, header string + drop map[string]struct{} + wantMid, wantInline bool + }{ + {"both", mid + "," + inline, nil, true, true}, + {"mid only", mid, nil, true, false}, + {"inline only", inline, nil, false, true}, + {"absent", "", nil, false, false}, + {"unknown", inline + "-other", nil, false, false}, + {"duplicate", inline + ", " + inline + "," + mid, nil, true, true}, + {"filter inline", mid + "," + inline, map[string]struct{}{inline: {}}, true, false}, + {"filter mid", mid + "," + inline, map[string]struct{}{mid: {}}, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil) + c.Request.Header.Set("anthropic-beta", tc.header+",unrelated-beta") + if tc.drop != nil { + c.Set(betaPolicyFilterSetKey, tc.drop) + } + account := &Account{ID: 701, Platform: PlatformAnthropic, Type: AccountTypeOAuth, + Credentials: map[string]any{"access_token": "test-token"}, Status: StatusActive, Schedulable: true} + body := []byte(`{"model":"claude-opus-5-5","max_tokens":1024,"messages":[{"role":"user","content":[{"type":"tool_addition","tool":{"type":"tool_reference","tool_name":"lookup"}},{"type":"text","text":"Use lookup"}]}]}`) + svc := &GatewayService{cfg: &config.Config{}} + req, _, err := svc.buildUpstreamRequest(context.Background(), c, account, body, + "test-token", "oauth", "claude-opus-5-5", false, true) + require.NoError(t, err) + out := readUpstreamBodyForTest(t, req) + require.JSONEq(t, gjson.GetBytes(body, "messages").Raw, gjson.GetBytes(out, "messages").Raw) + header := getHeaderRaw(req.Header, "anthropic-beta") + require.Equal(t, tc.wantMid, containsBetaToken(header, mid)) + require.Equal(t, tc.wantInline, containsBetaToken(header, inline)) + require.False(t, containsBetaToken(header, "unrelated-beta")) + require.False(t, containsBetaToken(header, inline+"-other")) + require.True(t, containsBetaToken(header, "oauth-2025-04-20")) + if tc.wantInline { + require.Equal(t, 1, strings.Count(header, inline)) + } + }) + } +} diff --git a/backend/internal/service/gateway_upstream_request.go b/backend/internal/service/gateway_upstream_request.go index b6e91a52a87c..06ee09822ce1 100644 --- a/backend/internal/service/gateway_upstream_request.go +++ b/backend/internal/service/gateway_upstream_request.go @@ -545,14 +545,20 @@ func (s *GatewayService) computeFinalAnthropicBeta( if tokenType == "oauth" { if mimicClaudeCode { - // Keep the default beta set, with a narrow opt-in for legacy structured - // output requests. Unknown client betas remain excluded and policy drops - // still take precedence over this compatibility token. - incomingBeta := "" - if containsBetaToken(clientBeta, claude.BetaStructuredOutputs) { - incomingBeta = claude.BetaStructuredOutputs + // Preserve explicitly requested compatibility tokens so the forwarded + // body and headers agree. Unknown betas remain excluded; policy drops + // still take precedence. Missing companion tokens are not inferred. + var incomingBetas []string + for _, token := range []string{ + claude.BetaStructuredOutputs, + claude.BetaMidConversationToolChanges, + claude.BetaInlineTools, + } { + if containsBetaToken(clientBeta, token) { + incomingBetas = append(incomingBetas, token) + } } - return mergeAnthropicBetaDropping(claude.FullClaudeCodeMimicryBetas(), incomingBeta, effectiveDropSet), true + return mergeAnthropicBetaDropping(claude.FullClaudeCodeMimicryBetas(), strings.Join(incomingBetas, ","), effectiveDropSet), true } // 真 Claude Code 客户端透传路径 return stripBetaTokensWithSet(s.getBetaHeader(modelID, clientBeta), effectiveDropSet), true diff --git a/backend/internal/service/model_protocol_catalog.go b/backend/internal/service/model_protocol_catalog.go new file mode 100644 index 000000000000..3a7d0c75cd94 --- /dev/null +++ b/backend/internal/service/model_protocol_catalog.go @@ -0,0 +1,263 @@ +package service + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strings" + "sync" + "time" + + "github.com/tidwall/gjson" +) + +// 上游模型目录中的协议能力。部分聚合平台(Command Code)在 {Chat Completions 基址}/models +// 的每个模型上给出 supported_endpoints,例如: +// +// {"id":"deepseek/deepseek-v4-flash","supported_endpoints":["/chat/completions","/responses"]} +// {"id":"claude-sonnet-4-6","supported_endpoints":["/messages"]} +// +// 按模型分流时据此判断模型是否支持入站协议:支持就同协议直通,省去一次协议转换 +// (见 modelProtocolSet)。 +// +// 缓存范围:所有目录均按账号和地址隔离,包括官方默认地址。目录可能随 Key、租户 +// 或请求头不同,鉴权失败的刷新退避也不能影响其他账号(见 modelProtocolCatalogKey)。 +// +// 刷新在后台进行:过期时继续使用旧目录。目录从未加载成功时,首批请求最多等待 +// modelProtocolCatalogFirstLoadWait,各请求共用本轮刷新开始时的截止时间,并响应请求 +// 取消。超时后同一轮刷新中的后续请求立即回落内置规则,失败退避期内不再等待。 + +const ( + modelProtocolCatalogTTL = 30 * time.Minute + modelProtocolCatalogRetryBackoff = 5 * time.Minute + modelProtocolCatalogTimeout = 15 * time.Second + modelProtocolCatalogFirstLoadWait = 2 * time.Second + modelProtocolCatalogMaxBodyBytes = 2 << 20 +) + +type modelProtocolCatalogEntry struct { + // models 以小写模型 ID 为键,值为上游声明的协议(保持上游顺序)。 + models map[string][]string + fetchedAt time.Time + retryAt time.Time + refreshing bool + // done 在进行中的刷新结束时关闭。 + done chan struct{} + // firstLoadDeadline 是本轮冷启动刷新共享的等待截止时间。 + firstLoadDeadline time.Time +} + +type modelProtocolCatalog struct { + mu sync.Mutex + entries map[string]*modelProtocolCatalogEntry +} + +// upstreamModelProtocols 是全进程共享的模型协议目录。 +var upstreamModelProtocols modelProtocolCatalog + +func (c *modelProtocolCatalog) entryLocked(key string) *modelProtocolCatalogEntry { + if c.entries == nil { + c.entries = make(map[string]*modelProtocolCatalogEntry) + } + entry := c.entries[key] + if entry == nil { + entry = &modelProtocolCatalogEntry{} + c.entries[key] = entry + } + return entry +} + +// lookup 返回 key 目录中 model 支持的协议;目录尚未就绪或模型不在目录中时返回 nil。 +// 目录缺失或过期、且不在刷新中也不在失败退避期时,异步调用 refresh(由其调用 store); +// 目录从未加载成功且刷新进行中时,只等待本轮冷启动窗口的剩余时间。 +func (c *modelProtocolCatalog) lookup(ctx context.Context, key, model string, now time.Time, refresh func()) []string { + if ctx.Err() != nil { + return nil + } + c.mu.Lock() + defer c.mu.Unlock() + entry := c.entryLocked(key) + stale := entry.models == nil || now.Sub(entry.fetchedAt) >= modelProtocolCatalogTTL + if stale && !entry.refreshing && !now.Before(entry.retryAt) && refresh != nil { + entry.refreshing = true + entry.done = make(chan struct{}) + if entry.models == nil { + entry.firstLoadDeadline = now.Add(modelProtocolCatalogFirstLoadWait) + } + go refresh() + } + if entry.models == nil && entry.refreshing && now.Before(entry.firstLoadDeadline) { + done := entry.done + deadline := entry.firstLoadDeadline + c.mu.Unlock() + timer := time.NewTimer(time.Until(deadline)) + select { + case <-done: + case <-timer.C: + case <-ctx.Done(): + } + timer.Stop() + c.mu.Lock() + } + if entry.models == nil || ctx.Err() != nil { + return nil + } + return entry.models[strings.ToLower(strings.TrimSpace(model))] +} + +// store 写入一次刷新的结果;失败时保留旧目录并进入退避。 +func (c *modelProtocolCatalog) store(key string, models map[string][]string, err error, now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + entry := c.entryLocked(key) + entry.refreshing = false + if entry.done != nil { + close(entry.done) + entry.done = nil + } + if err != nil { + entry.retryAt = now.Add(modelProtocolCatalogRetryBackoff) + return + } + entry.models = models + entry.fetchedAt = now + entry.retryAt = time.Time{} +} + +// modelEndpointProtocol 把 supported_endpoints 中的端点路径映射为上游协议。 +func modelEndpointProtocol(endpoint string) string { + endpoint = strings.ToLower(strings.TrimRight(strings.TrimSpace(endpoint), "/")) + switch { + case strings.HasSuffix(endpoint, "/chat/completions"): + return APIProtocolChatCompletions + case strings.HasSuffix(endpoint, "/responses"): + return APIProtocolResponses + case strings.HasSuffix(endpoint, "/messages"): + return APIProtocolAnthropic + default: + return "" + } +} + +// parseModelProtocolCatalog 解析 /models 响应;没有任何模型带可识别的 supported_endpoints +// 时返回错误(上游不提供该字段,或是中转站的普通 /models)。 +func parseModelProtocolCatalog(body []byte) (map[string][]string, error) { + entries, err := extractUpstreamModelRawEntries(body) + if err != nil { + return nil, err + } + models := make(map[string][]string) + for _, raw := range entries { + id := strings.ToLower(strings.TrimSpace(gjson.GetBytes(raw, "id").String())) + if id == "" { + continue + } + var protocols []string + for _, endpoint := range gjson.GetBytes(raw, "supported_endpoints").Array() { + protocol := modelEndpointProtocol(endpoint.String()) + if protocol != "" && !slices.Contains(protocols, protocol) { + protocols = append(protocols, protocol) + } + } + if len(protocols) > 0 { + models[id] = protocols + } + } + if len(models) == 0 { + return nil, fmt.Errorf("model list has no supported_endpoints") + } + return models, nil +} + +// modelCatalogProtocols 返回上游模型目录中 model 支持的协议;平台不提供目录、账号固定了 +// 上游协议或目录尚未就绪时返回 nil。 +func (s *OpenAIGatewayService) modelCatalogProtocols(ctx context.Context, account *Account, model string) []string { + if s == nil || s.httpUpstream == nil || strings.TrimSpace(model) == "" || !account.routesByModel() { + return nil + } + if profile := account.providerProfile(); profile == nil || !profile.ModelCatalog { + return nil + } + if account.GetAPIProtocol() != APIProtocolAdaptive { + return nil + } + base := strings.TrimSpace(account.GetCNProtocolBaseURL(APIProtocolChatCompletions)) + if base == "" { + return nil + } + url := buildOpenAIModelsURL(base) + key := modelProtocolCatalogKey(account, url) + headers := modelProtocolCatalogHeaders(account, url) + proxyURL := "" + if account.ProxyID != nil && account.Proxy != nil { + proxyURL = account.Proxy.URL() + } + accountID, concurrency := account.ID, account.Concurrency + return upstreamModelProtocols.lookup(ctx, key, model, time.Now(), func() { + ctx, cancel := context.WithTimeout(context.Background(), modelProtocolCatalogTimeout) + defer cancel() + models, err := s.fetchModelProtocolCatalog(ctx, url, headers, proxyURL, accountID, concurrency) + if err != nil { + slog.Warn("model_protocol_catalog_refresh_failed", "account_id", accountID, "url", url, "error", err) + } else { + slog.Info("model_protocol_catalog_refreshed", "account_id", accountID, "url", url, "models", len(models)) + } + upstreamModelProtocols.store(key, models, err, time.Now()) + }) +} + +// modelProtocolCatalogKey 按账号和地址隔离目录及刷新退避,避免一个账号的目录或 +// 鉴权失败影响同地址的其他账号。 +func modelProtocolCatalogKey(account *Account, url string) string { + return fmt.Sprintf("%s#account=%d", url, account.ID) +} + +// modelProtocolCatalogHeaders 构造目录请求头,UA 收敛与账号请求头覆写与转发一致 +// (见 buildUpstreamRequest)。在请求时构造,后台刷新不再读取账号。 +func modelProtocolCatalogHeaders(account *Account, url string) http.Header { + headers := http.Header{} + headers.Set("Accept", "application/json") + if apiKey := strings.TrimSpace(account.GetOpenAIProtocolAPIKey()); apiKey != "" { + headers.Set("Authorization", "Bearer "+apiKey) + } + applyOpenCodeUpstreamUserAgent(account, url, headers) + account.ApplyHeaderOverrides(headers) + return headers +} + +func (s *OpenAIGatewayService) fetchModelProtocolCatalog(ctx context.Context, url string, headers http.Header, proxyURL string, accountID int64, concurrency int) (map[string][]string, error) { + validatedURL, err := s.validateUpstreamBaseURL(url) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(WithHTTPUpstreamRedirectsDisabled(ctx), http.MethodGet, validatedURL, nil) + if err != nil { + return nil, err + } + req.Header = headers.Clone() + resp, err := s.httpUpstream.Do(req, proxyURL, accountID, maxInt(concurrency, 1)) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("model list returned HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, modelProtocolCatalogMaxBodyBytes+1)) + if err != nil { + return nil, err + } + if len(body) > modelProtocolCatalogMaxBodyBytes { + return nil, fmt.Errorf("model list exceeds %d bytes", modelProtocolCatalogMaxBodyBytes) + } + return parseModelProtocolCatalog(body) +} + +// resolveUpstreamProtocolFor 是网关入口使用的 resolveUpstreamProtocol:按模型分流且平台 +// 提供模型目录时,带上目录中该模型支持的协议。 +func (s *OpenAIGatewayService) resolveUpstreamProtocolFor(ctx context.Context, account *Account, inbound, model string) string { + return resolveUpstreamProtocol(account, inbound, model, s.modelCatalogProtocols(ctx, account, model)) +} diff --git a/backend/internal/service/model_protocol_catalog_test.go b/backend/internal/service/model_protocol_catalog_test.go new file mode 100644 index 000000000000..f8918eedf1c8 --- /dev/null +++ b/backend/internal/service/model_protocol_catalog_test.go @@ -0,0 +1,420 @@ +//go:build unit + +package service + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// 取自 https://api.commandcode.ai/provider/v1/models 的片段(2026-09-26)。 +const commandCodeModelsSample = `{"object":"list","data":[ + {"id":"claude-sonnet-4-6","supported_endpoints":["/messages"]}, + {"id":"gpt-5.5","supported_endpoints":["/chat/completions","/responses"]}, + {"id":"deepseek/deepseek-v4-flash","supported_endpoints":["/chat/completions","/responses"]}, + {"id":"deepseek/deepseek-v4-flash-fast","supported_endpoints":["/chat/completions"]}, + {"id":"zai-org/GLM-5.3","supported_endpoints":["/chat/completions","/responses"]}, + {"id":"typesafe/jev","supported_endpoints":["/systemone"]}, + {"id":"no-endpoints"} +]}` + +func TestParseModelProtocolCatalog(t *testing.T) { + models, err := parseModelProtocolCatalog([]byte(commandCodeModelsSample)) + require.NoError(t, err) + require.Equal(t, map[string][]string{ + "claude-sonnet-4-6": {APIProtocolAnthropic}, + "gpt-5.5": {APIProtocolChatCompletions, APIProtocolResponses}, + "deepseek/deepseek-v4-flash": {APIProtocolChatCompletions, APIProtocolResponses}, + "deepseek/deepseek-v4-flash-fast": {APIProtocolChatCompletions}, + "zai-org/glm-5.3": {APIProtocolChatCompletions, APIProtocolResponses}, + }, models) + + _, err = parseModelProtocolCatalog([]byte(`{"data":[{"id":"gpt-5"},{"id":"glm-5"}]}`)) + require.Error(t, err, "a plain /models without supported_endpoints is not a protocol catalog") +} + +func TestModelProtocolCatalogLookupRefreshesInBackground(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var catalog modelProtocolCatalog + const key = "https://catalog.example/v1/models" + now := time.Now() + models, err := parseModelProtocolCatalog([]byte(commandCodeModelsSample)) + require.NoError(t, err) + catalog.store(key, models, nil, now) + var refreshes atomic.Int32 + refresh := func() { refreshes.Add(1) } + + require.Equal(t, []string{APIProtocolChatCompletions, APIProtocolResponses}, catalog.lookup(t.Context(), key, "ZAI-ORG/GLM-5.3", now, refresh)) + require.Nil(t, catalog.lookup(t.Context(), key, "unknown-model", now, refresh)) + require.Zero(t, refreshes.Load()) + + // A stale catalog is returned immediately while one refresh runs in the background. + time.Sleep(modelProtocolCatalogTTL) // synctest advances virtual time. + later := time.Now() + require.NotNil(t, catalog.lookup(t.Context(), key, "gpt-5.5", later, refresh)) + require.Equal(t, later, time.Now(), "a stale result must not wait for refresh") + synctest.Wait() + require.EqualValues(t, 1, refreshes.Load()) + require.NotNil(t, catalog.lookup(t.Context(), key, "gpt-5.5", later, refresh)) + require.EqualValues(t, 1, refreshes.Load(), "concurrent stale reads share one refresh") + catalog.store(key, nil, errors.New("boom"), later) + require.NotNil(t, catalog.lookup(t.Context(), key, "gpt-5.5", later.Add(time.Minute), refresh)) + require.EqualValues(t, 1, refreshes.Load(), "failure backoff preserves the old catalog") + require.NotNil(t, catalog.lookup(t.Context(), key, "gpt-5.5", later.Add(modelProtocolCatalogRetryBackoff), refresh)) + synctest.Wait() + require.EqualValues(t, 2, refreshes.Load()) + }) +} + +func TestModelProtocolCatalogFirstLoadSharesWaitDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var catalog modelProtocolCatalog + const key = "https://catalog.example/v1/models" + models := map[string][]string{"gpt-5.5": {APIProtocolResponses}} + release := make(chan struct{}) + var refreshes atomic.Int32 + refresh := func() { + refreshes.Add(1) + <-release + catalog.store(key, models, nil, time.Now()) + } + type result struct { + protocols []string + elapsed time.Duration + } + first, second := make(chan result, 1), make(chan result, 1) + lookup := func(done chan<- result) { + start := time.Now() + protocols := catalog.lookup(t.Context(), key, "gpt-5.5", start, refresh) + done <- result{protocols, time.Since(start)} + } + start := time.Now() + go lookup(first) + synctest.Wait() + time.Sleep(modelProtocolCatalogFirstLoadWait / 2) // Virtual time only. + go lookup(second) + synctest.Wait() + time.Sleep(modelProtocolCatalogFirstLoadWait / 2) + synctest.Wait() + + r1, r2 := <-first, <-second + require.Nil(t, r1.protocols) + require.Nil(t, r2.protocols) + require.Equal(t, modelProtocolCatalogFirstLoadWait, r1.elapsed) + require.Equal(t, modelProtocolCatalogFirstLoadWait/2, r2.elapsed, "a later caller waits only for the remaining shared window") + require.Equal(t, modelProtocolCatalogFirstLoadWait, time.Since(start)) + require.Nil(t, catalog.lookup(t.Context(), key, "gpt-5.5", time.Now(), refresh)) + require.Equal(t, modelProtocolCatalogFirstLoadWait, time.Since(start), "later calls fall back immediately while the same refresh continues") + require.EqualValues(t, 1, refreshes.Load()) + + close(release) + synctest.Wait() + require.Equal(t, models["gpt-5.5"], catalog.lookup(t.Context(), key, "gpt-5.5", time.Now(), refresh), "a late successful refresh still populates the cache") + }) +} + +func TestModelProtocolCatalogFirstLoadWaitsForRefresh(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const key = "https://catalog.example/v1/models" + models, err := parseModelProtocolCatalog([]byte(commandCodeModelsSample)) + require.NoError(t, err) + var loaded modelProtocolCatalog + release := make(chan struct{}) + result := make(chan []string, 1) + go func() { + result <- loaded.lookup(t.Context(), key, "gpt-5.5", time.Now(), func() { + <-release + loaded.store(key, models, nil, time.Now()) + }) + }() + synctest.Wait() + close(release) + synctest.Wait() + require.Equal(t, []string{APIProtocolChatCompletions, APIProtocolResponses}, <-result) + + var failing modelProtocolCatalog + var refreshes atomic.Int32 + refresh := func() { + refreshes.Add(1) + failing.store(key, nil, errors.New("boom"), time.Now()) + } + start := time.Now() + require.Nil(t, failing.lookup(t.Context(), key, "gpt-5.5", start, refresh)) + require.Nil(t, failing.lookup(t.Context(), key, "gpt-5.5", start.Add(time.Minute), refresh)) + require.Equal(t, start, time.Now(), "a failed first load and its backoff must not hold requests") + require.EqualValues(t, 1, refreshes.Load()) + }) +} + +func TestModelProtocolCatalogFirstLoadWaitRespondsToCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var catalog modelProtocolCatalog + const key = "https://catalog.example/v1/models" + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + release := make(chan struct{}) + models := map[string][]string{"gpt-5.5": {APIProtocolResponses}} + result := make(chan []string, 1) + start := time.Now() + go func() { + result <- catalog.lookup(ctx, key, "gpt-5.5", start, func() { + <-release + catalog.store(key, models, nil, time.Now()) + }) + }() + synctest.Wait() + cancel() + synctest.Wait() + require.Nil(t, <-result) + require.Equal(t, start, time.Now(), "cancellation releases the caller without waiting for the cold-start deadline") + + // The caller's cancellation must not cancel the background refresh for future requests. + close(release) + synctest.Wait() + require.Equal(t, models["gpt-5.5"], catalog.lookup(t.Context(), key, "gpt-5.5", time.Now(), nil)) + }) +} + +// 官方默认地址和自定义上游均按账号隔离;请求头与转发一致。 +func TestModelProtocolCatalogKeyAndHeaders(t *testing.T) { + official := commandCodeTestAccount(21) + base := official.GetCNProtocolBaseURL(APIProtocolChatCompletions) + url := buildOpenAIModelsURL(base) + require.Equal(t, "https://api.commandcode.ai/provider/v1/models", url) + require.Equal(t, url+"#account=21", modelProtocolCatalogKey(official, url)) + require.Equal(t, url+"#account=22", modelProtocolCatalogKey(commandCodeTestAccount(22), url)) + + headers := modelProtocolCatalogHeaders(official, url) + require.Equal(t, "Bearer user_test_key", headers.Get("Authorization")) + require.Equal(t, CodexCanonicalUserAgent(), headers.Get("User-Agent"), "official host gets the canonical UA like forwarding") + + custom := commandCodeTestAccount(23) + custom.Credentials["api_base_urls"] = map[string]any{APIProtocolChatCompletions: "https://relay.example/v1"} + custom.Credentials["header_override_enabled"] = true + custom.Credentials["header_overrides"] = map[string]any{"X-Tenant": "t-1"} + customBase := custom.GetCNProtocolBaseURL(APIProtocolChatCompletions) + customURL := buildOpenAIModelsURL(customBase) + require.Equal(t, customURL+"#account=23", modelProtocolCatalogKey(custom, customURL)) + headers = modelProtocolCatalogHeaders(custom, customURL) + require.Equal(t, "t-1", getHeaderRaw(headers, "x-tenant")) + require.Empty(t, headers.Get("User-Agent")) +} + +type modelCatalogAccountUpstream struct { + HTTPUpstream + mu sync.Mutex + requests []*http.Request + failureStatus int + failureErr error +} + +func (u *modelCatalogAccountUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + u.mu.Lock() + u.requests = append(u.requests, req) + u.mu.Unlock() + if req.Header.Get("Authorization") == "Bearer invalid-key" { + if u.failureErr != nil { + return nil, u.failureErr + } + return newJSONResponse(u.failureStatus, `{"error":"invalid credentials"}`), nil + } + endpoint := "/chat/completions" + if getHeaderRaw(req.Header, "x-tenant") == "responses-tenant" { + endpoint = "/responses" + } + return newJSONResponse(http.StatusOK, fmt.Sprintf(`{"data":[{"id":"vendor/model","supported_endpoints":[%q]}]}`, endpoint)), nil +} + +func TestModelProtocolCatalogOfficialAccountsAreIsolated(t *testing.T) { + for _, tc := range []struct { + name string + status int + err error + }{ + {name: "unauthorized", status: http.StatusUnauthorized}, + {name: "forbidden", status: http.StatusForbidden}, + {name: "transport", err: errors.New("upstream connection failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := &modelCatalogAccountUpstream{failureStatus: tc.status, failureErr: tc.err} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + id := time.Now().UnixNano() + bad, chat, responses := commandCodeTestAccount(id), commandCodeTestAccount(id+1), commandCodeTestAccount(id+2) + bad.Credentials["api_key"] = "invalid-key" + responses.Credentials["header_override_enabled"] = true + responses.Credentials["header_overrides"] = map[string]any{"X-Tenant": "responses-tenant"} + url := buildOpenAIModelsURL(bad.GetCNProtocolBaseURL(APIProtocolChatCompletions)) + t.Cleanup(func() { + upstreamModelProtocols.mu.Lock() + defer upstreamModelProtocols.mu.Unlock() + for _, account := range []*Account{bad, chat, responses} { + delete(upstreamModelProtocols.entries, modelProtocolCatalogKey(account, url)) + } + }) + + require.Nil(t, svc.modelCatalogProtocols(t.Context(), bad, "vendor/model")) + require.Equal(t, APIProtocolChatCompletions, svc.resolveUpstreamProtocolFor(t.Context(), chat, APIProtocolResponses, "vendor/model"), "a healthy account fetches its own catalog despite another account's backoff") + require.Equal(t, APIProtocolResponses, svc.resolveUpstreamProtocolFor(t.Context(), responses, APIProtocolChatCompletions, "vendor/model"), "successful catalogs stay isolated when headers select different tenants") + require.Equal(t, []string{APIProtocolChatCompletions}, svc.modelCatalogProtocols(t.Context(), chat, "vendor/model")) + require.Nil(t, svc.modelCatalogProtocols(t.Context(), bad, "vendor/model")) + upstream.mu.Lock() + defer upstream.mu.Unlock() + require.Len(t, upstream.requests, 3, "each healthy account fetches once; the failed account remains in its own backoff") + for _, req := range upstream.requests { + require.Equal(t, url, req.URL.String(), "all accounts use the same official model-list URL") + } + }) + } +} + +func TestCommandCodeModelProtocolSetUsesCatalog(t *testing.T) { + account := commandCodeTestAccount(11) + both := []string{APIProtocolChatCompletions, APIProtocolResponses} + + // 目录声明两种协议:入站同协议直通;Anthropic 入站走首选(内置规则没有命中时为 Chat Completions)。 + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(account, APIProtocolResponses, "deepseek/deepseek-v4-flash", both)) + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolChatCompletions, "deepseek/deepseek-v4-flash", both)) + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolAnthropic, "deepseek/deepseek-v4-flash", both)) + // GPT 的首选沿用内置规则(Responses)。 + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(account, APIProtocolAnthropic, "gpt-5.5", both)) + // 目录只声明 Chat Completions:Responses 入站也转换到 Chat Completions。 + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolResponses, "deepseek/deepseek-v4-flash-fast", []string{APIProtocolChatCompletions})) + require.Equal(t, APIProtocolAnthropic, resolveUpstreamProtocol(account, APIProtocolResponses, "claude-sonnet-4-6", []string{APIProtocolAnthropic})) + + // 账号规则命中时优先于目录;已配置但未命中时仍使用目录。 + account.Credentials["protocol_rules"] = []any{map[string]any{"pattern": "deepseek/*", "protocol": APIProtocolChatCompletions}} + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolResponses, "deepseek/deepseek-v4-flash", both)) + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(account, APIProtocolResponses, "zai-org/glm-5.3", both)) + + // 显式协议优先于一切。 + account.Credentials["api_protocol"] = APIProtocolChatCompletions + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(account, APIProtocolResponses, "zai-org/glm-5.3", both)) +} + +func TestProtocolRulesWithProtocolSets(t *testing.T) { + credentials := map[string]any{"protocol_rules": []any{ + map[string]any{"pattern": " GPT-* ", "protocol": APIProtocolResponses, "protocols": []any{APIProtocolChatCompletions, APIProtocolResponses, APIProtocolChatCompletions}}, + map[string]any{"pattern": "qwen*", "protocols": []any{APIProtocolAnthropic}}, + map[string]any{"pattern": "glm-*", "protocol": APIProtocolChatCompletions}, + }} + require.NoError(t, NormalizeProtocolRulesCredentials(credentials)) + require.Equal(t, []any{ + map[string]any{"pattern": "gpt-*", "protocol": APIProtocolResponses, "protocols": []any{APIProtocolResponses, APIProtocolChatCompletions}}, + map[string]any{"pattern": "qwen*", "protocol": APIProtocolAnthropic}, + map[string]any{"pattern": "glm-*", "protocol": APIProtocolChatCompletions}, + }, credentials["protocol_rules"]) + + bad := map[string]any{"protocol_rules": []any{map[string]any{"pattern": "gpt-*", "protocols": []any{"responses", "bogus"}}}} + require.Error(t, NormalizeProtocolRulesCredentials(bad)) + + // 单一协议规则(OpenCode)不受影响:与入站无关。 + openCode := &Account{Platform: PlatformOpenCodeGo, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "k"}} + for _, inbound := range []string{APIProtocolChatCompletions, APIProtocolResponses, APIProtocolAnthropic} { + require.Equal(t, APIProtocolResponses, resolveUpstreamProtocol(openCode, inbound, "gpt-5.5", nil), inbound) + require.Equal(t, APIProtocolChatCompletions, resolveUpstreamProtocol(openCode, inbound, "glm-5.3", nil), inbound) + } +} + +// 经真实入口验证:目录声明模型支持入站协议时原样直通,不做协议转换。 +func TestCommandCodeGatewayPassesThroughCatalogProtocols(t *testing.T) { + type observation struct { + url string + body []byte + } + forward := func(t *testing.T, ingress routingMatrixIngress, model string, catalog map[string][]string) observation { + t.Helper() + base := fmt.Sprintf("http://cc-%s-%d.example", strings.ReplaceAll(t.Name(), "/", "-"), time.Now().UnixNano()) + account := commandCodeTestAccount(12) + account.Credentials["api_base_urls"] = map[string]any{ + APIProtocolChatCompletions: base + "/provider/v1", + APIProtocolResponses: base + "/provider/v1", + APIProtocolAnthropic: base + "/provider", + } + url := buildOpenAIModelsURL(base + "/provider/v1") + key := modelProtocolCatalogKey(account, url) + if catalog != nil { + upstreamModelProtocols.store(key, catalog, nil, time.Now()) + } else { + // 目录不可用(退避中):不触发刷新,回落内置规则。 + upstreamModelProtocols.store(key, nil, errors.New("unavailable"), time.Now()) + } + upstream := &httpUpstreamRecorder{err: errors.New("stop after capture")} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + body := routingMatrixCase{ingress: ingress, model: model}.body() + _ = ingress.forward(svc, adaptiveProtocolTestContext(ingress.path, body), account, body) + require.NotEmpty(t, upstream.requests) + return observation{url: upstream.requests[len(upstream.requests)-1].URL.String(), body: upstream.lastBody} + } + ingresses := map[string]routingMatrixIngress{} + for _, ingress := range routingMatrixIngresses() { + ingresses[ingress.name] = ingress + } + catalog := map[string][]string{ + "deepseek/deepseek-v4-flash": {APIProtocolChatCompletions, APIProtocolResponses}, + "deepseek/deepseek-v4-flash-fast": {APIProtocolChatCompletions}, + "gpt-5.5": {APIProtocolChatCompletions, APIProtocolResponses}, + } + + obs := forward(t, ingresses["responses"], "deepseek/deepseek-v4-flash", catalog) + require.True(t, strings.HasSuffix(obs.url, "/provider/v1/responses"), obs.url) + require.True(t, gjson.GetBytes(obs.body, "input").Exists(), "Responses body passes through unconverted") + + obs = forward(t, ingresses["responses"], "deepseek/deepseek-v4-flash", nil) + require.True(t, strings.HasSuffix(obs.url, "/provider/v1/chat/completions"), obs.url) + require.True(t, gjson.GetBytes(obs.body, "messages").Exists()) + + obs = forward(t, ingresses["responses"], "deepseek/deepseek-v4-flash-fast", catalog) + require.True(t, strings.HasSuffix(obs.url, "/provider/v1/chat/completions"), obs.url) + + obs = forward(t, ingresses["chat"], "gpt-5.5", catalog) + require.True(t, strings.HasSuffix(obs.url, "/provider/v1/chat/completions"), obs.url) + require.True(t, gjson.GetBytes(obs.body, "messages").Exists(), "Chat body passes through unconverted") +} + +// 目录缺失时网关拉取目录,首个请求即按目录分流;自定义上游按账号各自拉取,并带上 +// 账号的请求头覆写。 +func TestCommandCodeGatewayFetchesModelCatalog(t *testing.T) { + base := fmt.Sprintf("http://cc-fetch-%d.example/provider/v1", time.Now().UnixNano()) + newAccount := func(id int64) *Account { + account := commandCodeTestAccount(id) + account.Credentials["api_base_urls"] = map[string]any{ + APIProtocolChatCompletions: base, + APIProtocolResponses: base, + APIProtocolAnthropic: strings.TrimSuffix(base, "/v1"), + } + account.Credentials["header_override_enabled"] = true + account.Credentials["header_overrides"] = map[string]any{"X-Tenant": fmt.Sprintf("t-%d", id)} + return account + } + upstream := &commandCodeAlphaUpstream{responses: map[string]commandCodeAlphaResponse{ + "/provider/v1/models": {status: 200, body: commandCodeModelsSample}, + }} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + both := []string{APIProtocolChatCompletions, APIProtocolResponses} + + first, second := newAccount(13), newAccount(14) + require.Equal(t, both, svc.modelCatalogProtocols(t.Context(), first, "deepseek/deepseek-v4-flash")) + require.Equal(t, both, svc.modelCatalogProtocols(t.Context(), first, "deepseek/deepseek-v4-flash")) + require.Equal(t, both, svc.modelCatalogProtocols(t.Context(), second, "deepseek/deepseek-v4-flash")) + + upstream.mu.Lock() + defer upstream.mu.Unlock() + require.Len(t, upstream.requests, 2, "one fetch per account on a custom upstream") + for i, account := range []*Account{first, second} { + req := upstream.requests[i] + require.Equal(t, "/provider/v1/models", req.URL.Path) + require.Equal(t, "Bearer user_test_key", req.Header.Get("Authorization")) + require.Equal(t, fmt.Sprintf("t-%d", account.ID), getHeaderRaw(req.Header, "x-tenant")) + } +} diff --git a/backend/internal/service/model_rate_limit.go b/backend/internal/service/model_rate_limit.go index 88d1452f9594..153e60939052 100644 --- a/backend/internal/service/model_rate_limit.go +++ b/backend/internal/service/model_rate_limit.go @@ -99,6 +99,10 @@ func (a *Account) modelRateLimitKeysForRequest(ctx context.Context, requestedMod if isAnthropicFableModel(modelKey) && modelKey != anthropicFableRateLimitKey { keys = append(keys, anthropicFableRateLimitKey) } + case PlatformCline: + if walletKey := clineWalletRateLimitKey(modelKey); walletKey != "" { + keys = append(keys, walletKey) + } } return keys } @@ -226,6 +230,17 @@ func (a *Account) hasActiveModelRateLimitReasonWithContext(ctx context.Context, return false } +// modelRateLimitReason 返回指定 scope 冷却记录中的原因;没有记录时为空。 +func (a *Account) modelRateLimitReason(scope string) string { + if a == nil || a.Extra == nil { + return "" + } + rawLimits, _ := a.Extra[modelRateLimitsKey].(map[string]any) + rawLimit, _ := rawLimits[scope].(map[string]any) + reason, _ := rawLimit["reason"].(string) + return reason +} + func setAccountModelRateLimitSnapshot(account *Account, scope string, resetAt time.Time, reason string, now time.Time) { if account == nil || strings.TrimSpace(scope) == "" { return diff --git a/backend/internal/service/openai_apikey_responses_probe.go b/backend/internal/service/openai_apikey_responses_probe.go index d942afab08e7..e7dc1cb201b8 100644 --- a/backend/internal/service/openai_apikey_responses_probe.go +++ b/backend/internal/service/openai_apikey_responses_probe.go @@ -130,8 +130,8 @@ func (s *AccountTestService) ProbeOpenAIAPIKeyResponsesSupport(ctx context.Conte if account.Type != AccountTypeAPIKey { return } - if account.IsCNProvider() { - // 国产 OpenAI 兼容上游默认仅支持 /v1/chat/completions。直接落标 false + if account.RoutesProtocolByInbound() { + // 按入站协议分流的供应商(国产厂商等)默认仅支持 /v1/chat/completions。直接落标 false // 走 Chat Completions 直转,跳过网络探测。 // 例外:deepseek / kimi 的固定 responses 和 adaptive 账号使用官方原生 // Responses 端点,落标 force_responses;其余协议显式重置为 auto,避免 diff --git a/backend/internal/service/openai_codex_model_metadata.go b/backend/internal/service/openai_codex_model_metadata.go index 7c94e48a634d..897f0c140e23 100644 --- a/backend/internal/service/openai_codex_model_metadata.go +++ b/backend/internal/service/openai_codex_model_metadata.go @@ -22,6 +22,9 @@ func applyCodexToolCapabilities(dst, src map[string]json.RawMessage, overwrite b if len(value) == 0 { continue } + if field == "service_tiers" && bytes.Equal(value, []byte("null")) { + value = []byte("[]") + } // These Codex fields are nullable booleans or strings, never arbitrary objects. if !bytes.Equal(value, []byte("null")) { if field == "service_tiers" { @@ -294,8 +297,11 @@ func intersectUpstreamModelMetadata(modelID string, candidates []UpstreamModelMe result.CodexToolCapabilities[field] = value } else if declared { fallback := json.RawMessage("null") - if field == "supports_search_tool" || field == "use_responses_lite" { + switch field { + case "supports_search_tool", "use_responses_lite": fallback = json.RawMessage("false") + case "service_tiers": + fallback = json.RawMessage("[]") } result.CodexToolCapabilities[field] = fallback } diff --git a/backend/internal/service/openai_codex_model_metadata_test.go b/backend/internal/service/openai_codex_model_metadata_test.go index 8016e6b1b9f1..1cf36807ed98 100644 --- a/backend/internal/service/openai_codex_model_metadata_test.go +++ b/backend/internal/service/openai_codex_model_metadata_test.go @@ -108,6 +108,41 @@ func TestAstraCodexToolCapabilitiesUseAccountScopeAndSharedDeclarations(t *testi require.Equal(t, "3000", model["comp_hash"]) } +func TestCodexToolCapabilitiesNeverAdvertiseNullServiceTiers(t *testing.T) { + newAccount := func(id int64, tiers string) Account { + account := Account{ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{ + "base_url": "https://relay.example/v1", "model_mapping": map[string]any{"gpt-6-sol": "gpt-6-sol"}, + }} + if tiers != "" { + account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{ + "gpt-6-sol": {CodexToolCapabilities: map[string]json.RawMessage{"service_tiers": json.RawMessage(tiers)}}, + }}) + } + return account + } + for _, tt := range []struct { + name string + accounts []Account + }{ + {"conflicting peer tiers", []Account{newAccount(1, `[{"id":"priority"}]`), newAccount(2, `[]`)}}, + {"undeclared peer tiers", []Account{newAccount(1, `[{"id":"priority"}]`), newAccount(2, "")}}, + {"explicit null tiers", []Account{newAccount(1, "null")}}, + } { + t.Run(tt.name, func(t *testing.T) { + body, err := buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"gpt-6-sol"}, tt.accounts, nil, nil, true) + require.NoError(t, err) + model := decodeCodexManifestModels(t, body)[0] + require.Equal(t, []any{}, model["service_tiers"]) + }) + } + + merged := intersectUpstreamModelMetadata("gpt-6-sol", []UpstreamModelMetadata{ + {CodexToolCapabilities: map[string]json.RawMessage{"service_tiers": json.RawMessage(`[{"id":"priority"}]`)}}, + {CodexToolCapabilities: map[string]json.RawMessage{"service_tiers": json.RawMessage(`[]`)}}, + }) + require.JSONEq(t, `[]`, string(merged.CodexToolCapabilities["service_tiers"])) +} + func TestAstraCodexToolCapabilitiesPreserveLiveNullAndFalse(t *testing.T) { account := &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{"base_url": "https://api.openai.com/v1"}} diff --git a/backend/internal/service/openai_codex_models_pinned.go b/backend/internal/service/openai_codex_models_pinned.go index b74b3eafdc45..16faefd0680f 100644 --- a/backend/internal/service/openai_codex_models_pinned.go +++ b/backend/internal/service/openai_codex_models_pinned.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "net/http" + "slices" "strings" "sync" "time" @@ -91,6 +92,13 @@ func mergeCodexModelsManifestBodies(bodies [][]byte) ([]byte, error) { } } + if len(bodies) == 1 { + _, original, err := modelCatalogEntries(bodies[0], "models") + if err == nil && slices.EqualFunc(original, mergedModels, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { + return bodies[0], nil + } + } + encodedModels, err := json.Marshal(mergedModels) if err != nil { return nil, fmt.Errorf("encode merged codex models: %w", err) @@ -139,6 +147,9 @@ func (s *OpenAIGatewayService) FetchPinnedCodexModelsManifest(ctx context.Contex if err != nil { return nil, nil, fmt.Errorf("merge pinned codex models manifests: %w", err) } + if len(results) == 1 && bytes.Equal(merged, results[0].response.Body) { + return results[0].response, results[0].account, nil + } return &OpenAIModelsResponse{Body: merged, ETag: codexModelsManifestBodyETag(merged)}, results[0].account, nil } diff --git a/backend/internal/service/openai_codex_models_pinned_test.go b/backend/internal/service/openai_codex_models_pinned_test.go index eee3100c7636..3c389402f47c 100644 --- a/backend/internal/service/openai_codex_models_pinned_test.go +++ b/backend/internal/service/openai_codex_models_pinned_test.go @@ -63,11 +63,11 @@ func TestMergeCodexModelsManifestBodiesHandlesSluglessEntries(t *testing.T) { } func TestMergeCodexModelsManifestBodiesSingleBody(t *testing.T) { - body := `{"models":[{"slug":"model-a"}]}` + body := `{ "unknown": true, "models": [ { "slug": "model-a" } ] }` merged, err := mergeCodexModelsManifestBodies([][]byte{[]byte(body)}) require.NoError(t, err) - require.JSONEq(t, body, string(merged)) + require.Equal(t, body, string(merged)) } func TestMergeCodexModelsManifestBodiesRejectsInvalidInput(t *testing.T) { @@ -80,3 +80,9 @@ func TestMergeCodexModelsManifestBodiesRejectsInvalidInput(t *testing.T) { _, err = mergeCodexModelsManifestBodies([][]byte{[]byte(`{}`), []byte(`{"models":{}`)}) require.Error(t, err, "models 非数组必须报错") } + +func TestMergeCodexModelsManifestBodiesSingleBodyDeduplicates(t *testing.T) { + merged, err := mergeCodexModelsManifestBodies([][]byte{[]byte(`{"models":[{"slug":"a"},{"slug":"a","extra":true}]}`)}) + require.NoError(t, err) + require.JSONEq(t, `{"models":[{"slug":"a"}]}`, string(merged)) +} diff --git a/backend/internal/service/openai_codex_transform.go b/backend/internal/service/openai_codex_transform.go index d5160649cd23..e172e8de6b86 100644 --- a/backend/internal/service/openai_codex_transform.go +++ b/backend/internal/service/openai_codex_transform.go @@ -95,6 +95,8 @@ type codexOAuthTransformOptions struct { SkipDefaultInstructions bool PreserveToolCallIDs bool OmitPromotedSystemMessagesFromInput bool + // ResponsesLite selects where an injected hosted tool is declared. + ResponsesLite bool } const ( @@ -352,6 +354,12 @@ func applyCodexOAuthTransformWithOptions(reqBody map[string]any, opts codexOAuth result.Modified = true } + // Runs after input filtering so it sees the final replayed history. The + // compact endpoint has its own wire shape and is left untouched. + if !opts.IsCompact && ensureOpenAIOAuthWebSearchToolForHistory(reqBody, opts.ResponsesLite) { + result.Modified = true + } + return result } diff --git a/backend/internal/service/openai_compat_host_observation_test.go b/backend/internal/service/openai_compat_host_observation_test.go index 7600176e283c..c2a1ef1dbada 100644 --- a/backend/internal/service/openai_compat_host_observation_test.go +++ b/backend/internal/service/openai_compat_host_observation_test.go @@ -177,32 +177,6 @@ func TestCompatHostObservations_ShadowSuccessNotifiesParent(t *testing.T) { } } -func TestCompatHostObservations_RecoveryLogsOnlyFinalSuccess(t *testing.T) { - store := &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}} - completed, err := sjson.Delete(reasoningRecoveryChatCompleted, "response.usage") - require.NoError(t, err) - failed := "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}}}\n\n" - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - reasoningRecoverySSEResponse(failed, http.StatusOK), - reasoningRecoverySSEResponse("data: "+completed+"\n\n", http.StatusOK), - }} - svc := newOpenAIRejectedFieldTestService(upstream) - svc.cache = store - account := newOpenAIRejectedFieldTestAccount() - body := reasoningRecoveryChatBody(t, false) - c, _ := reasoningRecoveryChatContext(t, body) - core, logs := observer.New(zap.WarnLevel) - c.Request = c.Request.WithContext(logger.IntoContext(c.Request.Context(), zap.New(core))) - before := observeCompatMissingUsage(t) - result, err := svc.ForwardAsChatCompletions(c.Request.Context(), c, account, body, "", "") - require.NoError(t, err) - require.NotNil(t, result) - require.Len(t, upstream.requests, 2, "only the rejected attempt and its bounded recovery may send") - require.Len(t, store.values, 1) - require.Equal(t, uint64(1), openAIMissingUsageSampler.total.Load()-before) - require.Len(t, logs.FilterMessage("openai_usage.success_missing_usage").All(), 1) -} - func TestCompatHostObservations_FailedAttemptsEmitNoSuccessSignals(t *testing.T) { for _, protocol := range []string{"chat", "messages"} { for _, stream := range []bool{false, true} { @@ -231,7 +205,7 @@ func TestCompatHostObservations_IncompleteDoesNotCountAsCompleted(t *testing.T) before := observeCompatMissingUsage(t) c, body, logs := compatObservationContext(t, protocol, stream) payload := "data: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_incomplete\",\"model\":\"gpt-5.4\",\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"output\":[{\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"partial fixture\"}]}]}}\n\n" - upstream := &httpUpstreamRecorder{resp: reasoningRecoverySSEResponse(payload, http.StatusOK)} + upstream := &httpUpstreamRecorder{resp: &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(payload))}} svc := newOpenAIRejectedFieldTestService(upstream) result, err := forwardCompatObservation(svc, protocol, c, newOpenAIRejectedFieldTestAccount(), body) require.NoError(t, err, "the existing bounded-output response contract must remain unchanged") diff --git a/backend/internal/service/openai_continuation_diagnostic.go b/backend/internal/service/openai_continuation_diagnostic.go index 772f1be5753e..22951dd1e942 100644 --- a/backend/internal/service/openai_continuation_diagnostic.go +++ b/backend/internal/service/openai_continuation_diagnostic.go @@ -1,6 +1,7 @@ package service import ( + "bytes" "crypto/sha256" "encoding/binary" "encoding/hex" @@ -119,6 +120,19 @@ type openAIContinuationHistory struct { NestedEncrypted int `json:"nested_encrypted"` } +type openAIContinuationWireSnapshot struct { + request *http.Request + body []byte +} + +func freezeOpenAIContinuationWire(c *gin.Context, request *http.Request, body []byte) { + if c != nil { + c.Set("openai_continuation_wire", openAIContinuationWireSnapshot{request: request, body: bytes.Clone(body)}) + } + // Retry decisions belong to the gateway rather than transparent POST replay. + request.GetBody = nil +} + func buildOpenAIContinuationDiagnostic(c *gin.Context, incomingBody []byte, upstreamReq *http.Request, preparedBody, upstreamError []byte, classification string) *OpenAIContinuationDiagnostic { var incomingHeaders, wireHeaders http.Header if c != nil && c.Request != nil { @@ -127,16 +141,13 @@ func buildOpenAIContinuationDiagnostic(c *gin.Context, incomingBody []byte, upst if upstreamReq != nil { wireHeaders = upstreamReq.Header } - recovery := openAIReasoningRecoveryStateFromContext(c) - var wireBody []byte - var wireSource string - var limited bool - if recovery != nil && recovery.diagnosticRequest == upstreamReq && recovery.wire != nil { - // PrepareRequest clears GetBody to prohibit transparent POST replay. - // Its immutable send-boundary snapshot is exact, not a prepared fallback. - wireBody, wireSource, limited = recovery.wire, "frozen_request", len(recovery.wire) > openAIContinuationDiagnosticBodyLimit - } else { - wireBody, wireSource, limited = continuationDiagnosticWireBody(upstreamReq, preparedBody) + wireBody, wireSource, limited := continuationDiagnosticWireBody(upstreamReq, preparedBody) + if c != nil { + if value, exists := c.Get("openai_continuation_wire"); exists { + if snapshot, ok := value.(openAIContinuationWireSnapshot); ok && snapshot.request == upstreamReq { + wireBody, wireSource, limited = snapshot.body, "frozen_request", len(snapshot.body) > openAIContinuationDiagnosticBodyLimit + } + } } diagnostic := &OpenAIContinuationDiagnostic{ Version: 1, Classification: continuationDiagnosticClassification(classification), @@ -146,9 +157,6 @@ func buildOpenAIContinuationDiagnostic(c *gin.Context, incomingBody []byte, upst Incoming: continuationDiagnosticRequest(incomingBody, incomingHeaders, "forwarding_entry"), Wire: continuationDiagnosticRequest(wireBody, wireHeaders, wireSource), } - if recovery != nil { - diagnostic.Recovery = recovery.continuationDiagnosticRecovery(upstreamError) - } diagnostic.Wire.InspectionLimited = diagnostic.Wire.InspectionLimited || limited if limited && upstreamReq != nil && upstreamReq.ContentLength > int64(diagnostic.Wire.BodyBytes) { // LimitReader bounds inspection, not the reported size when the built diff --git a/backend/internal/service/openai_encrypted_content_lineage_legacy_test.go b/backend/internal/service/openai_encrypted_content_lineage.go similarity index 67% rename from backend/internal/service/openai_encrypted_content_lineage_legacy_test.go rename to backend/internal/service/openai_encrypted_content_lineage.go index 81d767548cb0..dbc360351d0d 100644 --- a/backend/internal/service/openai_encrypted_content_lineage_legacy_test.go +++ b/backend/internal/service/openai_encrypted_content_lineage.go @@ -1,6 +1,3 @@ -// Historical lineage helpers are retained only as regression fixtures. New -// requests must preserve their original state even if an old marker exists. -// No state-erasing lineage operation is part of the production forwarding path. package service import ( @@ -9,6 +6,7 @@ import ( "encoding/json" "strings" + "github.com/gin-gonic/gin" "github.com/tidwall/gjson" ) @@ -20,6 +18,12 @@ import ( // (OpenAIWSStateStore,带 TTL 与容量自保护);后续请求进场时仅剥离摘要命中 // 的项,新生成的密文摘要不同,不会被误删。 +const openAIWSFallbackReasonInvalidEncryptedContent = "invalid_encrypted_content" + +// openAIWSIngressSessionHashContextKey 在 gin context 中携带 ingress 会话哈希, +// 供 HTTP bridge turn 内的 lineage 记录复用同一会话键。 +const openAIWSIngressSessionHashContextKey = "openai_ws_ingress_session_hash" + func openAIEncryptedContentDigest(encrypted string) string { sum := sha256.Sum256([]byte(encrypted)) return hex.EncodeToString(sum[:]) @@ -256,3 +260,94 @@ func stripOpenAIInvalidEncryptedContentRaw(payload []byte, invalid map[string]st } return rebuilt, stripped, nil } + +// markOpenAIWSInvalidEncryptedContentLineage 把本次被上游拒绝的密文摘要写入 +// 会话 lineage。digests 须在剥离前收集。 +func (s *OpenAIGatewayService) markOpenAIWSInvalidEncryptedContentLineage(groupID int64, sessionHash string, digests []string) { + if s == nil || len(digests) == 0 || strings.TrimSpace(sessionHash) == "" { + return + } + stateStore := s.getOpenAIWSStateStore() + if stateStore == nil { + return + } + stateStore.MarkSessionInvalidEncryptedContent(groupID, sessionHash, digests, s.openAIWSSessionStickyTTL()) +} + +// sessionInvalidEncryptedContentDigests 返回会话已知失效密文摘要;全局无记录 +// 时(常态)零成本返回 nil。 +func (s *OpenAIGatewayService) sessionInvalidEncryptedContentDigests(groupID int64, sessionHash string) map[string]struct{} { + if s == nil || strings.TrimSpace(sessionHash) == "" { + return nil + } + stateStore := s.getOpenAIWSStateStore() + if stateStore == nil || !stateStore.HasAnySessionInvalidEncryptedContent() { + return nil + } + return stateStore.GetSessionInvalidEncryptedContentDigests(groupID, sessionHash) +} + +// openAIWSLineageSessionHashFromContext 取 lineage 会话键:优先 ingress 循环 +// 写入的会话哈希(与读取侧同键),否则按请求体派生。 +func (s *OpenAIGatewayService) openAIWSLineageSessionHashFromContext(c *gin.Context, body []byte) string { + if c != nil { + if fromCtx := strings.TrimSpace(c.GetString(openAIWSIngressSessionHashContextKey)); fromCtx != "" { + return fromCtx + } + } + return s.GenerateSessionHash(c, body) +} + +// markOpenAIWSInvalidEncryptedContentLineageFromPayload 在上游以 +// invalid_encrypted_content 拒绝 payload 时记录其密文摘要并输出观测日志。 +func (s *OpenAIGatewayService) markOpenAIWSInvalidEncryptedContentLineageFromPayload( + c *gin.Context, + payload []byte, + logKey string, + accountID int64, + turn int, +) { + digests := collectOpenAIEncryptedContentDigestsRaw(payload) + if len(digests) == 0 { + return + } + s.markOpenAIWSInvalidEncryptedContentLineage( + getOpenAIGroupIDFromContext(c), + s.openAIWSLineageSessionHashFromContext(c, payload), + digests, + ) + logOpenAIWSModeInfo("%s account_id=%d turn=%d digests=%d", logKey, accountID, turn, len(digests)) +} + +// stripSessionInvalidEncryptedContentLogged 对 payload 执行会话失效密文剥离并 +// 输出观测日志(logKey / logKey+"_skip"),返回(可能已替换的)payload 与剥离 +// 项数;未命中或剥离失败时原样返回。 +func (s *OpenAIGatewayService) stripSessionInvalidEncryptedContentLogged( + payload []byte, + invalid map[string]struct{}, + logKey string, + accountID int64, + turn int, +) ([]byte, int) { + strippedPayload, strippedCount, stripErr := stripOpenAIInvalidEncryptedContentRaw(payload, invalid) + if stripErr != nil { + logOpenAIWSModeInfo( + "%s_skip account_id=%d turn=%d reason=strip_error cause=%s", + logKey, + accountID, + turn, + truncateOpenAIWSLogValue(stripErr.Error(), openAIWSLogValueMaxLen), + ) + return payload, 0 + } + if strippedCount > 0 { + logOpenAIWSModeInfo( + "%s account_id=%d turn=%d stripped_items=%d", + logKey, + accountID, + turn, + strippedCount, + ) + } + return strippedPayload, strippedCount +} diff --git a/backend/internal/service/openai_gateway_chat_completions.go b/backend/internal/service/openai_gateway_chat_completions.go index 199b7b4d176c..667f7e81dbcd 100644 --- a/backend/internal/service/openai_gateway_chat_completions.go +++ b/backend/internal/service/openai_gateway_chat_completions.go @@ -2,7 +2,6 @@ package service import ( "bufio" - "bytes" "context" "encoding/json" "errors" @@ -74,9 +73,7 @@ func (s *OpenAIGatewayService) forwardAsChatCompletions( ) (*OpenAIForwardResult, error) { // A scheduler may reuse gin.Context for another account or protocol. // Never leave an earlier recovery state attached to a fallback route. - if c != nil { - c.Set(openAIReasoningRecoveryContextKey, (*openAIReasoningRecoveryState)(nil)) - } + rememberOpenCodeInboundBody(c, body) beginUpstreamResponseModelObservation(c) if account != nil && account.IsOpenAI() { @@ -125,85 +122,43 @@ func (s *OpenAIGatewayService) forwardAsChatCompletions( } // Cursor compatibility: some clients send a Responses-shaped body to the - // /v1/chat/completions URL. Detect it before adaptive routing so adaptive + // /v1/chat/completions URL. Detect it before protocol routing so adaptive // accounts never forward the body unchanged to a Chat Completions endpoint. isResponsesShape := !gjson.GetBytes(body, "messages").Exists() && gjson.GetBytes(body, "input").Exists() - - // OpenCode Go:按模型原生协议分流(与 inbound 协议正交)。 - // 规则未命中一律兜底 Chat Completions,只有显式 Responses 才走下方转换链。 - if account.IsOpenCodeGo() { - mapped := resolveOpenCodeGoMappedModel(account, body, defaultMappedModel) - proto := openCodeGoNativeProtocol(account, mapped) - if proto != APIProtocolResponses { - if isResponsesShape { - if proto == APIProtocolAnthropic { - return s.forwardResponsesViaNativeAnthropic(ctx, c, account, body, "") - } - var responsesReq apicompat.ResponsesRequest - if err := json.Unmarshal(body, &responsesReq); err != nil { - return nil, fmt.Errorf("parse responses-shaped chat completions request: %w", err) - } - chatReq, err := apicompat.ResponsesToChatCompletionsRequestWithOptions( - &responsesReq, - &apicompat.ResponsesToChatOptions{ReasoningContentByID: s.reasoningContentByID}, - ) - if err != nil { - return nil, fmt.Errorf("convert responses-shaped chat completions request: %w", err) - } - chatBody, err := json.Marshal(chatReq) - if err != nil { - return nil, fmt.Errorf("marshal converted chat completions request: %w", err) - } - return s.forwardAsRawChatCompletions(ctx, c, account, chatBody, defaultMappedModel) - } - if proto == APIProtocolAnthropic { - return s.forwardChatCompletionsViaNativeAnthropic(ctx, c, account, body, defaultMappedModel) - } - return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel) - } + inbound := APIProtocolChatCompletions + if isResponsesShape { + inbound = APIProtocolResponses } + // 按模型分流与 adaptive 账号按请求体形状处理 Responses 形状入站;其余账号 + // 维持原样把请求体交给所选路径。 + convertResponsesShape := isResponsesShape && (account.routesByModel() || account.IsAdaptiveAPIProtocol()) - // 自适应账号的标准 Chat Completions 入站使用供应商原生 CC 端点。 - // Responses 形状下,DeepSeek / Kimi 继续走下方原生 Responses 链;GLM - // 没有 Responses 端点,先转换成 Chat Completions 再直转。 - if account.IsAdaptiveAPIProtocol() && !account.IsOpenCodeGo() { - if !isResponsesShape { - return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel) + // 上游协议统一由 resolveUpstreamProtocol 判定(按模型分流时带上游模型目录)。Anthropic 分流必须先于 + // ShouldUseResponsesAPI:Anthropic 协议账号经 probe 落标 + // openai_responses_supported=false,否则会命中 CC 直转。 + routingModel := upstreamRoutingModel(account, body, defaultMappedModel) + if account.IsOpenCodeGo() && IsOpenCodeUnsupportedModel(routingModel) { + return nil, writeOpenCodeUnsupportedModelError(c, false, routingModel) + } + switch s.resolveUpstreamProtocolFor(ctx, account, inbound, routingModel) { + case APIProtocolAnthropic: + if convertResponsesShape { + return s.forwardResponsesViaNativeAnthropic(ctx, c, account, body, "") } - if !account.SupportsNativeCNResponses() { - var responsesReq apicompat.ResponsesRequest - if err := json.Unmarshal(body, &responsesReq); err != nil { - return nil, fmt.Errorf("parse responses-shaped chat completions request: %w", err) - } - chatReq, err := apicompat.ResponsesToChatCompletionsRequestWithOptions( - &responsesReq, - &apicompat.ResponsesToChatOptions{ReasoningContentByID: s.reasoningContentByID}, - ) - if err != nil { - return nil, fmt.Errorf("convert responses-shaped chat completions request: %w", err) - } - chatBody, err := json.Marshal(chatReq) + // CC 入站经 CC→Responses→Anthropic 转换链直通供应商原生 Anthropic 端点。 + return s.forwardChatCompletionsViaNativeAnthropic(ctx, c, account, body, defaultMappedModel) + case APIProtocolChatCompletions: + if convertResponsesShape { + chatBody, err := s.convertResponsesShapedChatBody(body) if err != nil { - return nil, fmt.Errorf("marshal converted chat completions request: %w", err) + return nil, err } return s.forwardAsRawChatCompletions(ctx, c, account, chatBody, defaultMappedModel) } - // DeepSeek / Kimi 原生 Responses 请求继续走下方 Responses→Chat 回程转换。 - } - - // 入口分流(国产供应商 Anthropic 协议):上游为供应商原生 Anthropic 端点, - // CC 入站请求经 CC→Responses→Anthropic 转换链直通该端点。必须先于 - // ShouldUseResponsesAPI 分流:该类账号经 probe 落标 - // openai_responses_supported=false,会先命中下方的 CC 直转分支。 - if account.IsAnthropicProtocol() { - return s.forwardChatCompletionsViaNativeAnthropic(ctx, c, account, body, defaultMappedModel) - } - - // 固定 chat_completions 的 CN 账号,以及强制或已探测确认不支持 Responses - // 的其他 APIKey 账号,均走 CC 直转。 - if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel) } + // 上游为 Responses(含 adaptive 下 Responses 形状入站且供应商有原生 + // Responses 端点):走下方 Chat Completions ↔ Responses 转换链。 SetActualOpenAIUpstreamEndpoint(c, openAIResponsesUpstreamEndpoint) startTime := time.Now() @@ -392,126 +347,93 @@ func (s *OpenAIGatewayService) forwardAsChatCompletions( if account.ProxyID != nil && account.Proxy != nil { proxyURL = account.Proxy.URL() } - recovery := s.newOpenAIReasoningRecoveryState(ctx, c, account, token) - defer recovery.Close() - var upstreamReq *http.Request var resp *http.Response var result *OpenAIForwardResult var handleErr error var wireBody []byte - for { - upstreamCtx, releaseUpstreamCtx := detachUpstreamContext(ctx) - upstreamReq, err = s.buildUpstreamRequest(upstreamCtx, c, account, responsesBody, token, true, upstreamPromptCacheKey, false) - releaseUpstreamCtx() - if err != nil { - return nil, fmt.Errorf("build upstream request: %w", err) - } - // A stream attempt cancels its own context before closing its body. That - // context is derived per attempt, so it never cancels the single - // same-source reasoning recovery; PrepareRequest reapplies client - // cancellation and its deadline to that recovery attempt. - cancelUpstream := func() {} - if clientStream { - var attemptCtx context.Context - attemptCtx, cancelUpstream = context.WithCancel(upstreamReq.Context()) - upstreamReq = upstreamReq.WithContext(attemptCtx) - } - upstreamReq, responsesBody, wireBody, err = prepareReasoningRecoveryRequest(recovery, upstreamReq, responsesBody, proxyURL) - if err != nil { - cancelUpstream() - return nil, recovery.StopError(err) - } - finalCacheKey := finalWirePromptCacheKey(wireBody, upstreamPromptCacheKey) - if finalCacheKey != "" { - sessionKey := finalCacheKey - if !compatPromptCacheTenantIsolated { - sessionKey = isolateOpenAIUpstreamSessionID(getAPIKeyIDFromContext(c), codexAccountIdentitySource(c, account), finalCacheKey) - } - if account.UsesOpenAICodexProtocol() { - fillCodexSessionIdentityHeaders(upstreamReq.Header, generateSessionUUID(sessionKey)) - } else { - upstreamReq.Header.Set("session_id", generateSessionUUID(sessionKey)) - } - } - resp, err = s.doOpenAICodexUpstream(upstreamReq, account, proxyURL, upstreamModel) - if err != nil { - cancelUpstream() - if recovery.RecoveryAttempt() { - return nil, recovery.StopError(err) - } - return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, false) - } - if clientStream { - // Stream/error handlers can close the body before this outer loop - // regains control. Attach cancellation to Close itself so every - // cleanup path unblocks the reader before waiting for it to stop. - resp.Body = &openAIRequestContextReadCloser{ReadCloser: resp.Body, cleanup: cancelUpstream} - } - closeUpstreamResponse := func() { - cancelUpstream() - _ = resp.Body.Close() - } - if resp.StatusCode >= 400 { - respBody, upstreamMsg := s.readOpenAIUpstreamError(resp, c) - if repaired, repair := recovery.TryRepairUnfoundItemIDs(resp.StatusCode, resp.Header, respBody); repair { - closeUpstreamResponse() - responsesBody, err = projectReasoningRecoveryEdits(responsesBody, wireBody, repaired) - if err != nil { - return nil, err - } - continue - } - if retryBody, retry := recovery.TryRecover(resp.StatusCode, resp.Header, respBody, false); retry { - closeUpstreamResponse() - responsesBody, err = projectReasoningRecoveryEdits(responsesBody, wireBody, retryBody) - if err != nil { - return nil, err - } - continue - } - if recovery.RecoveryAttempt() { - closeUpstreamResponse() - return nil, recovery.StopError(errors.New("upstream rejected reasoning recovery")) - } - defer closeUpstreamResponse() - if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) { - expectedTaskID := account.GetCredential("task_id") - if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil { - return nil, fmt.Errorf("agent identity task recovery failed: %w", err) - } - return s.forwardAsChatCompletions(markAgentIdentityTaskRecoveryTried(ctx), c, account, body, retryPromptCacheKey, defaultMappedModel, compatPromptCacheTenantIsolated) - } - if account.Type == AccountTypeAPIKey && - openai_compat.ResolveResponsesSupport(account.Extra) == openai_compat.ResponsesSupportUnknown && - !isResponsesEndpointSupportedByStatus(resp.StatusCode) { - logger.L().Info("openai chat_completions: /responses unsupported, falling back to raw chat completions", - zap.Int64("account_id", account.ID), - zap.Int("upstream_status", resp.StatusCode), - zap.String("upstream_message", upstreamMsg), - ) - return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel) - } - if foErr := s.failoverOpenAIUpstreamHTTPError(ctx, c, account, resp, respBody, upstreamModel); foErr != nil { - return nil, foErr - } - return s.handleChatCompletionsErrorResponse(resp, c, account, billingModel) + + upstreamCtx, releaseUpstreamCtx := detachUpstreamContext(ctx) + upstreamReq, err := s.buildUpstreamRequest(upstreamCtx, c, account, responsesBody, token, true, upstreamPromptCacheKey, false) + releaseUpstreamCtx() + if err != nil { + return nil, fmt.Errorf("build upstream request: %w", err) + } + // A stream attempt cancels its own context before closing its body. That + // context is derived per attempt, so it never cancels the single + // same-source reasoning recovery; PrepareRequest reapplies client + // cancellation and its deadline to that recovery attempt. + cancelUpstream := func() {} + if clientStream { + var attemptCtx context.Context + attemptCtx, cancelUpstream = context.WithCancel(upstreamReq.Context()) + upstreamReq = upstreamReq.WithContext(attemptCtx) + } + wireBody, err = finalWireRequestBody(upstreamReq, responsesBody) + if err != nil { + cancelUpstream() + return nil, err + } + finalCacheKey := finalWirePromptCacheKey(wireBody, upstreamPromptCacheKey) + if finalCacheKey != "" { + sessionKey := finalCacheKey + if !compatPromptCacheTenantIsolated { + sessionKey = isolateOpenAIUpstreamSessionID(getAPIKeyIDFromContext(c), codexAccountIdentitySource(c, account), finalCacheKey) } - if clientStream { - result, handleErr = s.handleChatStreamingResponse(resp, c, account, originalModel, billingModel, upstreamModel, startTime, len(body)) + if account.UsesOpenAICodexProtocol() { + fillCodexSessionIdentityHeaders(upstreamReq.Header, generateSessionUUID(sessionKey)) } else { - result, handleErr = s.handleChatBufferedStreamingResponse(resp, c, account, originalModel, billingModel, upstreamModel, startTime) + upstreamReq.Header.Set("session_id", generateSessionUUID(sessionKey)) } - closeUpstreamResponse() - if retryBody, retry := recovery.TryRecoverError(handleErr); retry { - responsesBody, err = projectReasoningRecoveryEdits(responsesBody, wireBody, retryBody) - if err != nil { - return nil, err + } + resp, err = s.doOpenAICodexUpstream(upstreamReq, account, proxyURL, upstreamModel) + if err != nil { + cancelUpstream() + + return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, false) + } + if clientStream { + // Stream/error handlers can close the body before this outer loop + // regains control. Attach cancellation to Close itself so every + // cleanup path unblocks the reader before waiting for it to stop. + resp.Body = &openAIRequestContextReadCloser{ReadCloser: resp.Body, cleanup: cancelUpstream} + } + closeUpstreamResponse := func() { + cancelUpstream() + _ = resp.Body.Close() + } + if resp.StatusCode >= 400 { + respBody, upstreamMsg := s.readOpenAIUpstreamError(resp, c) + + defer closeUpstreamResponse() + if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) { + expectedTaskID := account.GetCredential("task_id") + if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil { + return nil, fmt.Errorf("agent identity task recovery failed: %w", err) } - continue + return s.forwardAsChatCompletions(markAgentIdentityTaskRecoveryTried(ctx), c, account, body, retryPromptCacheKey, defaultMappedModel, compatPromptCacheTenantIsolated) + } + if account.Type == AccountTypeAPIKey && + openai_compat.ResolveResponsesSupport(account.Extra) == openai_compat.ResponsesSupportUnknown && + !isResponsesEndpointSupportedByStatus(resp.StatusCode) { + logger.L().Info("openai chat_completions: /responses unsupported, falling back to raw chat completions", + zap.Int64("account_id", account.ID), + zap.Int("upstream_status", resp.StatusCode), + zap.String("upstream_message", upstreamMsg), + ) + return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel) + } + if foErr := s.failoverOpenAIUpstreamHTTPError(ctx, c, account, resp, respBody, upstreamModel); foErr != nil { + return nil, foErr } - handleErr = recovery.StopError(handleErr) - break + return s.handleChatCompletionsErrorResponse(resp, c, account, billingModel) + } + if clientStream { + result, handleErr = s.handleChatStreamingResponse(resp, c, account, originalModel, billingModel, upstreamModel, startTime, len(body)) + } else { + result, handleErr = s.handleChatBufferedStreamingResponse(resp, c, account, originalModel, billingModel, upstreamModel, startTime) } + closeUpstreamResponse() + stampOpenAIResponsesUpstreamEndpoint(c, result) // cyber_policy:标记已设、error 已按 Chat Completions 格式发给客户端。丢弃 result、 @@ -633,9 +555,7 @@ func (s *OpenAIGatewayService) handleChatBufferedStreamingResponse( if len(payload) == 0 { payload, _ = json.Marshal(gin.H{"type": "response.failed", "response": finalResponse}) } - if signal := openAIReasoningRecoverySignal(c, payload, false); signal != nil { - return nil, signal - } + // cyber_policy 致命不可重试:不 failover,以 Chat Completions 错误格式回写(F4), // 标记供 handler 事后写风控/邮件/tokens=0 用量行。 if hit, code, msg := detectOpenAICyberPolicy(payload); hit { @@ -654,9 +574,7 @@ func (s *OpenAIGatewayService) handleChatBufferedStreamingResponse( writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", clientMsg) return nil, fmt.Errorf("openai cyber_policy: %s", msg) } - if recovery := openAIReasoningRecoveryStateFromContext(c); recovery != nil && recovery.RecoveryAttempt() { - return nil, recovery.StopError(errors.New("upstream rejected reasoning recovery")) - } + message := openAICompatFailedResponseMessage(finalResponse) if openAIStreamFailedEventShouldFailoverForAccount(account, payload, message) { return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header) @@ -817,7 +735,6 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse( refusalDetector := newOpenAIChatSilentRefusalDetector(requestBodyLen) var streamFailoverErr *UpstreamFailoverError var streamNonFailoverErr error - var pendingReasoningRejection []byte // Grok chat bridge reuses Responses SSE; count native search tools for surcharge. searchCount := 0 streamSearchSeen := make(map[string]struct{}) @@ -865,39 +782,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse( } processDataLine := func(payload string) bool { - rawPayloadBytes := []byte(payload) - observeOpenAIReasoningAttemptUsage(c, rawPayloadBytes) - rawEventType := strings.TrimSpace(gjson.GetBytes(rawPayloadBytes, "type").String()) - if isOpenAICompatResponsesTerminalEvent(rawEventType) { - pendingReasoningRejection = nil - } - if signal := openAIHTTPReasoningRejectionBeforeOutput(c, rawPayloadBytes, semanticOutputCommitted || clientDisconnected); signal != nil { - if rawEventType == "error" { - // The authoritative terminal can supersede this preliminary - // rejection and may carry the first attempt's actual usage. - pendingReasoningRejection = bytes.Clone(rawPayloadBytes) - return false - } - if parsed, ok := extractOpenAIUsageFromJSONBytes(rawPayloadBytes); ok { - usage = parsed - } - streamNonFailoverErr = signal - return true - } - if rawEventType == "response.failed" || rawEventType == "error" { - recovery := openAIReasoningRecoveryStateFromContext(c) - if recovery == nil || !recovery.RecoveryAttempt() { - if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover( - c.Request.Context(), c, account, resp.StatusCode, resp.Header, rawPayloadBytes, - ); ok { - if parsedUsage, parsed := extractOpenAIUsageFromJSONBytes(rawPayloadBytes); parsed { - usage = parsedUsage - } - streamFailoverErr = failoverErr - return true - } - } - } + if firstChunk { firstChunk = false ms := int(time.Since(startTime).Milliseconds()) @@ -964,10 +849,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse( } return true } - if recovery := openAIReasoningRecoveryStateFromContext(c); recovery != nil && recovery.RecoveryAttempt() { - streamNonFailoverErr = recovery.StopError(errors.New("upstream rejected reasoning recovery")) - return true - } + shouldFailover := openAIStreamFailedEventShouldFailoverForAccount(account, payloadBytes, message) if isBareErrorEvent { shouldFailover = openAIStreamErrorEventShouldFailoverForAccount(account, payloadBytes, message) @@ -1178,11 +1060,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse( } } missingTerminalErr := func() (*OpenAIForwardResult, error) { - if len(pendingReasoningRejection) > 0 { - if signal := openAIHTTPReasoningRejectionBeforeOutput(c, pendingReasoningRejection, semanticOutputCommitted || clientDisconnected); signal != nil { - return resultWithUsage(), signal - } - } + return resultWithUsage(), fmt.Errorf("stream usage incomplete: missing terminal event") } processFrame := func(frame openAICompatSSEFrame) bool { diff --git a/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go b/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go index 383c7a6f44c6..fe78a2d2116c 100644 --- a/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go +++ b/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go @@ -241,7 +241,7 @@ func (s *OpenAIGatewayService) handleCCBufferedFromNativeAnthropic( } if event.Type == "content_block_delta" && event.Delta != nil && finalResp != nil && event.Index != nil { idx := *event.Index - if idx < len(finalResp.Content) { + if idx >= 0 && idx < len(finalResp.Content) { switch event.Delta.Type { case "text_delta": finalResp.Content[idx].Text += event.Delta.Text diff --git a/backend/internal/service/openai_gateway_chat_completions_cancellation_test.go b/backend/internal/service/openai_gateway_chat_completions_cancellation_test.go index 87c3e40b4658..6f987962c43f 100644 --- a/backend/internal/service/openai_gateway_chat_completions_cancellation_test.go +++ b/backend/internal/service/openai_gateway_chat_completions_cancellation_test.go @@ -16,8 +16,6 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" ) type contextBoundBlockingReadCloser struct { @@ -153,74 +151,3 @@ func (u *contextBoundChatRecoveryUpstream) Do(req *http.Request, proxyURL string func (u *contextBoundChatRecoveryUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, accountConcurrency int, _ *tlsfingerprint.Profile) (*http.Response, error) { return u.Do(req, proxyURL, accountID, accountConcurrency) } - -func TestForwardAsChatCompletions_StreamCancellationIsPerReasoningAttempt(t *testing.T) { - gin.SetMode(gin.TestMode) - for _, outcome := range []string{"completed", "client_canceled"} { - t.Run(outcome, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - body := []byte(strings.Replace(reasoningRecoveryFixture, `"store":false`, `"stream":true,"store":false`, 1)) - c, recorder := reasoningRecoveryChatContext(t, body) - c.Request = c.Request.WithContext(ctx) - failed := newContextBoundBlockingReadCloser([]byte(`data: {"type":"response.failed","response":{"id":"resp_rejected","status":"failed","error":{"code":"invalid_encrypted_content","param":"input[1].encrypted_content"}}}` + "\n\n")) - completed := newContextBoundBlockingReadCloser([]byte("data: " + reasoningRecoveryChatCompleted + "\n\n")) - t.Cleanup(failed.forceUnblock) - t.Cleanup(completed.forceUnblock) - upstream := &contextBoundChatRecoveryUpstream{httpUpstreamRecorder: httpUpstreamRecorder{responses: []*http.Response{ - {StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: failed}, - {StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: completed}, - }}} - if outcome == "client_canceled" { - upstream.cancelRecovery = cancel - } - cfg := rawChatCompletionsTestConfig() - cfg.Gateway.StreamKeepaliveInterval = 1 - svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} - type forwardResult struct { - result *OpenAIForwardResult - err error - } - resultCh := make(chan forwardResult, 1) - go func() { - result, err := svc.ForwardAsChatCompletions(ctx, c, newOpenAIRejectedFieldTestAccount(), body, "", "") - resultCh <- forwardResult{result: result, err: err} - }() - - var got forwardResult - select { - case got = <-resultCh: - case <-time.After(2 * time.Second): - t.Fatal("stream attempt did not cancel before closing or canceled its reasoning recovery") - } - require.Len(t, upstream.requests, 2, "the rejected attempt permits exactly one same-source recovery") - require.ErrorIs(t, upstream.requests[0].Context().Err(), context.Canceled) - require.ErrorIs(t, upstream.requests[1].Context().Err(), context.Canceled) - deadline, ok := upstream.requests[1].Context().Deadline() - require.True(t, ok, "the recovery must retain the original client deadline") - clientDeadline, _ := ctx.Deadline() - require.Equal(t, clientDeadline, deadline) - for _, request := range upstream.requests { - require.Equal(t, HTTPUpstreamProfileOpenAI, HTTPUpstreamProfileFromContext(request.Context())) - } - require.Equal(t, upstream.requests[0].URL, upstream.requests[1].URL) - require.Equal(t, upstream.requests[0].Header, upstream.requests[1].Header) - require.True(t, gjson.GetBytes(upstream.bodies[0], "input.1.encrypted_content").Exists()) - expected, err := sjson.DeleteBytes(upstream.bodies[0], "input.1.encrypted_content") - require.NoError(t, err) - require.JSONEq(t, string(expected), string(upstream.bodies[1])) - require.NotContains(t, recorder.Body.String(), "resp_rejected") - if outcome == "completed" { - require.NoError(t, got.err) - require.NotNil(t, got.result) - require.Equal(t, 4, got.result.Usage.InputTokens) - require.Equal(t, 6, got.result.Usage.OutputTokens) - } else { - require.Error(t, got.err) - require.True(t, upstream.recoveryCancellationObserved, "the recovery must observe client cancellation before transport cleanup") - var failover *UpstreamFailoverError - require.False(t, errors.As(got.err, &failover), "a canceled recovery must not reenter the account pool") - } - }) - } -} diff --git a/backend/internal/service/openai_gateway_count_tokens.go b/backend/internal/service/openai_gateway_count_tokens.go index 89b5ad3a7fe3..0c89b4662ad8 100644 --- a/backend/internal/service/openai_gateway_count_tokens.go +++ b/backend/internal/service/openai_gateway_count_tokens.go @@ -150,7 +150,7 @@ func prepareNativeOpenAIInputTokensCountRequest(body []byte, account *Account) ( } func shouldEstimateOpenAIInputTokensLocally(account *Account) bool { - if account == nil || account.IsGrok() || account.IsCNProvider() || account.Type == AccountTypeUpstream { + if account == nil || account.IsGrok() || account.RoutesProtocolByInbound() || account.Type == AccountTypeUpstream { return true } if account.Type != AccountTypeAPIKey { @@ -264,14 +264,14 @@ func (s *OpenAIGatewayService) ForwardCountTokensAsAnthropic( return fmt.Errorf("count_tokens: missing account") } - // 国产供应商与 OpenCode(全部协议,含 anthropic):一律本地估算,不发上游请求。 + // 多协议 API Key 供应商(国产厂商与聚合平台,全部协议,含 anthropic):一律本地估算,不发上游请求。 // 依据(2026-08 核实):三家的 Anthropic 兼容层均未提供 // /v1/messages/count_tokens——DeepSeek 官方 anthropic_api 文档无此端点 // (且注明 anthropic-version 头被忽略),聚合网关 OpenModel 明确标注 // count_tokens 为 "Anthropic only",Kimi/智谱亦无任何文档承诺。转发上游 // 只会常态 404,且错误还会流入账号处置逻辑误伤整账号调度;Claude Code // 高频调用此端点,本地 tiktoken 估算是与 Grok 一致的既有方案。 - if account.IsCNProvider() || account.IsOpenCodeGo() { + if account.IsMultiProtocolAPIKey() { estimated, err := estimateAnthropicCountTokensLocally(body) if err != nil { writeAnthropicCountTokensError(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body") diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index 40e4f361aa3e..587a85ac6d63 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -211,24 +211,6 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco return s.forwardGrokResponses(ctx, c, account, body, originalModel, reqStream, startTime) } - if account.IsOpenCodeGo() { - mapped := resolveOpenCodeGoMappedModel(account, body, "") - switch openCodeGoNativeProtocol(account, mapped) { - case APIProtocolAnthropic: - return s.forwardResponsesViaNativeAnthropic(ctx, c, account, body, "") - case APIProtocolResponses: - break - default: - return s.forwardResponsesViaRawChatCompletions(ctx, c, account, body) - } - } - - // CN 供应商 anthropic 协议账号:/v1/responses 入站是交叉协议组合 - // (Responses 客户端 × Anthropic 上游),转成 Anthropic 请求走原生端点。 - // 不能落到下面的 raw-CC 分支——其 URL 构造会把 anthropic base 当 CC base 用。 - if account.IsAnthropicProtocol() { - return s.forwardResponsesViaNativeAnthropic(ctx, c, account, body, reqModel) - } if account.IsOpenAIApiKey() { if normalized, changed, normalizeErr := normalizeOpenAIParallelToolCallsWithoutTools(body, responsesLite); normalizeErr != nil { return nil, normalizeErr @@ -241,7 +223,19 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco originalModel = reqModel } - if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { + // 上游协议统一由 resolveUpstreamProtocol 判定(按模型分流时带上游模型目录)。OpenAI API Key 账号只会落到 + // Responses / Chat Completions,上面的归一化对两条路径都生效。 + routingModel := upstreamRoutingModel(account, body, "") + if account.IsOpenCodeGo() && IsOpenCodeUnsupportedModel(routingModel) { + return nil, writeOpenCodeUnsupportedModelError(c, false, routingModel) + } + switch s.resolveUpstreamProtocolFor(ctx, account, APIProtocolResponses, routingModel) { + case APIProtocolAnthropic: + // Responses 客户端 × Anthropic 上游:转成 Anthropic 请求走原生端点。不能落到 + // raw-CC 分支——其 URL 构造会把 anthropic base 当 CC base 用。 + // 账号映射未命中时以去除首尾空白的请求模型兜底,计费名与上游模型名一致。 + return s.forwardResponsesViaNativeAnthropic(ctx, c, account, body, reqModel) + case APIProtocolChatCompletions: return s.forwardResponsesViaRawChatCompletions(ctx, c, account, body) } if usesOfficialOpenAIResponsesInputContract(account) { @@ -545,6 +539,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco SkipDefaultInstructions: true, PreserveToolCallIDs: true, OmitPromotedSystemMessagesFromInput: omitPromotedSystemMessages, + ResponsesLite: responsesLite, }) ensureCodexOAuthInstructionsField(decoded) markDecodedModified() @@ -553,6 +548,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco IsCodexCLI: isCodexCLI, IsCompact: isCompactRequest, OmitPromotedSystemMessagesFromInput: omitPromotedSystemMessages, + ResponsesLite: responsesLite, }) } if codexResult.Modified { @@ -768,6 +764,26 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco // A prior rejection is not evidence that an encrypted state item is // dispensable. Preserve the incoming history and let its source validate it. + // 剥离本会话已被上游判定失效的加密项(invalid_encrypted_content lineage), + // 阻断同一失效密文随客户端历史在每一轮重复触发"被拒→剥离→重试/重连"。 + // lineage 会话键统一按进场形态的 body 派生:后续重试可能改写 body, + // 延迟计算会与下一请求的进场键漂移。 + lineageGroupID := getOpenAIGroupIDFromContext(c) + lineageEntryBody := body + lineageSessionHash := "" + if stateStore := s.getOpenAIWSStateStore(); stateStore != nil && stateStore.HasAnySessionInvalidEncryptedContent() { + lineageSessionHash = s.GenerateSessionHash(c, body) + if invalidDigests := stateStore.GetSessionInvalidEncryptedContentDigests(lineageGroupID, lineageSessionHash); len(invalidDigests) > 0 { + strippedBody, strippedCount := s.stripSessionInvalidEncryptedContentLogged( + body, invalidDigests, "invalid_encrypted_lineage_strip", account.ID, 0, + ) + if strippedCount > 0 { + body = strippedBody + requestView = newOpenAIRequestView(body) + reqBody = nil + } + } + } imageBillingModel := "" imageSizeTier := "" imageInputSize := "" @@ -805,7 +821,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if err != nil { return nil, err } - hasPreviousResponseID := strings.TrimSpace(openAIWSPayloadString(wsReqBody, "previous_response_id")) != "" + _, hasPreviousResponseID := wsReqBody["previous_response_id"] logOpenAIWSModeDebug( "forward_start account_id=%d account_type=%s model=%s stream=%v has_previous_response_id=%v", account.ID, @@ -820,9 +836,48 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco var wsErr error wsLastFailureReason := "" agentTaskRecoveryTried := false - // This single-request forwarder has no trusted full-history accumulator. - // Missing anchors and rejected encrypted state therefore terminate; - // proven full-input reconstruction belongs to the WS ingress lifecycle. + wsInvalidEncryptedContentRecoveryTried := false + recoverInvalidEncryptedContent := func(attempt int) bool { + if wsInvalidEncryptedContentRecoveryTried { + return false + } + // 写入 lineage 后,同一失效密文在后续 turn 进场时被预剥离,不再重复 + // 触发上游拒绝与重连。摘要取自进场形态的 body(密文项只可能来自 + // 客户端进场请求,重复摘要幂等)。 + invalidDigests := collectOpenAIEncryptedContentDigestsRaw(lineageEntryBody) + removedReasoningItems := trimOpenAIEncryptedReasoningItems(wsReqBody) + if !removedReasoningItems { + logOpenAIWSModeInfo( + "reconnect_invalid_encrypted_content_recovery_skip account_id=%d attempt=%d reason=missing_encrypted_reasoning_items", + account.ID, + attempt, + ) + return false + } + if len(invalidDigests) > 0 { + if lineageSessionHash == "" { + lineageSessionHash = s.GenerateSessionHash(c, lineageEntryBody) + } + s.markOpenAIWSInvalidEncryptedContentLineage(lineageGroupID, lineageSessionHash, invalidDigests) + } + previousResponseID := openAIWSPayloadString(wsReqBody, "previous_response_id") + hasFunctionCallOutput := HasFunctionCallOutput(wsReqBody) + if previousResponseID != "" && !hasFunctionCallOutput { + delete(wsReqBody, "previous_response_id") + } + wsInvalidEncryptedContentRecoveryTried = true + logOpenAIWSModeInfo( + "reconnect_invalid_encrypted_content_recovery account_id=%d attempt=%d action=drop_encrypted_reasoning_items retry=1 previous_response_id_present=%v previous_response_id=%s previous_response_id_kind=%s has_function_call_output=%v dropped_previous_response_id=%v", + account.ID, + attempt, + previousResponseID != "", + truncateOpenAIWSLogValue(previousResponseID, openAIWSIDValueMaxLen), + normalizeOpenAIWSLogValue(ClassifyOpenAIPreviousResponseIDKind(previousResponseID)), + hasFunctionCallOutput, + previousResponseID != "" && !hasFunctionCallOutput, + ) + return true + } retryBudget := s.openAIWSRetryTotalBudget() retryStartedAt := time.Now() wsRetryLoop: @@ -852,14 +907,6 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if c != nil && c.Writer != nil && c.Writer.Written() { break } - // Structured model_not_supported failures must cross the WS retry - // boundary intact. They are account/model scoped (and already cooled - // by the WS forwarder), so a reconnect retry must not swallow them. - var modelNotSupportedErr *UpstreamFailoverError - if errors.As(wsErr, &modelNotSupportedErr) && modelNotSupportedErr.IsOpenAIModelNotSupported() { - s.recordOpenAIWSModelNotSupportedAttempt(c, account, modelNotSupportedErr.ResponseHeaders, modelNotSupportedErr.ResponseBody) - return nil, modelNotSupportedErr - } var taskRecoveredErr *agentIdentityTaskRecoveredError if errors.As(wsErr, &taskRecoveredErr) { continue @@ -869,8 +916,11 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if reason != "" { wsLastFailureReason = reason } - if stateReason := strings.TrimPrefix(reason, "prewarm_"); stateReason == "previous_response_not_found" || stateReason == "invalid_encrypted_content" { - break + // previous_response_not_found 说明续链锚点不可用: + // 对非 function_call_output 场景,允许一次“去掉 previous_response_id 后重放”。 + + if reason == "invalid_encrypted_content" && recoverInvalidEncryptedContent(attempt) { + continue } if retryable && attempt < maxAttempts { backoff := s.openAIWSRetryBackoff(attempt) @@ -960,22 +1010,15 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco } return wsResult, nil } - continuationReason, _ := classifyOpenAIWSReconnectReason(wsErr) - switch strings.TrimPrefix(continuationReason, "prewarm_") { - case "invalid_encrypted_content", "previous_response_not_found": + if reason, _ := classifyOpenAIWSReconnectReason(wsErr); strings.TrimPrefix(reason, "prewarm_") == "previous_response_not_found" { s.recordOpenAIWSContinuationStateError(c, account, wsErr) return nil, NewOpenAIContinuationStateUnavailableError(http.StatusBadRequest, nil, nil) - case "thinking_signature_invalid": - // Ops keeps the rejected event; the client path below is unchanged. - s.recordOpenAIWSContinuationStateError(c, account, wsErr) } s.writeOpenAIWSFallbackErrorResponse(c, account, wsErr) return nil, wsErr } - reasoningRecovery := s.newOpenAIReasoningRecoveryState(ctx, c, account, token) - defer reasoningRecovery.Close() - defer func() { forwardErr = reasoningRecovery.StopError(forwardErr) }() + httpInvalidEncryptedContentRetryTried := false compactModelFallbackRetried := false agentTaskRecoveryTried := false rejectedFieldRetryState := newOpenAIResponsesRejectedFieldRetryState(body) @@ -1020,7 +1063,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if account.ProxyID != nil && account.Proxy != nil { proxyURL = account.Proxy.URL() } - upstreamReq, body, wireBody, err = prepareReasoningRecoveryRequest(reasoningRecovery, upstreamReq, body, proxyURL) + wireBody, err = finalWireRequestBody(upstreamReq, body) if err != nil { if headerGuard != nil { headerGuard.close() @@ -1058,7 +1101,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco headerGuard.close() headerGuard = nil } - upstreamReq = upstreamReq.WithContext(codexGatewayBorrowHTTPPreparedContext(ctx, upstreamReq.Context(), replacedGuard, reasoningRecovery.RecoveryAttempt())) + upstreamReq = upstreamReq.WithContext(codexGatewayBorrowHTTPPreparedContext(ctx, upstreamReq.Context(), replacedGuard, httpInvalidEncryptedContentRetryTried)) } firstOutputStartedAt := startTime.Add(borrowPreparationElapsed) if firstOutputTimeout > 0 && headerGuard == nil { @@ -1083,11 +1126,11 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco // Bind only after preparation, context/guard changes and the final // cached Cookie canonicalization. Dispatch preserves this plaintext // pointer; wire compression still uses its separate outbound copy. - reasoningRecovery.BindDiagnosticRequest(diagnosticIncomingBody, upstreamReq) // Send request + freezeOpenAIContinuationWire(c, upstreamReq, wireBody) upstreamStart := time.Now() - reasoningRecovery.MarkAttemptDispatched() + resp, err := s.doOpenAICodexUpstream(upstreamReq, account, proxyURL, upstreamModel) if pelicanHeaderPolicy != nil { headerGuard = pelicanHeaderPolicy.guard @@ -1095,7 +1138,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco firstOutputStartedAt = pelicanHeaderPolicy.startedAt } } - reasoningRecovery.ObserveResponse(resp) + SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds()) if IsCodexGatewayBorrowRequestFailure(err) { if resp != nil && resp.Body != nil { @@ -1117,9 +1160,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco _ = resp.Body.Close() } headerGuard.close() - if reasoningRecovery.RecoveryAttempt() { - return nil, errors.New("reasoning recovery upstream header timeout") - } + return nil, s.newOpenAIFirstOutputTimeoutError( ctx, c, account, opsUpstreamProxyID(account), opsUpstreamProxyName(account), startTime, originalModel, reasoningEffortValue, @@ -1133,9 +1174,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if headerGuard != nil { headerGuard.close() } - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + // Transport-level failure (proxy/DNS/TCP/TLS — no HTTP response). Convert to // a failover so the handler switches to a healthy account, and temporarily // unschedule the account on durable faults (e.g. rejected proxy credentials). @@ -1150,41 +1189,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco respBody := s.readUpstreamErrorBody(resp) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(respBody)) - if repaired, repair := reasoningRecovery.TryRepairUnfoundItemIDs(resp.StatusCode, resp.Header, respBody); repair { - body, err = projectReasoningRecoveryEdits(body, wireBody, repaired) - if err != nil { - return nil, err - } - requestView = newOpenAIRequestView(body) - reqBody = nil - continue - } - if retryBody, retry := reasoningRecovery.TryRecover(resp.StatusCode, resp.Header, respBody, false); retry { - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - requestView = newOpenAIRequestView(body) - reqBody = nil - continue - } - if reasoningRecovery.RecoveryAttempt() { - failure := reasoningRecovery.FailureForResponse(resp.StatusCode, resp.Header, respBody) - // Only an answer that a first send would move to another account - // is the account's failure. Any other ends the request here, - // unless StopError hands it on as a ciphertext mismatch. - retryMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) - if !s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, retryMsg, respBody) { - failure.NextAccountAction = NextAccountStop - } else if failure.ShouldRetryNextAccount() { - failure = s.openAIRecoveryRetryHTTPFailure(account, resp.StatusCode, resp.Header, respBody, retryMsg, - openAIHTTPPoolRetryable(ctx, account, resp.StatusCode, retryMsg, respBody, false)) - } - return nil, failure - } - if _, rejected := parseOpenAIReasoningRejection(respBody); rejected { - return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, respBody) - } + if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( ctx, c, account, resp.StatusCode, resp.Header, respBody, ); ok { @@ -1193,6 +1198,11 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) + if classifyOpenAIContinuationStateError(upstreamMsg, respBody) == openAIContinuationStateErrorPreviousResponseNotFound { + s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, respBody, "continuation_state", false, + buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, respBody, "previous_response_not_found")) + return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, respBody) + } if !agentTaskRecoveryTried && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) { agentTaskRecoveryTried = true expectedTaskID := account.GetCredential("task_id") @@ -1203,17 +1213,35 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco } respBody = s.redactAgentIdentitySensitiveBody(ctx, account, respBody) resp.Body = io.NopCloser(bytes.NewReader(respBody)) - continuationStateError := classifyOpenAIContinuationStateError(upstreamMsg, respBody) - if continuationStateError != openAIContinuationStateErrorNone { - // The failure describes request history, not account health, whatever - // status a compatibility proxy wraps it in. Whether rejected - // ciphertext is offered to another account is decided once, when the - // attempt ends (StopError). - s.recordOpenAIRequestTerminalUpstreamError(ctx, c, account, resp.StatusCode, resp.Header, body, respBody, - "continuation_state", false, - buildOpenAIContinuationDiagnostic(c, diagnosticIncomingBody, upstreamReq, body, respBody, string(continuationStateError)), - ) - return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, respBody) + upstreamCode := strings.TrimSpace(gjson.GetBytes(respBody, "error.code").String()) + invalidEncryptedContentError := upstreamCode == "invalid_encrypted_content" || + (upstreamCode == "thinking_signature_invalid" && + strings.Contains(upstreamMsg, "The encrypted content") && + strings.Contains(upstreamMsg, "could not be verified") && + strings.Contains(upstreamMsg, "could not be decrypted or parsed")) + if !httpInvalidEncryptedContentRetryTried && resp.StatusCode == http.StatusBadRequest && invalidEncryptedContentError { + decoded, decodeErr := ensureReqBody() + if decodeErr != nil { + return nil, decodeErr + } + invalidDigests := collectOpenAIEncryptedContentDigestsRaw(lineageEntryBody) + if trimOpenAIEncryptedReasoningItems(decoded) { + body, err = marshalOpenAIUpstreamJSON(decoded) + if err != nil { + return nil, fmt.Errorf("serialize invalid_encrypted_content retry body: %w", err) + } + if len(invalidDigests) > 0 { + if lineageSessionHash == "" { + lineageSessionHash = s.GenerateSessionHash(c, lineageEntryBody) + } + s.markOpenAIWSInvalidEncryptedContentLineage(lineageGroupID, lineageSessionHash, invalidDigests) + } + httpInvalidEncryptedContentRetryTried = true + rejectedFieldRetryState.remember(body) + logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Retrying non-WSv2 request once after invalid_encrypted_content (account: %s)", account.Name) + continue + } + logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Skip non-WSv2 invalid_encrypted_content retry because encrypted reasoning items are missing (account: %s)", account.Name) } if retryBody, reason, changed, retryErr := normalizeSystemPromptRejectedFieldRetryBody(c, resp.StatusCode, body, respBody); retryErr != nil { return nil, fmt.Errorf("normalize rejected Responses field retry body: %w", retryErr) @@ -1308,19 +1336,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco } streamResult, err := s.handleStreamingResponseWithReasoning(streamCtx, resp, c, account, startTime, originalModel, upstreamModel, reasoningEffortValue) if err != nil { - if retryBody, retry := reasoningRecovery.TryRecoverError(err); retry { - _ = resp.Body.Close() - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - requestView = newOpenAIRequestView(body) - reqBody = nil - continue - } - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + if signal, ok := asOpenAICompactFallbackSignal(err); ok { if retryBody, fallbackModel, retry := s.prepareOpenAICompactFallbackRetry( c, account, originalModel, body, http.StatusBadRequest, signal.message, signal.payload, compactModelFallbackRetried, @@ -1370,19 +1386,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco } else { nonStreamResult, err := s.handleNonStreamingResponse(ctx, resp, c, account, originalModel, upstreamModel) if err != nil { - if retryBody, retry := reasoningRecovery.TryRecoverError(err); retry { - _ = resp.Body.Close() - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - requestView = newOpenAIRequestView(body) - reqBody = nil - continue - } - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + if signal, ok := asOpenAICompactFallbackSignal(err); ok { if retryBody, fallbackModel, retry := s.prepareOpenAICompactFallbackRetry( c, account, originalModel, body, http.StatusBadRequest, signal.message, signal.payload, compactModelFallbackRetried, @@ -1463,14 +1467,15 @@ func shouldForwardOpenAIResponsesViaRawChatCompletions(account *Account) bool { if account == nil || account.Type != AccountTypeAPIKey { return false } - if account.IsOpenCodeGo() { + if account.routesByModel() { // Model protocol_rules are the authority. Probe Extra must not collapse // Grok/GPT/Muse into Chat Completions. return false } - if account.IsCNProvider() { - // CN 的显式协议配置优先于异步探针 Extra;adaptive 仅 DeepSeek / Kimi - // 有原生 Responses,GLM 回退 Chat Completions。 + if account.RoutesProtocolByInbound() { + // 按入站协议分流的供应商(国产厂商等):显式协议配置优先于异步探针 + // Extra;adaptive 仅在供应商有原生 Responses 端点时直转,否则回退 + // Chat Completions(如 GLM)。 switch account.GetAPIProtocol() { case APIProtocolChatCompletions: return true diff --git a/backend/internal/service/openai_gateway_messages.go b/backend/internal/service/openai_gateway_messages.go index 01e8be31d9cd..ed5031c180da 100644 --- a/backend/internal/service/openai_gateway_messages.go +++ b/backend/internal/service/openai_gateway_messages.go @@ -53,29 +53,21 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic( return nil, err } - // OpenCode Go:按模型原生协议分流。规则未命中兜底 Chat Completions。 - if account.IsOpenCodeGo() { - mapped := resolveOpenCodeGoMappedModel(account, body, defaultMappedModel) - switch openCodeGoNativeProtocol(account, mapped) { - case APIProtocolAnthropic: - return s.forwardAnthropicViaNativeAnthropicEndpoint(ctx, c, account, body, defaultMappedModel) - case APIProtocolResponses: - break - default: - return s.forwardAnthropicViaRawChatCompletions(ctx, c, account, body, defaultMappedModel) - } - } else if account.IsAnthropicProtocol() || account.IsAdaptiveAPIProtocol() { - // 入口分流(国产供应商 Anthropic 协议):上游为供应商原生 Anthropic 端点时, - // /v1/messages 请求零转换直通(仅模型名映射 + 少量 body 清洗),完整保留 - // thinking / tool_use / cache 语义,适配 Claude Code 等原生客户端。 - // 必须先于 ShouldUseResponsesAPI 分流:Anthropic 协议账号经 probe 落标 - // openai_responses_supported=false,会先命中下方的 CC 直转分支。 + // 上游协议统一由 resolveUpstreamProtocol 判定(按模型分流时带上游模型目录)。Anthropic 分流必须先于 + // ShouldUseResponsesAPI:Anthropic 协议账号经 probe 落标 + // openai_responses_supported=false,否则会命中 CC 直转。 + routingModel := upstreamRoutingModel(account, body, defaultMappedModel) + if account.IsOpenCodeGo() && IsOpenCodeUnsupportedModel(routingModel) { + return nil, writeOpenCodeUnsupportedModelError(c, true, routingModel) + } + switch s.resolveUpstreamProtocolFor(ctx, account, APIProtocolAnthropic, routingModel) { + case APIProtocolAnthropic: + // 上游为供应商原生 Anthropic 端点:/v1/messages 零转换直通(仅模型名映射 + + // 少量 body 清洗),完整保留 thinking / tool_use / cache 语义。 return s.forwardAnthropicViaNativeAnthropicEndpoint(ctx, c, account, body, defaultMappedModel) - } - - // 固定 chat_completions 的 CN 账号,以及不支持 Responses 的其他 APIKey - // 账号,均将 Messages 转为 CC;固定 responses 的 CN 账号不受探针旧值覆盖。 - if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { + case APIProtocolChatCompletions: + // 固定 chat_completions 的多协议账号、规则命中 CC 的按模型分流账号,以及 + // 不支持 Responses 的其他 APIKey 账号,均将 Messages 转为 CC。 return s.forwardAnthropicViaRawChatCompletions(ctx, c, account, body, defaultMappedModel) } SetActualOpenAIUpstreamEndpoint(c, openAIResponsesUpstreamEndpoint) @@ -840,14 +832,14 @@ func (s *OpenAIGatewayService) readOpenAICompatBufferedTerminal( var parser openAICompatSSEFrameParser var pendingSignatureResponse *apicompat.ResponsesResponse var pendingSignaturePayload []byte - deferSignature := len(deferSignatureRejection) > 0 && deferSignatureRejection[0] + for { select { case ev, ok := <-events: if !ok { if frame, ok := parser.Finish(); ok { payload := openAICompatPayloadWithEventType(frame.Data, frame.EventType) - observeOpenAIReasoningAttemptUsage(c, []byte(payload)) + var event apicompat.ResponsesStreamEvent if err := json.Unmarshal([]byte(payload), &event); err == nil { s.parseSSEUsageBytesWithType([]byte(payload), event.Type, &usage) @@ -890,7 +882,6 @@ func (s *OpenAIGatewayService) readOpenAICompatBufferedTerminal( continue } payload := openAICompatPayloadWithEventType(frame.Data, frame.EventType) - observeOpenAIReasoningAttemptUsage(c, []byte(payload)) var event apicompat.ResponsesStreamEvent if err := json.Unmarshal([]byte(payload), &event); err != nil { @@ -917,11 +908,7 @@ func (s *OpenAIGatewayService) readOpenAICompatBufferedTerminal( return response, usage, acc, []byte(payload), nil } if failedResponse, ok := openAICompatBareErrorResponse(&event, []byte(payload)); ok { - if _, signatureRejected := parseOpenAIReasoningRejection([]byte(payload)); deferSignature && signatureRejected { - pendingSignatureResponse = failedResponse - pendingSignaturePayload = append([]byte(nil), []byte(payload)...) - continue - } + return failedResponse, usage, acc, []byte(payload), nil } diff --git a/backend/internal/service/openai_gateway_messages_anthropic_native.go b/backend/internal/service/openai_gateway_messages_anthropic_native.go index e1dea9ef374d..2e65a1bb61b1 100644 --- a/backend/internal/service/openai_gateway_messages_anthropic_native.go +++ b/backend/internal/service/openai_gateway_messages_anthropic_native.go @@ -133,14 +133,20 @@ func (s *OpenAIGatewayService) nativeAnthropicTargetURL(account *Account) (strin if err != nil { return "", fmt.Errorf("invalid base_url: %w", err) } - if account.IsOpenCodeGo() { - // OpenCode Go 的 Chat Completions base 带 /v1;用版本感知拼接避免 /v1/v1/messages。 - return buildOpenAIEndpointURL(validatedURL, "/v1/messages"), nil + return nativeAnthropicMessagesURL(account, validatedURL), nil +} + +// nativeAnthropicMessagesURL 由已校验的 Anthropic 协议基址拼出 messages 端点,转发与 +// 连接测试共用。按模型分流的聚合平台(OpenCode、Command Code 等)的基址可能沿用带 /v1 +// 的 Chat Completions 基址,用版本感知拼接避免 /v1/v1/messages;其余供应商朴素拼接。 +func nativeAnthropicMessagesURL(account *Account, validatedBaseURL string) string { + if account.routesByModel() { + return buildOpenAIEndpointURL(validatedBaseURL, "/v1/messages") } - return strings.TrimRight(validatedURL, "/") + "/v1/messages", nil + return strings.TrimRight(validatedBaseURL, "/") + "/v1/messages" } -func resolveOpenCodeGoMappedModel(account *Account, body []byte, defaultMappedModel string) string { +func resolveMappedUpstreamModel(account *Account, body []byte, defaultMappedModel string) string { original := strings.TrimSpace(gjson.GetBytes(body, "model").String()) billing := resolveOpenAIForwardModel(account, original, defaultMappedModel) return normalizeOpenAIModelForUpstream(account, billing) diff --git a/backend/internal/service/openai_gateway_ops_visibility_test.go b/backend/internal/service/openai_gateway_ops_visibility_test.go index 78822ddf68b0..e46cc2f33d60 100644 --- a/backend/internal/service/openai_gateway_ops_visibility_test.go +++ b/backend/internal/service/openai_gateway_ops_visibility_test.go @@ -16,7 +16,6 @@ import ( "github.com/Wei-Shaw/sub2api/internal/config" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" ) func opsVisibilityTestContext() *gin.Context { @@ -253,19 +252,6 @@ func TestOpenAIImagesModelTextKeepsFullTextForAdministrators(t *testing.T) { // OPS-10: the reasoning-recovery detail is bounded by its encoded size, so the // Ops queue never shrinks it to an object without its fields. -func TestOpenAIReasoningRecoveryDetailFitsOpsBudget(t *testing.T) { - payload := []byte(`{"type":"error","error":{"code":"invalid_encrypted_content","message":"` + strings.Repeat("", 2000) + `"}}`) - detail := map[string]any{"action": "retry_without_encrypted_content", "items": 3, "usage_status": "unavailable", "payload_sha256": openAIReasoningDigest(payload)} - - encoded := encodeOpenAIReasoningRecoveryDetail(detail, extractOpenAISSEErrorMessage(payload), payload) - - require.LessOrEqual(t, len(encoded), openAIReasoningRecoveryDetailEncodedLimit) - stored, _ := sanitizeErrorBodyForStorage(string(encoded), OpsErrorLogQueueBodyMaxBytes) - for _, key := range []string{"action", "items", "usage_status", "payload_sha256", "upstream_message", "payload"} { - require.True(t, gjson.Get(stored, key).Exists(), key) - } - require.True(t, gjson.Get(stored, "payload_truncated").Bool()) -} // OPS-4: a delivered cyber-policy block keeps only its dedicated row; other // delivered failures are marked as client-visible. diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 6bff6f7a0047..a5526c23b4a5 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -236,7 +236,10 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough( } if account != nil && account.IsOpenAI() { responsesLite := isOpenAIResponsesLiteHeader(c.GetHeader(responsesLiteHeader)) || isOpenAIResponsesLiteWebSocketPayload(body) - normalizedBody, normalized, normalizeErr := normalizeOpenAIResponsesWebSocketCompatibilityBody(body, account, responsesLite) + normalizedBody, normalized, normalizeErr := normalizeOpenAIResponsesCompatibilityBodyWithOptions(body, account, openAIResponsesCompatibilityOptions{ + ResponsesLite: responsesLite, + Compact: isOpenAIResponsesCompactPath(c), + }) if normalizeErr != nil { return nil, fmt.Errorf("normalize passthrough Responses compatibility: %w", normalizeErr) } @@ -374,9 +377,6 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough( c.Set("openai_passthrough", true) } - reasoningRecovery := s.newOpenAIReasoningRecoveryState(ctx, c, account, token) - defer reasoningRecovery.Close() - defer func() { forwardErr = reasoningRecovery.StopError(forwardErr) }() agentTaskRecoveryTried := false compactModelFallbackRetried := false var reasoningEffort *string @@ -400,7 +400,7 @@ retryUpstream: if buildErr != nil { return nil, buildErr } - upstreamReq, body, wireBody, buildErr = prepareReasoningRecoveryRequest(reasoningRecovery, upstreamReq, body, proxyURL) + wireBody, buildErr = finalWireRequestBody(upstreamReq, body) if buildErr != nil { return nil, buildErr } @@ -412,17 +412,15 @@ retryUpstream: if integrityErr := s.checkStagedRequestIntegrity(c, account, "http_passthrough", wireBody, integrityOpts); integrityErr != nil { return nil, integrityErr } - reasoningRecovery.BindDiagnosticRequest(diagnosticIncomingBody, upstreamReq) + freezeOpenAIContinuationWire(c, upstreamReq, wireBody) upstreamStart := time.Now() - reasoningRecovery.MarkAttemptDispatched() + resp, err = s.doOpenAICodexUpstream(upstreamReq, account, proxyURL, actualModel) - reasoningRecovery.ObserveResponse(resp) + SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds()) if err != nil { - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + // Transport-level failure (proxy/DNS/TCP/TLS — no HTTP response). Convert to // a failover so the handler switches to a healthy account. return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, true) @@ -436,37 +434,7 @@ retryUpstream: probeBody := s.readUpstreamErrorBody(resp) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(probeBody)) - if repaired, repair := reasoningRecovery.TryRepairUnfoundItemIDs(resp.StatusCode, resp.Header, probeBody); repair { - body, err = projectReasoningRecoveryEdits(body, wireBody, repaired) - if err != nil { - return nil, err - } - continue - } - if retryBody, retry := reasoningRecovery.TryRecover(resp.StatusCode, resp.Header, probeBody, false); retry { - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - continue - } - if reasoningRecovery.RecoveryAttempt() { - failure := reasoningRecovery.FailureForResponse(resp.StatusCode, resp.Header, probeBody) - // Only an answer that a first send would move to another account is - // the account's failure. Any other ends the request here, unless - // StopError hands it on as a ciphertext mismatch. - if !shouldFailoverOpenAIPassthroughResponse(account, resp.StatusCode, probeBody) { - failure.NextAccountAction = NextAccountStop - } else if failure.ShouldRetryNextAccount() { - retryMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(probeBody))) - failure = s.openAIRecoveryRetryHTTPFailure(account, resp.StatusCode, resp.Header, probeBody, retryMsg, - account != nil && account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode)) - } - return nil, failure - } - if _, rejected := parseOpenAIReasoningRejection(probeBody); rejected { - return nil, NewOpenAIContinuationStateUnavailableError(resp.StatusCode, resp.Header, probeBody) - } + if failoverErr, ok := s.handleOpenAIBudgetExceededHTTPFailover( ctx, c, account, resp.StatusCode, resp.Header, probeBody, ); ok { @@ -543,17 +511,7 @@ retryUpstream: setOpenAIRefusalEarlyStreamEligibility(c, account, body) result, err := s.handleStreamingResponsePassthrough(ctx, resp, c, account, startTime, reqModel, upstreamPassthroughModel) if err != nil { - if retryBody, retry := reasoningRecovery.TryRecoverError(err); retry { - _ = resp.Body.Close() - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - goto retryUpstream - } - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + if retryBody, fallbackModel, retry := s.applyOpenAIPassthroughCompactFallbackFromSignal( c, account, reqModel, body, err, compactModelFallbackRetried, resp, ); retry { @@ -580,17 +538,7 @@ retryUpstream: } else { result, err := s.handleNonStreamingResponsePassthrough(ctx, resp, c, account, reqModel, upstreamPassthroughModel) if err != nil { - if retryBody, retry := reasoningRecovery.TryRecoverError(err); retry { - _ = resp.Body.Close() - body, err = projectReasoningRecoveryEdits(body, wireBody, retryBody) - if err != nil { - return nil, err - } - goto retryUpstream - } - if reasoningRecovery.RecoveryAttempt() { - return nil, err - } + if retryBody, fallbackModel, retry := s.applyOpenAIPassthroughCompactFallbackFromSignal( c, account, reqModel, body, err, compactModelFallbackRetried, resp, ); retry { @@ -2099,7 +2047,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( terminalFramePending := false terminalEventType := "" var terminalResponseErr error - var pendingReasoningRecoveryErr error + pendingSSEEventType := "" sawFailedEvent := false sawBareError := false @@ -2210,9 +2158,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( failureDelivered = true flushPending = true flushPendingOutput() - if !clientDisconnected { - markOpenAIReasoningFailureTerminalForwarded(c) - } + } scanner := bufio.NewScanner(resp.Body) @@ -2262,7 +2208,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( } if data, ok := extractOpenAISSEDataLine(line); ok { rawDataBytes := []byte(data) - observeOpenAIReasoningAttemptUsage(c, rawDataBytes) + if strings.TrimSpace(data) == "[DONE]" { // Transport markers cannot supply a missing Responses terminal. continue @@ -2270,17 +2216,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( upstreamEventType := effectiveOpenAISSEEventType(rawDataBytes, pendingSSEEventType) s.parseSSEUsageBytesWithType(rawDataBytes, upstreamEventType, usage) if upstreamEventType == "response.failed" || upstreamEventType == "error" || (upstreamEventType == "response.done" && gjson.GetBytes(rawDataBytes, "response.status").String() == "failed") { - if upstreamEventType != "error" { - pendingReasoningRecoveryErr = nil - } - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, rawDataBytes, openAIStreamClientOutputStarted(c, clientOutputStarted)); recoveryErr != nil { - if upstreamEventType != "error" { - return resultWithUsage(), recoveryErr - } - pendingReasoningRecoveryErr = recoveryErr - pendingErrorEventHeader = false - continue - } + if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover(ctx, c, account, resp.StatusCode, resp.Header, rawDataBytes); ok { // Classification must happen on the untouched event, before error // rules, namespace restoration or the client stream see it. The @@ -2330,7 +2266,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( } eventType := effectiveOpenAISSEEventType(dataBytes, pendingSSEEventType) if eventType == "response.completed" || eventType == "response.done" { - pendingReasoningRecoveryErr = nil + // A response terminal supersedes a non-authoritative bare error. sawBareError = false bareErrorPayload = nil @@ -2411,11 +2347,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( return resultWithUsage(), compactErr } } - if _, rejectedReasoning := parseOpenAIReasoningRejection(dataBytes); !rejectedReasoning { - if continuationErr := openAIContinuationStateErrorFromFailedEvent(resp.StatusCode, resp.Header, dataBytes); continuationErr != nil { - return resultWithUsage(), continuationErr - } - } + if !cyberPolicyHit && !openAIStreamClientOutputStarted(c, clientOutputStarted) { if openAIStreamFailedEventShouldFailoverForAccount(account, dataBytes, failedMessage) { return resultWithUsage(), @@ -2487,7 +2419,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( clientDisconnected = true } else { flusher.Flush() - markOpenAIReasoningFailureTerminalForwarded(c) + } } return resultWithUsage(), fmt.Errorf("upstream response failed: %s", failedMessage) @@ -2591,9 +2523,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( flushPending = true if line == "" { flushPendingOutput() - if terminalFramePending && sawFailedEvent && !clientDisconnected { - markOpenAIReasoningFailureTerminalForwarded(c) - } + } } } @@ -2604,9 +2534,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( } } } - if pendingReasoningRecoveryErr != nil && !openAIStreamClientOutputStarted(c, clientOutputStarted) { - return resultWithUsage(), pendingReasoningRecoveryErr - } + ensureResponseFailedTerminal() if err := documentScanner.Err(); err != nil { if sawFailedEvent { @@ -2685,9 +2613,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponsePassthrough( if err != nil { return nil, err } - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, body, openAIStreamClientOutputStarted(c, false)); recoveryErr != nil { - return nil, recoveryErr - } + observer := upstreamResponseModelObserverFromContext(c) if observer == nil { observer = beginUpstreamResponseModelObservation(c) @@ -2804,12 +2730,6 @@ func (s *OpenAIGatewayService) handleNonStreamingResponsePassthrough( // preserving passthrough payloads, except compact-only model remapping may // rewrite model fields back to the original requested model. func (s *OpenAIGatewayService) handlePassthroughSSEToJSON(resp *http.Response, c *gin.Context, account *Account, body []byte, originalModel string, mappedModel string) (*openaiNonStreamingResultPassthrough, error) { - rawTerminalType, rawTerminalPayload, rawTerminalOK := extractOpenAISSETerminalEvent(string(body)) - if rawTerminalOK && (rawTerminalType == "error" || rawTerminalType == "response.failed" || rawTerminalType == "response.done") { - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, rawTerminalPayload, openAIStreamClientOutputStarted(c, false)); recoveryErr != nil { - return nil, recoveryErr - } - } body = restoreSystemPromptEchoSSE(c, body) bodyText := string(body) terminalType, terminalPayload, terminalOK := extractOpenAISSETerminalEvent(bodyText) diff --git a/backend/internal/service/openai_gateway_provider_claude_billing_test.go b/backend/internal/service/openai_gateway_provider_claude_billing_test.go new file mode 100644 index 000000000000..84f11bd3d1c1 --- /dev/null +++ b/backend/internal/service/openai_gateway_provider_claude_billing_test.go @@ -0,0 +1,120 @@ +//go:build unit + +package service + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestOpenAIGatewayServiceRecordUsage_AggregatorClaudeUsesDefaultPricing(t *testing.T) { + for _, platform := range []string{PlatformCommandCode, PlatformCline} { + for _, model := range []string{"claude-sonnet-4-6", "anthropic/claude-sonnet-4-6"} { + t.Run(platform+"/"+model, func(t *testing.T) { + usageRepo := &openAIRecordUsageLogRepoStub{inserted: true} + userRepo := &openAIRecordUsageUserRepoStub{} + svc := newOpenAIRecordUsageServiceForTest(usageRepo, userRepo, &openAIRecordUsageSubRepoStub{}, nil) + svc.resolver = NewModelPricingResolver(nil, svc.billingService) + group := &Group{ID: 1, Platform: platform, RateMultiplier: 1.1} + usage := OpenAIUsage{InputTokens: 1000, OutputTokens: 200} + expected := expectedOpenAICost(t, svc, model, usage, group.RateMultiplier) + require.Greater(t, expected.ActualCost, 0.0) + + err := svc.RecordUsage(context.Background(), providerClaudeRecordUsageInput(platform, model, group, usage)) + + require.NoError(t, err) + require.Equal(t, 1, usageRepo.calls) + require.NotNil(t, usageRepo.lastLog) + require.Equal(t, usage.InputTokens, usageRepo.lastLog.InputTokens) + require.Equal(t, usage.OutputTokens, usageRepo.lastLog.OutputTokens) + require.InDelta(t, expected.TotalCost, usageRepo.lastLog.TotalCost, 1e-12) + require.InDelta(t, expected.ActualCost, usageRepo.lastLog.ActualCost, 1e-12) + require.Equal(t, 1, userRepo.deductCalls) + require.InDelta(t, expected.ActualCost, userRepo.lastAmount, 1e-12) + }) + } + } +} + +func TestOpenAIGatewayServiceRecordUsage_CNAndOpenCodeClaudeRequireExplicitPricing(t *testing.T) { + for _, platform := range []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo} { + t.Run(platform, func(t *testing.T) { + usageRepo := &openAIRecordUsageLogRepoStub{inserted: true} + userRepo := &openAIRecordUsageUserRepoStub{} + svc := newOpenAIRecordUsageServiceForTest(usageRepo, userRepo, &openAIRecordUsageSubRepoStub{}, nil) + svc.resolver = NewModelPricingResolver(nil, svc.billingService) + group := &Group{ID: 1, Platform: platform, RateMultiplier: 1.1} + usage := OpenAIUsage{InputTokens: 1000, OutputTokens: 200} + + err := svc.RecordUsage(context.Background(), providerClaudeRecordUsageInput(platform, "claude-sonnet-4-6", group, usage)) + + require.NoError(t, err) + require.Equal(t, 1, usageRepo.calls) + require.NotNil(t, usageRepo.lastLog) + require.Equal(t, usage.InputTokens, usageRepo.lastLog.InputTokens) + require.Equal(t, usage.OutputTokens, usageRepo.lastLog.OutputTokens) + require.Zero(t, usageRepo.lastLog.TotalCost) + require.Zero(t, usageRepo.lastLog.ActualCost) + require.Zero(t, userRepo.deductCalls) + }) + } +} + +func TestOpenAIGatewayServiceRecordUsage_CNAndOpenCodeClaudeHonorExplicitPricing(t *testing.T) { + const model = "claude-sonnet-4-6" + for _, platform := range []string{PlatformKimi, PlatformOpenCodeGo} { + for _, source := range []string{PricingSourceGroup, PricingSourceChannel} { + t.Run(platform+"/"+source, func(t *testing.T) { + usageRepo := &openAIRecordUsageLogRepoStub{inserted: true} + userRepo := &openAIRecordUsageUserRepoStub{} + svc := newOpenAIRecordUsageServiceForTest(usageRepo, userRepo, &openAIRecordUsageSubRepoStub{}, nil) + group := &Group{ID: 1, Platform: platform, RateMultiplier: 1.25} + inputPrice, outputPrice := 1e-6, 2e-6 + pricing := ChannelModelPricing{ + Models: []string{model}, BillingMode: BillingModeToken, + InputPrice: &inputPrice, OutputPrice: &outputPrice, + } + if source == PricingSourceGroup { + group.ModelPricing = []ChannelModelPricing{pricing} + } else { + cache := newEmptyChannelCache() + cache.pricingByGroupModel[channelModelKey{groupID: group.ID, platform: platform, model: model}] = &pricing + cache.channelByGroupID[group.ID] = &Channel{ID: 1, Status: StatusActive} + cache.groupPlatform[group.ID] = platform + cache.loadedAt = time.Now() + svc.channelService = &ChannelService{} + svc.channelService.cache.Store(cache) + } + svc.resolver = NewModelPricingResolver(svc.channelService, svc.billingService) + usage := OpenAIUsage{InputTokens: 1000, OutputTokens: 200} + expectedTotal := float64(usage.InputTokens)*inputPrice + float64(usage.OutputTokens)*outputPrice + expectedActual := expectedTotal * group.RateMultiplier + + err := svc.RecordUsage(context.Background(), providerClaudeRecordUsageInput(platform, model, group, usage)) + + require.NoError(t, err) + require.NotNil(t, usageRepo.lastLog) + require.InDelta(t, expectedTotal, usageRepo.lastLog.TotalCost, 1e-12) + require.InDelta(t, expectedActual, usageRepo.lastLog.ActualCost, 1e-12) + require.Equal(t, 1, userRepo.deductCalls) + require.InDelta(t, expectedActual, userRepo.lastAmount, 1e-12) + }) + } + } +} + +func providerClaudeRecordUsageInput(platform, model string, group *Group, usage OpenAIUsage) *OpenAIRecordUsageInput { + return &OpenAIRecordUsageInput{ + Result: &OpenAIForwardResult{ + RequestID: "provider_claude_" + platform, + Model: model, BillingModel: model, UpstreamModel: model, + Usage: usage, Duration: time.Second, + }, + APIKey: &APIKey{ID: 10, GroupID: &group.ID, Group: group}, + User: &User{ID: 20}, + Account: &Account{ID: 30, Platform: platform, Type: AccountTypeAPIKey}, + } +} diff --git a/backend/internal/service/openai_gateway_request_body.go b/backend/internal/service/openai_gateway_request_body.go index 600f4e323c6b..1f2894aeeb38 100644 --- a/backend/internal/service/openai_gateway_request_body.go +++ b/backend/internal/service/openai_gateway_request_body.go @@ -6,12 +6,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "net/http" "net/url" "strings" "unicode/utf8" + "github.com/Wei-Shaw/sub2api/internal/pkg/openai" + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" "github.com/Wei-Shaw/sub2api/internal/pkg/openai_compat" @@ -55,11 +56,11 @@ func buildOpenAIResponsesURL(base string) string { } // buildOpenAIResponsesURLForPlatform 组装 Responses 端点(平台感知)。 -// DeepSeek 官方 Responses 端点为 /responses(无 /v1 前缀,适配 Codex); -// 其余平台维持 /v1/responses。 +// 供应商 profile 声明了 ResponsesPath 时按其拼接(如 DeepSeek 为无 /v1 前缀的 +// /responses);其余平台维持 /v1/responses。 func buildOpenAIResponsesURLForPlatform(platform string, base string) string { - if platform == PlatformDeepseek { - return buildOpenAIEndpointURL(base, "/responses") + if profile := LookupProviderProfile(platform); profile != nil && profile.ResponsesPath != "" { + return buildOpenAIEndpointURL(base, profile.ResponsesPath) } return buildOpenAIResponsesURL(base) } @@ -1510,10 +1511,27 @@ func normalizeOpenAIResponseFormatSchemasBody(body []byte) ([]byte, bool, error) } func normalizeOpenAIResponsesWebSocketCompatibilityBody(body []byte, account *Account, responsesLite bool) ([]byte, bool, error) { - return normalizeOpenAIResponsesWebSocketCompatibilityBodyForModel(body, account, responsesLite, gjson.GetBytes(body, "model").String()) + return normalizeOpenAIResponsesCompatibilityBodyWithOptions(body, account, openAIResponsesCompatibilityOptions{ResponsesLite: responsesLite}) } func normalizeOpenAIResponsesWebSocketCompatibilityBodyForModel(body []byte, account *Account, responsesLite bool, resolvedModel string) ([]byte, bool, error) { + return normalizeOpenAIResponsesCompatibilityBodyWithOptions(body, account, openAIResponsesCompatibilityOptions{ResponsesLite: responsesLite, ResolvedModel: resolvedModel}) +} + +type openAIResponsesCompatibilityOptions struct { + ResolvedModel string + ResponsesLite bool + // Compact marks the /responses/compact wire shape, which is left as-is + // by request-shape compatibility rewrites such as web_search history. + Compact bool +} + +func normalizeOpenAIResponsesCompatibilityBodyWithOptions(body []byte, account *Account, opts openAIResponsesCompatibilityOptions) ([]byte, bool, error) { + responsesLite := opts.ResponsesLite + resolvedModel := opts.ResolvedModel + if resolvedModel == "" { + resolvedModel = gjson.GetBytes(body, "model").String() + } if account == nil || !account.IsOpenAI() { return body, false, nil } @@ -1576,6 +1594,14 @@ func normalizeOpenAIResponsesWebSocketCompatibilityBodyForModel(body []byte, acc normalized = next changed = true } + if !opts.Compact { + webSearchBody, webSearchChanged, err := ensureOpenAIOAuthWebSearchToolForHistoryBody(normalized, responsesLite) + if err != nil { + return body, false, fmt.Errorf("normalize websocket body: %w", err) + } + normalized = webSearchBody + changed = changed || webSearchChanged + } } needsOrphanCleanup := account.IsOpenAIOAuthLike() && gjson.GetBytes(normalized, "input").IsArray() diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index 65b083665719..c3f42fd8e4db 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -378,7 +378,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. codexFailureTerminal := account.IsOpenAI() upstreamRequestID := strings.TrimSpace(resp.Header.Get("x-request-id")) var streamEarlyErr error - var pendingReasoningRecoveryErr error + terminalFailurePending := false failureDelivered := false suppressCurrentEvent := false @@ -495,9 +495,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. lastDownstreamWriteAt = time.Now() } finalizeStream := func() (*openaiStreamingResult, error) { - if pendingReasoningRecoveryErr != nil && !sawResponseFailed && !openAIStreamClientOutputStarted(c, clientOutputStarted) { - return resultWithUsage(), pendingReasoningRecoveryErr - } + if stageFirstOutput && eventInProgress { // EOF dispatches the final SSE event even without a trailing blank line. completeGuardedEvent(true) @@ -542,9 +540,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. return resultWithUsage(), fmt.Errorf("stream usage incomplete: missing terminal event") } if sawFailedEvent { - if !clientDisconnected && (failureDelivered || !eventInProgress) { - markOpenAIReasoningFailureTerminalForwarded(c) - } + return resultWithUsage(), fmt.Errorf("upstream response failed: %s", failedMessage) } if terminalResponseErr != nil { @@ -635,7 +631,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. if data, ok := extractOpenAISSEDataLine(line); ok { rawData := data rawDataBytes := []byte(rawData) - observeOpenAIReasoningAttemptUsage(c, rawDataBytes) + // [DONE] is a transport marker, not an authoritative Responses result. // Do not commit it as the first semantic output or claim success at EOF. if strings.TrimSpace(rawData) == "[DONE]" { @@ -645,21 +641,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. rawEventType := effectiveOpenAISSEEventType(rawDataBytes, pendingSSEEventType) if rawEventType == "response.failed" || rawEventType == "error" || (rawEventType == "response.done" && gjson.GetBytes(rawDataBytes, "response.status").String() == "failed") { s.parseSSEUsageBytesWithType(rawDataBytes, rawEventType, usage) - if rawEventType != "error" { - pendingReasoningRecoveryErr = nil - } - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, rawDataBytes, openAIStreamClientOutputStarted(c, clientOutputStarted)); recoveryErr != nil { - if rawEventType != "error" { - streamEarlyErr = recoveryErr - } else { - // A bare error can precede a richer response.failed carrying - // authoritative usage. Hold only this error, not the answer. - pendingReasoningRecoveryErr = recoveryErr - suppressCurrentEvent = true - pendingErrorEventHeader = false - } - return - } + if failoverErr, ok := s.openAIBudgetExceededHTTPResponseTerminalFailover( ctx, c, account, resp.StatusCode, resp.Header, rawDataBytes, ); ok { @@ -692,9 +674,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. bareErrorAccountSideEffectsPending = false failedMessage = "" } - if eventType == "response.completed" || eventType == "response.done" { - pendingReasoningRecoveryErr = nil - } + if codexFailureTerminal && sawBareError && !sawResponseFailed && eventType != "response.failed" { suppressCurrentEvent = true } @@ -802,6 +782,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. refusalAction = openAIRefusalStreamPass } } + if classifyOpenAIContinuationStateError(failedMessage, dataBytes) == openAIContinuationStateErrorPreviousResponseNotFound { + sawFailedEvent = true + streamEarlyErr = NewOpenAIContinuationStateUnavailableError(http.StatusBadRequest, resp.Header, dataBytes) + s.recordOpenAIStreamUpstreamError(c, account, false, upstreamRequestID, "continuation_state", dataBytes, failedMessage) + return + } outputStarted := openAIStreamClientOutputStarted(c, clientOutputStarted) if !outputStarted && !cyberPolicyHit { if compactErr := newOpenAICompactFallbackSignal(c, dataBytes, failedMessage); compactErr != nil { @@ -810,15 +796,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. return } } - if !cyberPolicyHit { - if _, rejectedReasoning := parseOpenAIReasoningRejection(dataBytes); !rejectedReasoning { - if continuationErr := openAIContinuationStateErrorFromFailedEvent(resp.StatusCode, resp.Header, dataBytes); continuationErr != nil { - sawFailedEvent = true - streamEarlyErr = continuationErr - return - } - } - } + if !cyberPolicyHit && !outputStarted { if openAIStreamFailedEventShouldFailoverForAccount(account, dataBytes, failedMessage) { sawFailedEvent = true @@ -926,7 +904,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } clientOutputStarted = true lastDownstreamWriteAt = time.Now() - markOpenAIReasoningFailureTerminalForwarded(c) + } streamEarlyErr = fmt.Errorf("upstream response failed: %s", failedMessage) return @@ -1033,8 +1011,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. // OpenAI Responses streams that terminate with an empty // response.completed (no output, no usage, no error, nothing sent // to the client) are silent upstream refusals: fail over instead of - // recording a successful 0/0 usage turn (issue #5009). - if account != nil && account.Platform == PlatformOpenAI && + // recording a successful 0/0 usage turn (issue #5009). Grok + // subscriptions behind vendor gateways show the same silent-refusal + // shape on /v1/responses (forwardGrokResponses reaches this same + // handler), so cover PlatformGrok too — the passthrough-path guard + // is already unconditional. + if account != nil && (account.Platform == PlatformOpenAI || account.Platform == PlatformGrok) && (eventType == "response.completed" || eventType == "response.done") && !sawFailedEvent && !responsesSemanticOutputSeen && !clientOutputStarted && openAIResponsesCompletedEventIsEmpty(dataBytes, usage) { @@ -1773,9 +1755,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r if err != nil { return nil, err } - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, body, openAIStreamClientOutputStarted(c, false)); recoveryErr != nil { - return nil, recoveryErr - } + observer := upstreamResponseModelObserverFromContext(c) if observer == nil { observer = beginUpstreamResponseModelObservation(c) @@ -1928,43 +1908,6 @@ func openAIHTTPAuthoritativeTerminalEvent(payload []byte, eventType string) bool // disabled or already consumed. It must not fall into generic account-pool // retries. Once semantic bytes were committed the parser retains the genuine // failure event and returns a plain error instead of replaying. -func openAIHTTPReasoningRejectionBeforeOutput(c *gin.Context, payload []byte, semanticCommitted bool) error { - if bodyHasSSEFraming(payload) { - forEachOpenAISSEFrame(string(payload), func(_ string, data []byte) { - observeOpenAIReasoningAttemptUsage(c, data) - }) - } else { - observeOpenAIReasoningAttemptUsage(c, payload) - } - recovery := openAIReasoningRecoveryStateFromContext(c) - _, signatureRejected := parseOpenAIReasoningRejection(payload) - if recovery != nil && (signatureRejected || (gjson.ValidBytes(payload) && (gjson.GetBytes(payload, "type").String() == "error" || openAIHTTPResponseTerminalError(payload) != nil))) { - recovery.ObserveFailure(payload, semanticCommitted) - } - if semanticCommitted { - return nil - } - if recoveryErr := openAIReasoningRecoverySignal(c, payload, false); recoveryErr != nil { - return recoveryErr - } - if signatureRejected { - status, headers := http.StatusBadRequest, http.Header(nil) - if recovery != nil { - status, headers = recovery.upstreamStatus(status), recovery.responseHeaders - } - // ObserveFailure already retained the private payload for the bounded - // diagnostic. Keep the terminal's existing body-free error contract. - return NewOpenAIContinuationStateUnavailableError(status, headers, nil) - } - if recovery != nil { - // A retry's actual validation failure is still request-scoped. Handle it - // before generic stream-failure side effects can punish account health. - if rejected := recovery.requestRejectionFromStream(payload); rejected != nil { - return rejected - } - } - return nil -} func openAIHTTPResponseTerminalError(payload []byte) error { response := gjson.ParseBytes(payload) @@ -2071,12 +2014,6 @@ func bodyHasSSEFraming(body []byte) bool { } func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Context, account *Account, body []byte, originalModel, mappedModel string) (*openaiNonStreamingResult, error) { - rawTerminalType, rawTerminalPayload, rawTerminalOK := extractOpenAISSETerminalEvent(string(body)) - if rawTerminalOK && (rawTerminalType == "error" || rawTerminalType == "response.failed" || rawTerminalType == "response.done") { - if recoveryErr := openAIHTTPReasoningRejectionBeforeOutput(c, rawTerminalPayload, openAIStreamClientOutputStarted(c, false)); recoveryErr != nil { - return nil, recoveryErr - } - } body = restoreSystemPromptEchoSSE(c, body) bodyText := string(body) terminalType, terminalPayload, terminalOK := extractOpenAISSETerminalEvent(bodyText) diff --git a/backend/internal/service/openai_gateway_responses_anthropic_native.go b/backend/internal/service/openai_gateway_responses_anthropic_native.go index 00c165d04833..7181e080db09 100644 --- a/backend/internal/service/openai_gateway_responses_anthropic_native.go +++ b/backend/internal/service/openai_gateway_responses_anthropic_native.go @@ -242,7 +242,7 @@ func (s *OpenAIGatewayService) handleResponsesBufferedFromNativeAnthropic( } if event.Type == "content_block_delta" && event.Delta != nil && finalResp != nil && event.Index != nil { idx := *event.Index - if idx < len(finalResp.Content) { + if idx >= 0 && idx < len(finalResp.Content) { switch event.Delta.Type { case "text_delta": finalResp.Content[idx].Text += event.Delta.Text diff --git a/backend/internal/service/openai_gateway_responses_empty_completed_test.go b/backend/internal/service/openai_gateway_responses_empty_completed_test.go index 55bfda089ccd..c5989136cda5 100644 --- a/backend/internal/service/openai_gateway_responses_empty_completed_test.go +++ b/backend/internal/service/openai_gateway_responses_empty_completed_test.go @@ -166,3 +166,71 @@ func TestOpenAIResponsesCompletedEventIsEmpty(t *testing.T) { }) } } + +// TestOpenAIResponsesEmptyCompletedFailsOverGrok verifies that Grok platform +// accounts — which reach the stream relay through forwardGrokResponses instead +// of the passthrough path — get the same #5009 silent-refusal failover for an +// empty response.completed (issue #7774). +func TestOpenAIResponsesEmptyCompletedFailsOverGrok(t *testing.T) { + gin.SetMode(gin.TestMode) + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader( + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_grok\",\"object\":\"response\",\"status\":\"in_progress\"}}\n\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_grok\",\"object\":\"response\",\"status\":\"completed\",\"output\":[]}}\n\n", + )), + }} + svc := newOpenAIImageGenerationControlTestService(upstream) + c, recorder := newOpenAIImageGenerationControlTestContext(true, "codex_cli_rs/0.144.1") + account := newOpenAIImageGenerationControlTestAccount() + account.Platform = PlatformGrok + + body := []byte(`{ + "model":"grok-4.7", + "stream":true, + "input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]}] + }`) + + _, err := svc.Forward(context.Background(), c, account, body) + require.Error(t, err) + var failoverErr *UpstreamFailoverError + require.True(t, errors.As(err, &failoverErr), "empty completed must produce UpstreamFailoverError, got: %v", err) + require.Equal(t, http.StatusBadGateway, failoverErr.StatusCode) + require.Empty(t, recorder.Body.String(), "no empty success stream may reach the client") +} + +// TestOpenAIResponsesEmptyCompletedWithOutputSucceedsGrok ensures normal Grok +// streams with real semantic output still succeed. +func TestOpenAIResponsesEmptyCompletedWithOutputSucceedsGrok(t *testing.T) { + gin.SetMode(gin.TestMode) + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader( + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_grok_ok\",\"object\":\"response\",\"status\":\"in_progress\"}}\n\n" + + "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello grok\"}\n\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_grok_ok\",\"object\":\"response\",\"status\":\"completed\",\"usage\":{\"input_tokens\":11,\"output_tokens\":4,\"total_tokens\":15}}}\n\n", + )), + }} + svc := newOpenAIImageGenerationControlTestService(upstream) + c, recorder := newOpenAIImageGenerationControlTestContext(true, "codex_cli_rs/0.144.1") + account := newOpenAIImageGenerationControlTestAccount() + account.Platform = PlatformGrok + + body := []byte(`{ + "model":"grok-4.7", + "stream":true, + "input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]}] + }`) + + result, err := svc.Forward(context.Background(), c, account, body) + require.NoError(t, err) + require.NotNil(t, result) + require.Contains(t, recorder.Body.String(), "hello grok") + require.NotNil(t, result.Usage) + require.Equal(t, 11, result.Usage.InputTokens) + require.Equal(t, 4, result.Usage.OutputTokens) +} diff --git a/backend/internal/service/openai_gateway_scheduling.go b/backend/internal/service/openai_gateway_scheduling.go index 1657819ad2ec..8d618a196913 100644 --- a/backend/internal/service/openai_gateway_scheduling.go +++ b/backend/internal/service/openai_gateway_scheduling.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "github.com/gin-gonic/gin" @@ -265,18 +266,43 @@ func (s *OpenAIGatewayService) SelectAccountForModelWithExclusions(ctx context.C return s.selectAccountForModelWithExclusions(s.withOpenAIQuotaAutoPauseContext(ctx), groupID, PlatformOpenAI, sessionHash, requestedModel, excludedIDs, false, 0, "", false) } -// NormalizeOpenAICompatiblePlatform 保留 grok 与国产 OpenAI 兼容供应商(kimi/zhipu/ -// deepseek)的原值,其他值一律归一为 openai。调度器据此对账号与请求做精确平台匹配: +// SelectAccountForTokenCount selects an account for a non-billable token-count +// request. It applies the normal platform, model, capability, and runtime +// eligibility checks without acquiring or waiting for a generation slot. +func (s *OpenAIGatewayService) SelectAccountForTokenCount( + ctx context.Context, + groupID *int64, + sessionHash string, + requestedModel string, + requiredCapability OpenAIEndpointCapability, + platform string, +) (*Account, error) { + ctx = WithOpenAIProfitControlSuppressed(ctx) + ctx = s.withOpenAIQuotaAutoPauseContext(ctx) + return s.selectAccountForModelWithExclusions( + ctx, + groupID, + platform, + sessionHash, + requestedModel, + nil, + false, + 0, + requiredCapability, + false, + ) +} + +// NormalizeOpenAICompatiblePlatform 保留走 OpenAI 网关的平台(openai、grok 与多协议 +// API Key 供应商,见平台清单)的原值,其他值一律归一为 openai。调度器据此对账号与请求做精确平台匹配: // kimi 分组请求只命中 kimi 账号,语义与 openai/grok 一致。 // (upstream 曾将本函数改为未导出 normalizeOpenAICompatiblePlatform,本分支的 // handler 调度入口仍需导出,保持导出名。) func NormalizeOpenAICompatiblePlatform(platform string) string { - switch platform { - case PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo: + if domain.UsesOpenAIGateway(platform) { return platform - default: - return PlatformOpenAI } + return PlatformOpenAI } // noAvailableOpenAISelectionError builds the standard "no account available" error diff --git a/backend/internal/service/openai_gateway_service.go b/backend/internal/service/openai_gateway_service.go index e8a36b344ce5..fc6ec5607ce2 100644 --- a/backend/internal/service/openai_gateway_service.go +++ b/backend/internal/service/openai_gateway_service.go @@ -475,7 +475,6 @@ type OpenAIGatewayService struct { settingService *SettingService userPlatformQuotaRepo UserPlatformQuotaRepository systemPrompts *SystemPromptService - reasoningRecovery *ReasoningRecoveryService gatewayBorrow *CodexGatewayBorrowService usageCache *UsageCache usageCommitObserver UsageCommitObserver diff --git a/backend/internal/service/openai_gateway_service_hotpath_test.go b/backend/internal/service/openai_gateway_service_hotpath_test.go index 0c5800d40166..9ee07a509170 100644 --- a/backend/internal/service/openai_gateway_service_hotpath_test.go +++ b/backend/internal/service/openai_gateway_service_hotpath_test.go @@ -544,7 +544,7 @@ func TestOpenAIGatewayService_Forward_ImageToolWithImageOnlyModelIsNormalized(t require.Equal(t, openAIImagesResponsesMainModel, gjson.GetBytes(upstream.lastBody, "model").String()) } -func TestOpenAIGatewayService_Forward_HTTPInvalidEncryptedContentStopsWithoutMutatingRawInput(t *testing.T) { +func TestOpenAIGatewayService_Forward_HTTPRetryRecoveryDoesNotDecodeBeforeError(t *testing.T) { gin.SetMode(gin.TestMode) upstream := &httpUpstreamRecorder{ responses: []*http.Response{ @@ -580,26 +580,18 @@ func TestOpenAIGatewayService_Forward_HTTPInvalidEncryptedContentStopsWithoutMut c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - body := []byte(`{"model":"gpt-5","stream":false,"store":false,"previous_response_id":"resp_previous","input":[{"type":"reasoning","encrypted_content":"gAAA","summary":[{"type":"summary_text","text":"keep me"}]},{"type":"message","content":[{"type":"input_text","text":"hi","nonce":9007199254740993}]}]}`) + body := []byte(`{"model":"gpt-5","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA","summary":[{"type":"summary_text","text":"keep me"}]},{"type":"message","content":[{"type":"input_text","text":"hi","nonce":9007199254740993}]}]}`) result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Len(t, upstream.bodies, 1, "invalid encrypted history must not be stripped and retried") - require.JSONEq(t, string(body), string(upstream.bodies[0])) - require.Equal(t, "resp_previous", gjson.GetBytes(upstream.bodies[0], "previous_response_id").String()) + require.NoError(t, err) + require.NotNil(t, result) + require.Len(t, upstream.bodies, 2) require.Equal(t, "gAAA", gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").String()) require.Equal(t, "9007199254740993", gjson.GetBytes(upstream.bodies[0], "input.1.content.0.nonce").Raw) - require.Equal(t, "summary_text", gjson.GetBytes(upstream.bodies[0], "input.0.summary.0.type").String()) - require.Equal(t, "keep me", gjson.GetBytes(upstream.bodies[0], "input.0.summary.0.text").String()) - require.False(t, c.Writer.Written(), "the caller must receive the terminal error before any response is committed") - require.NotContains(t, rec.Body.String(), `"output_tokens":2`) + require.False(t, gjson.GetBytes(upstream.bodies[1], "input.0.encrypted_content").Exists()) + require.Equal(t, "summary_text", gjson.GetBytes(upstream.bodies[1], "input.0.summary.0.type").String()) } -func TestOpenAIGatewayService_Forward_HTTPInvalidEncryptedContentStopsWithoutDroppingCompaction(t *testing.T) { +func TestOpenAIGatewayService_Forward_HTTPRetryRecoveryDropsCompaction(t *testing.T) { gin.SetMode(gin.TestMode) upstream := &httpUpstreamRecorder{ responses: []*http.Response{ @@ -635,75 +627,14 @@ func TestOpenAIGatewayService_Forward_HTTPInvalidEncryptedContentStopsWithoutDro c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - body := []byte(`{"model":"gpt-5.6-sol","stream":false,"previous_response_id":"resp_previous","input":[{"id":"cmp_stale","type":"compaction","encrypted_content":"gAAA"},{"type":"message","content":[{"type":"input_text","text":"hi"}]}]}`) + body := []byte(`{"model":"gpt-5.6-sol","stream":false,"input":[{"id":"cmp_stale","type":"compaction","encrypted_content":"gAAA"},{"type":"message","content":[{"type":"input_text","text":"hi"}]}]}`) result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Len(t, upstream.bodies, 1, "compaction history must not be dropped and retried") - require.JSONEq(t, string(body), string(upstream.bodies[0])) - require.Equal(t, "resp_previous", gjson.GetBytes(upstream.bodies[0], "previous_response_id").String()) - require.Equal(t, "cmp_stale", gjson.GetBytes(upstream.bodies[0], "input.0.id").String()) + require.NoError(t, err) + require.NotNil(t, result) + require.Len(t, upstream.bodies, 2) require.Equal(t, "compaction", gjson.GetBytes(upstream.bodies[0], "input.0.type").String()) - require.Equal(t, "gAAA", gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").String()) - require.Equal(t, "message", gjson.GetBytes(upstream.bodies[0], "input.1.type").String()) - require.False(t, c.Writer.Written(), "the caller must receive the terminal error before any response is committed") - require.NotContains(t, rec.Body.String(), `"output_tokens":2`) -} - -func TestOpenAIGatewayService_Forward_HTTPWrappedInvalidEncryptedContentStopsWithoutReplayWhenRecoveryDisabled(t *testing.T) { - gin.SetMode(gin.TestMode) - upstream := &httpUpstreamRecorder{ - responses: []*http.Response{ - { - StatusCode: http.StatusNotFound, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_encrypted_content","type":"server_error","message":"encrypted content could not be verified"}}`)), - }, - { - StatusCode: http.StatusOK, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"usage":{"input_tokens":1,"output_tokens":2}}`)), - }, - }, - } - cfg := &config.Config{} - cfg.Security.URLAllowlist.Enabled = false - svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} - svc.SetReasoningRecoveryService(newReasoningRecoverySwitchForTest(false)) - account := &Account{ - ID: 12, - Name: "openai-apikey", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{ - "api_key": "sk-test", - "base_url": "https://example.com", - }, - Extra: map[string]any{"use_responses_api": true}, - } - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) - SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - - body := []byte(`{"model":"gpt-5","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA"},{"type":"message","content":"hi"}]}`) - result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Len(t, upstream.bodies, 1, "wrapped state errors must not cause history stripping or replay") - require.JSONEq(t, string(body), string(upstream.bodies[0])) - require.Equal(t, "gAAA", gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").String()) - require.False(t, c.Writer.Written(), "the caller must receive the terminal error before any response is committed") - require.NotContains(t, rec.Body.String(), `"output_tokens":2`) + require.Equal(t, "message", gjson.GetBytes(upstream.bodies[1], "input.0.type").String()) + require.False(t, gjson.GetBytes(upstream.bodies[1], "input.1").Exists()) } func TestOpenAIGatewayService_Forward_ResponseFailedInvalidEncryptedContentStopsWithoutReplay(t *testing.T) { @@ -812,45 +743,68 @@ func TestOpenAIGatewayService_Forward_StreamingResponseFailedInvalidEncryptedCon require.NotContains(t, rec.Body.String(), "[DONE]") } -func TestOpenAIGatewayService_Forward_ContinuationStateWithToolOutputIsNotReplayedOnTheAccount(t *testing.T) { +func TestOpenAIGatewayService_Forward_HTTPThinkingSignatureRecoveryAndLineage(t *testing.T) { gin.SetMode(gin.TestMode) - upstream := &httpUpstreamRecorder{ - resp: &http.Response{ - StatusCode: http.StatusBadGateway, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_encrypted_content","type":"server_error","message":"encrypted content could not be verified"}}`)), - }, + response := func(status int, body string) *http.Response { + return &http.Response{StatusCode: status, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(body))} } + upstream := &httpUpstreamRecorder{responses: []*http.Response{ + response(http.StatusBadRequest, `{"error":{"code":"thinking_signature_invalid","message":"The encrypted content abc could not be verified. Reason: Encrypted content could not be decrypted or parsed."}}`), + response(http.StatusOK, `{"id":"resp_recovered","usage":{"input_tokens":1,"output_tokens":2}}`), + response(http.StatusOK, `{"id":"resp_next","usage":{"input_tokens":1,"output_tokens":2}}`), + }} cfg := &config.Config{} cfg.Security.URLAllowlist.Enabled = false svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} - account := &Account{ - ID: 13, - Name: "openai-apikey", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{ - "api_key": "sk-test", - "base_url": "https://example.com", - }, - Extra: map[string]any{"use_responses_api": true}, + account := &Account{ID: 10, Name: "openai-apikey", Platform: PlatformOpenAI, Type: AccountTypeAPIKey, + Concurrency: 1, Credentials: map[string]any{"api_key": "sk-test", "base_url": "https://example.com"}, + Extra: map[string]any{"use_responses_api": true}} + newContext := func() *gin.Context { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) + SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) + return c } - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) - SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) + input := `{"model":"gpt-5","stream":false,"prompt_cache_key":"issue-7700","tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}],"input":[{"type":"reasoning","encrypted_content":"stale","summary":[{"type":"summary_text","text":"keep"}]},{"type":"message","role":"user","content":[{"type":"input_text","text":"hello"}]},{"type":"function_call_output","call_id":"call_1","output":"result"}]}` + first, err := svc.Forward(context.Background(), newContext(), account, []byte(input)) + require.NoError(t, err) + require.NotNil(t, first) + require.Len(t, upstream.bodies, 2) + require.Equal(t, "stale", gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").String()) + require.False(t, gjson.GetBytes(upstream.bodies[1], "input.0.encrypted_content").Exists()) + require.Equal(t, "keep", gjson.GetBytes(upstream.bodies[1], "input.0.summary.0.text").String()) + require.Equal(t, "hello", gjson.GetBytes(upstream.bodies[1], "input.1.content.0.text").String()) + require.Equal(t, "result", gjson.GetBytes(upstream.bodies[1], "input.2.output").String()) + require.Equal(t, "lookup", gjson.GetBytes(upstream.bodies[1], "tools.0.name").String()) + + second, err := svc.Forward(context.Background(), newContext(), account, []byte(input)) + require.NoError(t, err) + require.NotNil(t, second) + require.Len(t, upstream.bodies, 3, "later turn should be stripped before its first upstream request") + require.False(t, gjson.GetBytes(upstream.bodies[2], "input.0.encrypted_content").Exists()) + require.Equal(t, "result", gjson.GetBytes(upstream.bodies[2], "input.2.output").String()) +} - body := []byte(`{"model":"gpt-5","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA"},{"type":"function_call_output","call_id":"call_1","output":"ok"}]}`) - result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.True(t, failoverErr.IsOpenAICiphertextAccountMismatch(), "another account may accept the unchanged request") - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Len(t, upstream.bodies, 1, "tool-output continuation must not be rewritten or replayed") - require.Equal(t, "gAAA", gjson.GetBytes(upstream.bodies[0], "input.0.encrypted_content").String()) +func TestOpenAIGatewayService_Forward_HTTPThinkingSignatureDoesNotRetryUnrelatedErrors(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tc := range []struct{ name, code, message, input string }{ + {"no encrypted reasoning", "thinking_signature_invalid", "The encrypted content could not be verified. Reason: Encrypted content could not be decrypted or parsed.", `[{"type":"message","role":"user","content":"hello"}]`}, + {"ordinary 400", "invalid_request_error", "bad request", `[{"type":"reasoning","encrypted_content":"stale"},{"type":"message","content":"hello"}]`}, + {"unrelated signature error", "thinking_signature_invalid", "Invalid signature", `[{"type":"reasoning","encrypted_content":"stale"},{"type":"message","content":"hello"}]`}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := &httpUpstreamRecorder{resp: &http.Response{StatusCode: http.StatusBadRequest, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"` + tc.code + `","message":"` + tc.message + `"}}`))}} + cfg := &config.Config{} + cfg.Security.URLAllowlist.Enabled = false + svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} + account := &Account{ID: 10, Name: "openai-apikey", Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1, Credentials: map[string]any{"api_key": "sk-test", "base_url": "https://example.com"}, Extra: map[string]any{"use_responses_api": true}} + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) + SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) + _, _ = svc.Forward(context.Background(), c, account, []byte(`{"model":"gpt-5","stream":false,"input":`+tc.input+`}`)) + require.Len(t, upstream.bodies, 1) + }) + } } func TestOpenAIGatewayService_Forward_CodexSparkRejectsEscapedInputImage(t *testing.T) { diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index 6f614ac60a1b..c118a65bf91f 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "strings" - "time" "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" @@ -403,7 +402,7 @@ func classifyOpenAIContinuationStateError(upstreamMsg string, upstreamBody []byt // This new code is accepted only from a structured rejection envelope, never // from a body substring. An unusable signature is request state even when // recovery is disabled or cannot identify a safe, exact removal range. - if rejection, ok := parseOpenAIReasoningRejection(upstreamBody); ok && rejection.code == "thinking_signature_invalid" { + if gjson.GetBytes(upstreamBody, "error.code").String() == "thinking_signature_invalid" || gjson.GetBytes(upstreamBody, "response.error.code").String() == "thinking_signature_invalid" { return openAIContinuationStateErrorThinkingSignatureInvalid } const ( @@ -658,25 +657,9 @@ func (e *UpstreamFailoverError) IsOpenAIContinuationStateUnavailable() bool { // may still be accepted by another account. The caller tries the next account; // when none accepts, the failure renders with its own classification, exactly // as it would have without the mark. -func openAICiphertextAccountMismatch(failure *UpstreamFailoverError) *UpstreamFailoverError { - marked := *failure - marked.ResponseHeaders = failure.ResponseHeaders.Clone() - marked.CiphertextAccountMismatch = true - marked.NextAccountAction = NextAccountRetry - marked.Scope = GatewayFailureScopeRequest - marked.SuppressAccountHealthPenalty = true - marked.RetryableOnSameAccount = false - marked.SameAccountRetryDelay = 0 - marked.SameAccountRetryDeadline = time.Time{} - marked.SameAccountRetryMax = 0 - return &marked -} // IsOpenAICiphertextAccountMismatch reports a failure that another account // may not have for the same request. -func (e *UpstreamFailoverError) IsOpenAICiphertextAccountMismatch() bool { - return e != nil && e.CiphertextAccountMismatch -} // openAIRecoveryRetryAccountFailure marks the failure of a stripped retry that // is the account's own, such as exhausted credit or a provider error. The @@ -684,12 +667,6 @@ func (e *UpstreamFailoverError) IsOpenAICiphertextAccountMismatch() bool { // changes nothing else: classification, body and the fields account health // reads stay as the attempt built them. The mark alone tells the handler that // this account, whose one stripped retry is spent, is not asked again. -func openAIRecoveryRetryAccountFailure(failure *UpstreamFailoverError) *UpstreamFailoverError { - marked := *failure - marked.ResponseHeaders = failure.ResponseHeaders.Clone() - marked.RecoveryRetrySpent = true - return &marked -} // openAIRecoveryRetryHTTPFailure classifies an HTTP answer to the stripped // retry that a first send would move to another account, as that first send @@ -700,54 +677,18 @@ func openAIRecoveryRetryAccountFailure(failure *UpstreamFailoverError) *Upstream // It writes no account state and opens no same-account retry window. Whether // a first send would also have disabled the account, which makes it count a // pool-retryable status against the account, is not known here. -func (s *OpenAIGatewayService) openAIRecoveryRetryHTTPFailure( - account *Account, - statusCode int, - headers http.Header, - body []byte, - upstreamMsg string, - poolRetryable bool, -) *UpstreamFailoverError { - body = bytes.Clone(body) - switch { - case account != nil && account.IsOpenAICompatible() && isOpenAIBudgetExceededResponse(statusCode, body): - failure := newOpenAIUpstreamFailoverError(statusCode, headers, body, openAIBudgetExceededMessage(body), false) - failure.Scope = GatewayFailureScopeAccount - failure.NextAccountAction = NextAccountRetry - return failure - case isOpenAIRequestBudgetRejection(account, statusCode, body), - account != nil && account.IsOpenAICompatible() && isOpenAIModelNotSupportedError(statusCode, upstreamMsg, body): - return s.newOpenAIAccountFailoverError(account, statusCode, headers, body, upstreamMsg, false, false) - } - return newOpenAIUpstreamFailoverError(statusCode, headers, body, upstreamMsg, poolRetryable) -} // IsOpenAIRecoveryRetrySpent reports an account failure that ended an attempt // whose stripped retry was already sent. -func (e *UpstreamFailoverError) IsOpenAIRecoveryRetrySpent() bool { - return e != nil && e.RecoveryRetrySpent -} // openAIUpstreamReportsUndecryptableCiphertext reads an upstream failure for a // statement that ciphertext in the request could not be decrypted or verified. // Besides the continuation-state classification it accepts the wording used // for inter-agent message bodies and encrypted tool arguments, which arrives // without a stable code. -func openAIUpstreamReportsUndecryptableCiphertext(payload []byte) bool { - message := extractOpenAISSEErrorMessage(payload) - switch classifyOpenAIContinuationStateError(message, payload) { - case openAIContinuationStateErrorInvalidEncryptedContent, openAIContinuationStateErrorThinkingSignatureInvalid: - return true - } - return strings.Contains(strings.ToLower(message), "could not be decrypted") -} // openAIRequestHoldsServerContext reports a request whose history lives with // the upstream account that created it. -func openAIRequestHoldsServerContext(body []byte) bool { - conversation := gjson.GetBytes(body, "conversation") - return gjson.GetBytes(body, "previous_response_id").String() != "" || (conversation.Exists() && conversation.Type != gjson.Null) -} func marshalOpenAIUpstreamJSON(v any) ([]byte, error) { var buf bytes.Buffer diff --git a/backend/internal/service/openai_http_terminal_boundary_test.go b/backend/internal/service/openai_http_terminal_boundary_test.go index b305b7c1a072..c9585502b5e6 100644 --- a/backend/internal/service/openai_http_terminal_boundary_test.go +++ b/backend/internal/service/openai_http_terminal_boundary_test.go @@ -2,7 +2,6 @@ package service import ( "context" - "errors" "io" "net/http" "net/http/httptest" @@ -191,37 +190,6 @@ func TestOpenAIHTTPTerminalMissingOrMalformedNeverSucceeds(t *testing.T) { } } -func TestOpenAIHTTPTerminalRecoverySignalBeforeOutputOnly(t *testing.T) { - for _, mode := range []string{"native", "passthrough", "buffered", "passthrough_buffered"} { - for _, prefix := range []string{"", "data: {\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"visible reasoning\"}\n\n"} { - t.Run(mode+map[bool]string{true: "/preoutput", false: "/postoutput"}[prefix == ""], func(t *testing.T) { - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) - state := &openAIReasoningRecoveryState{ctx: context.Background(), enabled: true, wire: []byte(`{"input":[{"type":"reasoning","summary":[],"encrypted_content":"cipher_fixture"}]}`)} - c.Set(openAIReasoningRecoveryContextKey, state) - bare := `{"type":"error","error":{"code":"invalid_encrypted_content","param":"input[0].encrypted_content"}}` - failed := `{"type":"response.failed","response":{"status":"failed","error":{"code":"invalid_encrypted_content","param":"input[0].encrypted_content"},"usage":{"input_tokens":9,"output_tokens":3}}}` - body := prefix + "data: " + bare + "\n\ndata: " + failed + "\n\n" - usage, err := runOpenAIHTTPTerminalHandler(mode, c, io.NopCloser(strings.NewReader(body)), "text/event-stream") - require.Error(t, err) - var signal *openAIReasoningRecoverySignalError - wantSignal := prefix == "" || strings.Contains(mode, "buffered") - require.Equal(t, wantSignal, errors.As(err, &signal)) - if wantSignal { - require.Equal(t, "response.failed", gjson.GetBytes(signal.payload, "type").String()) - require.Equal(t, int64(9), gjson.GetBytes(signal.payload, "response.usage.input_tokens").Int()) - require.Empty(t, rec.Body.String()) - } else { - require.Equal(t, 9, usage.InputTokens) - require.Contains(t, rec.Body.String(), "visible reasoning") - } - require.False(t, state.retryUsed, "parsing a candidate must not consume the retry") - }) - } - } -} - func TestOpenAIHTTPTerminalBufferedBodyLimitAndCancellation(t *testing.T) { svc := &OpenAIGatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{UpstreamResponseReadMaxBytes: 32}}} resp := &http.Response{Body: io.NopCloser(strings.NewReader(strings.Repeat("x", 33)))} @@ -232,55 +200,6 @@ func TestOpenAIHTTPTerminalBufferedBodyLimitAndCancellation(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestOpenAIHTTPTerminalPreambleAndKeepaliveDoNotCommitRecovery(t *testing.T) { - for _, mode := range []string{"native", "native_async", "passthrough"} { - for _, platform := range []string{PlatformOpenAI} { - t.Run(mode+"/"+platform, func(t *testing.T) { - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) - // Only the neutral comment has actually been sent. Old response IDs - // and attempt-specific headers must remain private when recovering. - n, err := c.Writer.Write([]byte(":\n\n")) - require.NoError(t, err) - recordOpenAIStreamKeepaliveBytes(c, n) - state := &openAIReasoningRecoveryState{ctx: context.Background(), enabled: true, wire: []byte(`{"input":[{"type":"reasoning","summary":[],"encrypted_content":"cipher_fixture"}]}`)} - c.Set(openAIReasoningRecoveryContextKey, state) - body := "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"old_attempt\"}}\n\n" + - "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"thinking_signature_invalid\",\"param\":\"input[0].encrypted_content\"}}}\n\n" - _, err = runOpenAIHTTPTerminalHandler(mode, c, io.NopCloser(strings.NewReader(body)), "text/event-stream", &Account{ID: 1, Platform: platform, Type: AccountTypeAPIKey}) - var signal *openAIReasoningRecoverySignalError - require.ErrorAs(t, err, &signal) - require.Equal(t, ":\n\n", rec.Body.String()) - require.False(t, state.retryUsed) - }) - } - } -} - -func TestOpenAIHTTPTerminalRecoveryDisabledOrCanceledDoesNotRotate(t *testing.T) { - for _, disabled := range []bool{true, false} { - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - if !disabled { - cancel() - } - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) - c.Set(openAIReasoningRecoveryContextKey, &openAIReasoningRecoveryState{ctx: ctx, enabled: !disabled, wire: []byte(`{"input":[{"type":"reasoning","summary":[],"encrypted_content":"cipher_fixture"}]}`)}) - payload := []byte(`{"type":"response.failed","response":{"status":"failed","error":{"code":"thinking_signature_invalid","param":"input[0].encrypted_content"}}}`) - err := openAIHTTPReasoningRejectionBeforeOutput(c, payload, false) - var signal *openAIReasoningRecoverySignalError - require.False(t, errors.As(err, &signal)) - var terminal *UpstreamFailoverError - require.ErrorAs(t, err, &terminal) - require.False(t, terminal.ShouldRetryNextAccount()) - require.False(t, terminal.ShouldReportAccountScheduleFailure()) - require.Empty(t, terminal.ResponseBody) - } -} - func TestOpenAIHTTPTerminalReadErrorCannotDispatchUnfinishedFrame(t *testing.T) { for _, mode := range []string{"native", "native_async", "passthrough", "buffered", "passthrough_buffered"} { t.Run(mode, func(t *testing.T) { diff --git a/backend/internal/service/openai_lineage_regression_helpers_test.go b/backend/internal/service/openai_lineage_regression_helpers_test.go deleted file mode 100644 index b09bc258f1be..000000000000 --- a/backend/internal/service/openai_lineage_regression_helpers_test.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build unit - -package service - -import "strings" - -// markOpenAIWSInvalidEncryptedContentLineage 把本次被上游拒绝的密文摘要写入 -// 会话 lineage。digests 须在剥离前收集。 -func (s *OpenAIGatewayService) markOpenAIWSInvalidEncryptedContentLineage(groupID int64, sessionHash string, digests []string) { - if s == nil || len(digests) == 0 || strings.TrimSpace(sessionHash) == "" { - return - } - stateStore := s.getOpenAIWSStateStore() - if stateStore == nil { - return - } - stateStore.MarkSessionInvalidEncryptedContent(groupID, sessionHash, digests, s.openAIWSSessionStickyTTL()) -} - -// sessionInvalidEncryptedContentDigests 返回会话已知失效密文摘要;全局无记录 -// 时(常态)零成本返回 nil。 -func (s *OpenAIGatewayService) sessionInvalidEncryptedContentDigests(groupID int64, sessionHash string) map[string]struct{} { - if s == nil || strings.TrimSpace(sessionHash) == "" { - return nil - } - stateStore := s.getOpenAIWSStateStore() - if stateStore == nil || !stateStore.HasAnySessionInvalidEncryptedContent() { - return nil - } - return stateStore.GetSessionInvalidEncryptedContentDigests(groupID, sessionHash) -} diff --git a/backend/internal/service/openai_models_list.go b/backend/internal/service/openai_models_list.go index 8e480538a2ed..7929c7398dec 100644 --- a/backend/internal/service/openai_models_list.go +++ b/backend/internal/service/openai_models_list.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "sort" "strings" @@ -229,6 +230,10 @@ func projectAccountModelsBody(body []byte, account *Account, group *Group, codex continue } seen[id] = struct{}{} + if id == target { + projected = append(projected, raw) + continue + } var entry map[string]json.RawMessage if err := json.Unmarshal(raw, &entry); err != nil { return nil, err @@ -243,6 +248,9 @@ func projectAccountModelsBody(body []byte, account *Account, group *Group, codex } projected = append(projected, encoded) } + if slices.EqualFunc(entries, projected, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { + return body, nil + } envelope[field], err = json.Marshal(projected) if err != nil { return nil, err @@ -250,18 +258,21 @@ func projectAccountModelsBody(body []byte, account *Account, group *Group, codex return json.Marshal(envelope) } -// ApplyPinnedCodexModelsMapping is used by pinned discovery and its scheduler -// fallback. The ordinary (non-pinned) Codex path retains its local catalog policy. +// ApplyPinnedCodexModelsMapping projects all remotely discovered Codex catalogs, +// including ordinary discovery and pinned scheduler fallback. Locally generated +// catalogs retain their existing policy and do not pass through this function. func ApplyPinnedCodexModelsMapping(response *OpenAIModelsResponse, account *Account, group *Group) error { - if group == nil || group.Platform != PlatformOpenAI || !group.CodexModelsManifestConfig.Enabled { + if group == nil || group.Platform != PlatformOpenAI { return nil } body, err := projectAccountModelsBody(response.Body, account, group, true) if err != nil { return err } - response.Body = body - response.ETag = codexModelsManifestBodyETag(body) + if !bytes.Equal(response.Body, body) { + response.Body = body + response.ETag = codexModelsManifestBodyETag(body) + } return nil } diff --git a/backend/internal/service/openai_models_list_test.go b/backend/internal/service/openai_models_list_test.go index 08df6e249c90..edb50a569e50 100644 --- a/backend/internal/service/openai_models_list_test.go +++ b/backend/internal/service/openai_models_list_test.go @@ -3,6 +3,7 @@ package service import ( "context" "encoding/json" + "fmt" "io" "net/http" "strings" @@ -346,3 +347,18 @@ func TestOpenAIModelsCacheSeparatesRepresentationsForIdenticalRequests(t *testin require.JSONEq(t, manifestBody, string(manifest.Body)) require.EqualValues(t, 2, calls.Load()) } + +func TestApplyCodexModelsMappingPreservesUnchangedBodyAndETag(t *testing.T) { + for _, pinned := range []bool{false, true} { + t.Run(fmt.Sprint(pinned), func(t *testing.T) { + account := newCodexModelsAPIKeyTestAccount("https://models.example/v1") + account.Credentials["model_mapping"] = map[string]any{"gpt-5.4": "gpt-5.4"} + body := []byte(`{ "unknown": { "keep": true }, "models": [ { "slug": "gpt-5.4", "custom": 123 } ] }`) + response := &OpenAIModelsResponse{Body: body, ETag: `W/"upstream-validator"`} + group := &Group{Platform: PlatformOpenAI, CodexModelsManifestConfig: GroupCodexModelsManifestConfig{Enabled: pinned}} + require.NoError(t, ApplyPinnedCodexModelsMapping(response, account, group)) + require.Equal(t, body, response.Body) + require.Equal(t, `W/"upstream-validator"`, response.ETag) + }) + } +} diff --git a/backend/internal/service/openai_native_fidelity_test.go b/backend/internal/service/openai_native_fidelity_test.go index a13b776958a2..fe907c9ebfa3 100644 --- a/backend/internal/service/openai_native_fidelity_test.go +++ b/backend/internal/service/openai_native_fidelity_test.go @@ -39,8 +39,6 @@ func TestNativeResponsesFidelityPreservesCompatibleState(t *testing.T) { c, _ := gin.CreateTestContext(recorder) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - // Even a previous rejection marker is not authority to delete history. - svc.markOpenAIWSInvalidEncryptedContentLineage(getOpenAIGroupIDFromContext(c), svc.GenerateSessionHash(c, body), collectOpenAIEncryptedContentDigestsRaw(body)) result, err := svc.Forward(context.Background(), c, account, body) diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index b48bcf6b0db5..e9c80b660932 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -2122,113 +2122,6 @@ func TestOpenAIGatewayService_OpenAIPassthrough_InvalidEncryptedContentFirstErro require.Empty(t, repo.overloadCalls) } -func TestOpenAIGatewayService_OpenAIPassthrough_ContinuationErrorsHaveNoAccountSideEffects(t *testing.T) { - gin.SetMode(gin.TestMode) - tests := []struct { - name string - statusCode int - responseBody string - requestBody string - // accountMismatch: the rejection concerns ciphertext another account may - // read. A lost previous response, or a status that concerns the - // account's own access, ends the request on this account. - accountMismatch bool - }{ - { - name: "structured previous response 400", - statusCode: http.StatusBadRequest, - responseBody: `{"error":{"code":"previous_response_not_found","message":"previous response not found"}}`, - requestBody: `{"model":"gpt-5.2","stream":false,"previous_response_id":"resp_stale","input":"hi"}`, - }, - { - name: "structured invalid encrypted tool continuation 403", - statusCode: http.StatusForbidden, - responseBody: `{"response":{"error":{"code":"invalid_encrypted_content","message":"encrypted content could not be verified"}}}`, - requestBody: `{"model":"gpt-5.2","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA"},{"type":"function_call_output","call_id":"call_1","output":"ok"}]}`, - }, - { - name: "narrow previous response message 502", - statusCode: http.StatusBadGateway, - responseBody: `{"error":{"message":"Previous response not found"}}`, - requestBody: `{"model":"gpt-5.2","stream":false,"previous_response_id":"resp_stale","input":"hi"}`, - }, - { - name: "narrow invalid encrypted message 503", - statusCode: http.StatusServiceUnavailable, - responseBody: `{"error":{"message":"Encrypted content could not be verified"}}`, - requestBody: `{"model":"gpt-5.2","stream":false,"input":"hi"}`, - accountMismatch: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(nil)) - - repo := &openAIPassthroughFailoverRepo{} - upstream := &httpUpstreamRecorder{resp: &http.Response{ - StatusCode: tt.statusCode, - Header: http.Header{ - "Content-Type": []string{"application/json"}, - "X-Request-Id": []string{"rid-continuation"}, - }, - Body: io.NopCloser(strings.NewReader(tt.responseBody)), - }} - svc := &OpenAIGatewayService{ - cfg: &config.Config{Gateway: config.GatewayConfig{ForceCodexCLI: false}}, - httpUpstream: upstream, - rateLimitService: &RateLimitService{accountRepo: repo, cfg: &config.Config{}}, - } - account := &Account{ - ID: 132, - Name: "passthrough-continuation-terminal", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{"api_key": "sk-test", "base_url": "https://api.example.test"}, - Extra: map[string]any{"openai_passthrough": true}, - Status: StatusActive, - Schedulable: true, - } - - result, err := svc.Forward(context.Background(), c, account, []byte(tt.requestBody)) - - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.Equal(t, http.StatusBadRequest, failoverErr.ClientStatusCode) - require.Equal(t, GatewayFailureScopeRequest, failoverErr.Scope) - require.Equal(t, tt.accountMismatch, failoverErr.IsOpenAICiphertextAccountMismatch()) - require.Equal(t, tt.accountMismatch, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Len(t, upstream.bodies, 1, "unsafe continuation requests must not be replayed") - require.False(t, c.Writer.Written(), "continuation classification must happen before downstream response commitment") - require.Empty(t, repo.rateLimitCalls) - require.Empty(t, repo.overloadCalls) - - value, ok := c.Get(OpsUpstreamErrorsKey) - require.True(t, ok) - events, ok := value.([]*OpsUpstreamErrorEvent) - require.True(t, ok) - require.NotEmpty(t, events) - if tt.accountMismatch { - // The hand-off to another account is one more event of the attempt. - handOff := events[len(events)-1] - require.Equal(t, "reasoning_recovery", handOff.Kind) - require.True(t, handOff.ContinuationDiagnostic.Recovery.AccountMismatch) - events = events[:len(events)-1] - } - require.Equal(t, "continuation_state", events[len(events)-1].Kind) - require.True(t, events[len(events)-1].Passthrough) - require.Equal(t, openAIOpsUpstreamErrorMessage([]byte(tt.responseBody)), events[len(events)-1].Message) - require.NotEmpty(t, events[len(events)-1].Message, "Ops keeps the upstream text, not the client sentence") - }) - } -} - func TestOpenAIGatewayService_APIKeyPassthrough_Transient5xxTriggersFailover(t *testing.T) { gin.SetMode(gin.TestMode) requestBody := []byte(`{"model":"gpt-5.2","stream":false,"input":"hello"}`) diff --git a/backend/internal/service/openai_oauth_web_search_history.go b/backend/internal/service/openai_oauth_web_search_history.go new file mode 100644 index 000000000000..6a68eb719531 --- /dev/null +++ b/backend/internal/service/openai_oauth_web_search_history.go @@ -0,0 +1,264 @@ +package service + +import ( + "bytes" + "fmt" + "strings" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// The ChatGPT internal Codex endpoint rejects a request whose input replays a +// hosted web_search_call item unless the request also declares the web_search +// tool; the stream then fails with "response protection is unavailable". +// Codex's local context compaction sends the full history with tools:[], so +// every compaction after a web search fails (#7927). +// +// Declare a cached-only web_search tool for such requests. Responses Lite +// rejects hosted tools at the top level, so Lite requests carry it in an +// input additional_tools item instead. When the caller declared no tools at +// all, also pin tool_choice to "none" so the injected tool cannot be invoked +// and the request keeps its no-tools semantics. + +const ( + openAIWebSearchCallItemType = "web_search_call" + openAIAdditionalToolsItemType = "additional_tools" + openAICompactionTriggerItemType = "compaction_trigger" + openAIAdditionalToolsDefaultRole = "developer" +) + +var openAIWebSearchHistoryTool = map[string]any{ + "type": "web_search", + "external_web_access": false, +} + +func isOpenAIWebSearchToolType(toolType string) bool { + return strings.HasPrefix(strings.TrimSpace(toolType), "web_search") +} + +func openAIToolsContainWebSearch(rawTools any) bool { + tools, ok := rawTools.([]any) + if !ok { + return false + } + for _, rawTool := range tools { + tool, ok := rawTool.(map[string]any) + if ok && isOpenAIWebSearchToolType(firstNonEmptyString(tool["type"])) { + return true + } + } + return false +} + +// shouldPinOpenAIWebSearchHistoryToolChoice reports whether tool_choice may be +// forced to "none": only when the caller offered no tools, so the choice +// cannot meaningfully be anything else. +func shouldPinOpenAIWebSearchHistoryToolChoice(choice any) bool { + switch typed := choice.(type) { + case nil: + return true + case string: + normalized := strings.ToLower(strings.TrimSpace(typed)) + return normalized == "" || normalized == "auto" || normalized == "none" + default: + return false + } +} + +// openAIAdditionalToolsInsertIndex keeps a trailing compaction trigger last, +// as required by the remote compaction v2 wire format. +func openAIAdditionalToolsInsertIndex(itemTypes []string) int { + if n := len(itemTypes); n > 0 && itemTypes[n-1] == openAICompactionTriggerItemType { + return n - 1 + } + return len(itemTypes) +} + +// ensureOpenAIOAuthWebSearchToolForHistory is the map variant used by the +// Codex OAuth transform. +func ensureOpenAIOAuthWebSearchToolForHistory(reqBody map[string]any, responsesLite bool) bool { + if reqBody == nil { + return false + } + input, ok := reqBody["input"].([]any) + if !ok { + return false + } + hasWebSearchCall := false + callerDeclaredTools := false + additionalToolsIndex := -1 + itemTypes := make([]string, len(input)) + for i, rawItem := range input { + item, ok := rawItem.(map[string]any) + if !ok { + continue + } + itemTypes[i] = strings.TrimSpace(firstNonEmptyString(item["type"])) + switch itemTypes[i] { + case openAIWebSearchCallItemType: + hasWebSearchCall = true + case openAIAdditionalToolsItemType: + if openAIToolsContainWebSearch(item["tools"]) { + return false + } + if tools, _ := item["tools"].([]any); len(tools) > 0 { + callerDeclaredTools = true + } + if additionalToolsIndex < 0 { + additionalToolsIndex = i + } + } + } + if !hasWebSearchCall || openAIToolsContainWebSearch(reqBody["tools"]) { + return false + } + tools, _ := reqBody["tools"].([]any) + callerDeclaredTools = callerDeclaredTools || len(tools) > 0 + + switch { + case !responsesLite: + reqBody["tools"] = append(tools, cloneOpenAIWebSearchHistoryTool()) + case additionalToolsIndex >= 0: + // additionalToolsIndex is only recorded for map items. + item, _ := input[additionalToolsIndex].(map[string]any) + existing, _ := item["tools"].([]any) + item["tools"] = append(existing, cloneOpenAIWebSearchHistoryTool()) + default: + at := openAIAdditionalToolsInsertIndex(itemTypes) + additional := map[string]any{ + "type": openAIAdditionalToolsItemType, + "role": openAIAdditionalToolsDefaultRole, + "tools": []any{cloneOpenAIWebSearchHistoryTool()}, + } + next := make([]any, 0, len(input)+1) + next = append(next, input[:at]...) + next = append(next, additional) + next = append(next, input[at:]...) + reqBody["input"] = next + } + if !callerDeclaredTools { + if choice, exists := reqBody["tool_choice"]; !exists || shouldPinOpenAIWebSearchHistoryToolChoice(choice) { + reqBody["tool_choice"] = "none" + } + } + return true +} + +// ensureOpenAIOAuthWebSearchToolForHistoryBody is the raw-body variant used by +// the passthrough and WebSocket paths; it avoids decoding the whole body. +func ensureOpenAIOAuthWebSearchToolForHistoryBody(body []byte, responsesLite bool) ([]byte, bool, error) { + if len(body) == 0 || !bytes.Contains(body, []byte(openAIWebSearchCallItemType)) { + return body, false, nil + } + input := gjson.GetBytes(body, "input") + if !input.IsArray() { + return body, false, nil + } + items := input.Array() + hasWebSearchCall := false + callerDeclaredTools := false + additionalToolsIndex := -1 + itemTypes := make([]string, len(items)) + for i, item := range items { + itemTypes[i] = strings.TrimSpace(item.Get("type").String()) + switch itemTypes[i] { + case openAIWebSearchCallItemType: + hasWebSearchCall = true + case openAIAdditionalToolsItemType: + itemTools := item.Get("tools") + if gjsonToolsContainWebSearch(itemTools) { + return body, false, nil + } + if itemTools.IsArray() && len(itemTools.Array()) > 0 { + callerDeclaredTools = true + } + if additionalToolsIndex < 0 { + additionalToolsIndex = i + } + } + } + if !hasWebSearchCall { + return body, false, nil + } + tools := gjson.GetBytes(body, "tools") + if gjsonToolsContainWebSearch(tools) { + return body, false, nil + } + topLevelTools := tools.IsArray() && len(tools.Array()) > 0 + callerDeclaredTools = callerDeclaredTools || topLevelTools + + var ( + next []byte + err error + ) + switch { + case !responsesLite && topLevelTools: + next, err = sjson.SetBytes(body, "tools.-1", cloneOpenAIWebSearchHistoryTool()) + case !responsesLite: + next, err = sjson.SetBytes(body, "tools", []any{cloneOpenAIWebSearchHistoryTool()}) + case additionalToolsIndex >= 0 && items[additionalToolsIndex].Get("tools").IsArray(): + next, err = sjson.SetBytes(body, fmt.Sprintf("input.%d.tools.-1", additionalToolsIndex), cloneOpenAIWebSearchHistoryTool()) + case additionalToolsIndex >= 0: + next, err = sjson.SetBytes(body, fmt.Sprintf("input.%d.tools", additionalToolsIndex), []any{cloneOpenAIWebSearchHistoryTool()}) + default: + next, err = insertOpenAIAdditionalToolsItemRaw(body, items, openAIAdditionalToolsInsertIndex(itemTypes)) + } + if err != nil { + return body, false, fmt.Errorf("declare web_search tool for web_search_call history: %w", err) + } + if !callerDeclaredTools { + choice := gjson.GetBytes(next, "tool_choice") + if !choice.Exists() || choice.Type == gjson.Null || (choice.Type == gjson.String && shouldPinOpenAIWebSearchHistoryToolChoice(choice.String())) { + next, err = sjson.SetBytes(next, "tool_choice", "none") + if err != nil { + return body, false, fmt.Errorf("pin tool_choice for web_search_call history: %w", err) + } + } + } + return next, true, nil +} + +func insertOpenAIAdditionalToolsItemRaw(body []byte, items []gjson.Result, at int) ([]byte, error) { + additional, err := marshalOpenAIUpstreamJSON(map[string]any{ + "type": openAIAdditionalToolsItemType, + "role": openAIAdditionalToolsDefaultRole, + "tools": []any{cloneOpenAIWebSearchHistoryTool()}, + }) + if err != nil { + return nil, err + } + rawItems := make([][]byte, 0, len(items)+1) + for i, item := range items { + if i == at { + rawItems = append(rawItems, additional) + } + rawItems = append(rawItems, []byte(item.Raw)) + } + if at >= len(items) { + rawItems = append(rawItems, additional) + } + raw := append([]byte{'['}, bytes.Join(rawItems, []byte{','})...) + raw = append(raw, ']') + return sjson.SetRawBytes(body, "input", raw) +} + +func gjsonToolsContainWebSearch(tools gjson.Result) bool { + if !tools.IsArray() { + return false + } + for _, tool := range tools.Array() { + if isOpenAIWebSearchToolType(tool.Get("type").String()) { + return true + } + } + return false +} + +func cloneOpenAIWebSearchHistoryTool() map[string]any { + tool := make(map[string]any, len(openAIWebSearchHistoryTool)) + for key, value := range openAIWebSearchHistoryTool { + tool[key] = value + } + return tool +} diff --git a/backend/internal/service/openai_oauth_web_search_history_forward_test.go b/backend/internal/service/openai_oauth_web_search_history_forward_test.go new file mode 100644 index 000000000000..9735f179cefe --- /dev/null +++ b/backend/internal/service/openai_oauth_web_search_history_forward_test.go @@ -0,0 +1,71 @@ +//go:build unit + +package service + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// TestForward_OAuthWebSearchHistoryDeclaresTool drives the Codex compaction +// shape (web_search_call history, tools:[]) through Forward and asserts the +// body reaching chatgpt.com declares web_search (#7927): top-level for the +// standard wire, input additional_tools for Responses Lite. +func TestForward_OAuthWebSearchHistoryDeclaresTool(t *testing.T) { + for _, tc := range []struct { + name string + passthrough bool + lite bool + }{ + {"transform", false, false}, + {"passthrough", true, false}, + {"transform lite", false, true}, + {"passthrough lite", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + s := newAstraOAuthSetup(t, tc.passthrough) + if tc.lite { + s.c.Request.Header.Set(responsesLiteHeader, "true") + } + inner := `{"id":"resp_test","model":"gpt-5.5","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}` + s.upstream.resp = &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(codexCompletedSSE(inner))), + } + body, err := sjson.SetRawBytes([]byte(openAIWebSearchHistoryCompactionBody), "input.-1", []byte(`{"type":"compaction_trigger"}`)) + require.NoError(t, err) + + result, err := s.svc.Forward(context.Background(), s.c, s.account, body) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, s.upstream.lastReq) + require.Equal(t, astraProCodexResponsesURL, s.upstream.lastReq.URL.String()) + + forwarded := s.upstream.lastBody + require.Equal(t, "web_search_call", gjson.GetBytes(forwarded, "input.1.type").String()) + require.Equal(t, "none", gjson.GetBytes(forwarded, "tool_choice").String()) + items := gjson.GetBytes(forwarded, "input").Array() + require.Equal(t, "compaction_trigger", items[len(items)-1].Get("type").String()) + if !tc.lite { + tools := gjson.GetBytes(forwarded, "tools").Array() + require.Len(t, tools, 1) + require.Equal(t, "web_search", tools[0].Get("type").String()) + return + } + require.False(t, gjsonToolsContainWebSearch(gjson.GetBytes(forwarded, "tools"))) + additional := items[len(items)-2] + require.Equal(t, "additional_tools", additional.Get("type").String()) + require.Equal(t, "web_search", additional.Get("tools.0.type").String()) + }) + } +} diff --git a/backend/internal/service/openai_oauth_web_search_history_test.go b/backend/internal/service/openai_oauth_web_search_history_test.go new file mode 100644 index 000000000000..19b7ef0cfe71 --- /dev/null +++ b/backend/internal/service/openai_oauth_web_search_history_test.go @@ -0,0 +1,152 @@ +package service + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// Codex local compaction replays hosted web_search_call history with tools:[]; +// ChatGPT then fails with "response protection is unavailable" (#7927). +const openAIWebSearchHistoryCompactionBody = `{"model":"gpt-5.5","instructions":"x","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"look it up"}]},{"type":"web_search_call","id":"ws_123","status":"completed","action":{"type":"search","query":"kwin inputmethod"}},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"found"}]},{"type":"message","role":"user","content":[{"type":"input_text","text":"summarize"}]}],"tools":[],"tool_choice":"auto","parallel_tool_calls":false,"stream":true,"store":false}` + +func TestOAuthWebSearchHistoryToolAcrossPaths(t *testing.T) { + oauth := &Account{Platform: PlatformOpenAI, Type: AccountTypeOAuth} + transform := func(compact bool) func([]byte) ([]byte, bool, error) { + return func(b []byte) ([]byte, bool, error) { + var req map[string]any + if err := json.Unmarshal(b, &req); err != nil { + return nil, false, err + } + result := applyCodexOAuthTransform(req, true, compact) + out, err := json.Marshal(req) + return out, result.Modified, err + } + } + tests := []struct { + name string + normalize func([]byte) ([]byte, bool, error) + inject bool + }{ + {"transformed OAuth", transform(false), true}, + {"transformed OAuth compact", transform(true), false}, + {"OAuth websocket", func(b []byte) ([]byte, bool, error) { + return normalizeOpenAIResponsesWebSocketCompatibilityBody(b, oauth, false) + }, true}, + {"OAuth passthrough compact", func(b []byte) ([]byte, bool, error) { + return normalizeOpenAIResponsesCompatibilityBodyWithOptions(b, oauth, openAIResponsesCompatibilityOptions{Compact: true}) + }, false}, + {"API key websocket", func(b []byte) ([]byte, bool, error) { + return normalizeOpenAIResponsesWebSocketCompatibilityBody(b, &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, false) + }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, _, err := tt.normalize([]byte(openAIWebSearchHistoryCompactionBody)) + require.NoError(t, err) + require.Equal(t, "web_search_call", gjson.GetBytes(out, "input.1.type").String()) + if !tt.inject { + require.False(t, gjsonToolsContainWebSearch(gjson.GetBytes(out, "tools"))) + return + } + tools := gjson.GetBytes(out, "tools").Array() + require.Len(t, tools, 1) + require.Equal(t, "web_search", tools[0].Get("type").String()) + require.False(t, tools[0].Get("external_web_access").Bool()) + require.True(t, tools[0].Get("external_web_access").Exists()) + require.Equal(t, "none", gjson.GetBytes(out, "tool_choice").String()) + }) + } +} + +func TestEnsureOpenAIOAuthWebSearchToolForHistory(t *testing.T) { + const ( + user = `{"type":"message","role":"user","content":[{"type":"input_text","text":"q"}]}` + search = `{"type":"web_search_call","id":"ws_1","status":"completed","action":{"type":"search","query":"q"}}` + trigger = `{"type":"compaction_trigger"}` + history = `[` + user + `,` + search + `]` + ) + tests := []struct { + name string + body string + lite bool + changed bool + toolsPath string + toolCount int + toolChoice string + itemTypes []string + }{ + {"tools absent", `{"input":` + history + `}`, false, true, "tools", 1, "none", nil}, + {"tools null", `{"input":` + history + `,"tools":null,"tool_choice":null}`, false, true, "tools", 1, "none", nil}, + {"explicit none kept", `{"input":` + history + `,"tools":[],"tool_choice":"none"}`, false, true, "tools", 1, "none", nil}, + {"required not rewritten", `{"input":` + history + `,"tools":[],"tool_choice":"required"}`, false, true, "tools", 1, "required", nil}, + {"caller tools keep choice", `{"input":` + history + `,"tools":[{"type":"function","name":"echo"}],"tool_choice":"auto"}`, false, true, "tools", 2, "auto", nil}, + {"web_search already declared", `{"input":` + history + `,"tools":[{"type":"web_search"}]}`, false, false, "", 0, "", nil}, + {"web_search_preview declared", `{"input":` + history + `,"tools":[{"type":"web_search_preview"}]}`, false, false, "", 0, "", nil}, + {"declared via additional_tools", `{"input":[{"type":"additional_tools","tools":[{"type":"web_search"}]},` + search + `]}`, false, false, "", 0, "", nil}, + {"no web_search_call item", `{"input":[{"type":"message","role":"user","content":"web_search_call"}],"tools":[]}`, false, false, "", 0, "", nil}, + {"string input", `{"input":"web_search_call"}`, false, false, "", 0, "", nil}, + {"lite inserts before compaction trigger", `{"input":[` + user + `,` + search + `,` + trigger + `]}`, true, true, "input.2.tools", 1, "none", + []string{"message", "web_search_call", "additional_tools", "compaction_trigger"}}, + {"lite appends without trigger", `{"input":` + history + `,"tool_choice":"auto"}`, true, true, "input.2.tools", 1, "none", + []string{"message", "web_search_call", "additional_tools"}}, + {"lite extends caller additional_tools", `{"input":[{"type":"additional_tools","role":"developer","tools":[{"type":"function","name":"echo"}]},` + search + `,` + trigger + `],"tool_choice":"auto"}`, true, true, "input.0.tools", 2, "auto", + []string{"additional_tools", "web_search_call", "compaction_trigger"}}, + {"lite fills empty additional_tools", `{"input":[{"type":"additional_tools","role":"developer"},` + search + `]}`, true, true, "input.0.tools", 1, "none", + []string{"additional_tools", "web_search_call"}}, + {"lite keeps top-level tools untouched", `{"input":` + history + `,"tools":[{"type":"function","name":"echo"}]}`, true, true, "input.2.tools", 1, "", + []string{"message", "web_search_call", "additional_tools"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, changed, err := ensureOpenAIOAuthWebSearchToolForHistoryBody([]byte(tt.body), tt.lite) + require.NoError(t, err) + require.Equal(t, tt.changed, changed) + var req map[string]any + require.NoError(t, json.Unmarshal([]byte(tt.body), &req)) + require.Equal(t, tt.changed, ensureOpenAIOAuthWebSearchToolForHistory(req, tt.lite)) + if !tt.changed { + require.JSONEq(t, tt.body, string(out)) + return + } + require.True(t, gjson.ValidBytes(out)) + tools := gjson.GetBytes(out, tt.toolsPath).Array() + require.Len(t, tools, tt.toolCount) + require.Equal(t, "web_search", tools[len(tools)-1].Get("type").String()) + require.Equal(t, tt.toolChoice, gjson.GetBytes(out, "tool_choice").String()) + if tt.lite { + require.False(t, gjsonToolsContainWebSearch(gjson.GetBytes(out, "tools")), "Lite rejects top-level hosted tools") + var types []string + for _, item := range gjson.GetBytes(out, "input").Array() { + types = append(types, item.Get("type").String()) + } + require.Equal(t, tt.itemTypes, types) + } + + // The map variant used by the Codex transform must agree. + mapOut, err := json.Marshal(req) + require.NoError(t, err) + require.JSONEq(t, string(out), string(mapOut)) + }) + } +} + +func TestEnsureOpenAIOAuthWebSearchToolForHistoryDoesNotShareToolMap(t *testing.T) { + first := map[string]any{"input": []any{map[string]any{"type": "web_search_call"}}} + second := map[string]any{"input": []any{map[string]any{"type": "web_search_call"}}} + require.True(t, ensureOpenAIOAuthWebSearchToolForHistory(first, false)) + require.True(t, ensureOpenAIOAuthWebSearchToolForHistory(second, false)) + firstTools, ok := first["tools"].([]any) + require.True(t, ok) + firstTool, ok := firstTools[0].(map[string]any) + require.True(t, ok) + firstTool["external_web_access"] = true + secondTools, ok := second["tools"].([]any) + require.True(t, ok) + secondTool, ok := secondTools[0].(map[string]any) + require.True(t, ok) + require.Equal(t, false, secondTool["external_web_access"]) + require.Equal(t, false, openAIWebSearchHistoryTool["external_web_access"]) +} diff --git a/backend/internal/service/openai_reasoning_recovery.go b/backend/internal/service/openai_reasoning_recovery.go deleted file mode 100644 index fc1418fd10bf..000000000000 --- a/backend/internal/service/openai_reasoning_recovery.go +++ /dev/null @@ -1,1174 +0,0 @@ -package service - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "maps" - "net/http" - "slices" - "strconv" - "strings" - "time" - - "github.com/Wei-Shaw/sub2api/internal/pkg/logger" - "github.com/gin-gonic/gin" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - "go.uber.org/zap" -) - -const openAIReasoningRecoveryContextKey = "openai_reasoning_recovery_state" -const openAIReasoningRecoveryHeader = "X-Sub2API-Reasoning-Recovery" - -// The reasoning_recovery Ops detail is bounded by its encoded size: the Ops -// queue keeps at most OpsErrorLogQueueBodyMaxBytes per detail and shrinks a -// larger JSON detail to fields it does not know, which would lose everything. -// The upstream message and payload start at these caps and are halved until -// the encoded detail fits; action, items, usage and payload_sha256 always stay. -const ( - openAIReasoningRecoveryDetailEncodedLimit = 6 << 10 - openAIReasoningRecoveryMessageDetailLimit = 1 << 10 - openAIReasoningRecoveryPayloadDetailLimit = 4 << 10 -) - -// openAIReasoningRecoveryState owns the single, explicitly lossy retry for an -// HTTP Responses request on one account. It never re-enters a protocol -// converter or scheduler itself; when the account cannot serve the request -// because of ciphertext it cannot verify, StopError says so and the handler -// decides whether another account is tried. -// original is immutable; wire is the exact payload of the latest sent attempt. -// -// Apart from that retry it may re-send once more to undo a side effect of its -// own strip: an upstream that looks up by id an item left without ciphertext -// and does not find it (TryRepairUnfoundItemIDs). -type openAIReasoningRecoveryState struct { - ctx context.Context - c *gin.Context - account *Account - enabled bool - token string - original []byte - wire []byte - scope OpenAIReasoningCacheScope - store OpenAIReasoningStateStore - budget *OpenAIReasoningCacheBudget - - identity string - attemptUsage map[string]int64 - retryBody []byte - retryUsed bool - stopRecorded bool - retryCleanups []func() - - diagnosticIncoming []byte - diagnosticRequest *http.Request - responseStatus int - responseHeaders http.Header - failurePayload []byte - semanticCommitted bool - cacheSkippedItems int - diagnosticState string - retryDispatched bool - diagnosticStopReason string - failureObservedBytes int - failureTerminalForwarded bool - // redactPayload applies the upstream Agent Identity credential redaction - // before an upstream payload is recorded for administrators. - redactPayload func([]byte) []byte - - // switchAccounts is the global switch for this request, whether or not - // this account type supports the stripped retry. - switchAccounts bool - // accountMismatch records that this attempt ended as an account mismatch. - accountMismatch bool - // accountFailure records that the stripped retry failed for a reason of - // the account's own. - accountFailure bool - - // stripped lists the reasoning items whose ciphertext this attempt removed, - // by their position in the wire body. - stripped []openAIReasoningCipherItem - // dropItemIDs makes a strip remove the item's id with its ciphertext. It - // is set by the repair, for the rest of this attempt. - dropItemIDs bool - // idRepairUsed records the one re-send without the ids of stripped items; - // idRepairUnsent, that its body has not been sent yet. - idRepairUsed bool - idRepairUnsent bool - itemIDsRemoved int -} - -func (s *OpenAIGatewayService) newOpenAIReasoningRecoveryState(ctx context.Context, c *gin.Context, account *Account, token string) *openAIReasoningRecoveryState { - if ctx == nil { - ctx = context.Background() - } - // The global switch is read here, once for this forwarding attempt. - switchedOn := s.reasoningRecovery.Enabled() && !isOpenAICompatMessagesBridgeContext(c) - r := &openAIReasoningRecoveryState{ - ctx: ctx, c: c, account: account, token: token, - enabled: switchedOn && account.supportsOpenAIReasoningRecovery(), switchAccounts: switchedOn, - store: s.openAIReasoningStateStore(), budget: openAIReasoningCacheBudgetForRequest(c), - } - r.redactPayload = func(payload []byte) []byte { return s.redactAgentIdentitySensitiveBody(ctx, account, payload) } - if c != nil { - c.Set(openAIReasoningRecoveryContextKey, r) - } - return r -} - -// SetReasoningRecoveryService installs the global switch. Without one, recovery -// is enabled. -func (s *OpenAIGatewayService) SetReasoningRecoveryService(recovery *ReasoningRecoveryService) { - if s != nil { - s.reasoningRecovery = recovery - } -} - -func (r *openAIReasoningRecoveryState) Close() { - if r == nil { - return - } - for _, cleanup := range r.retryCleanups { - cleanup() - } - r.retryCleanups = nil -} - -func (r *openAIReasoningRecoveryState) RecoveryAttempt() bool { return r != nil && r.retryUsed } - -func openAIReasoningRecoveryStateFromContext(c *gin.Context) *openAIReasoningRecoveryState { - if c == nil { - return nil - } - value, _ := c.Get(openAIReasoningRecoveryContextKey) - state, _ := value.(*openAIReasoningRecoveryState) - return state -} - -// BindDiagnosticRequest retains only the forwarding-entry body and the exact -// request used by this attempt. The immutable wire snapshot already captured by -// PrepareRequest remains the source of truth after GetBody is disabled. -func (r *openAIReasoningRecoveryState) BindDiagnosticRequest(incoming []byte, req *http.Request) { - if r == nil { - return - } - if r.diagnosticIncoming == nil { - r.diagnosticIncoming = bytes.Clone(incoming) - } - r.diagnosticRequest = req - r.responseStatus, r.responseHeaders = 0, nil - r.failurePayload = nil - r.semanticCommitted = false - r.failureObservedBytes = 0 - r.failureTerminalForwarded = false -} - -func (r *openAIReasoningRecoveryState) ObserveResponse(resp *http.Response) { - if r == nil || resp == nil { - return - } - r.responseStatus, r.responseHeaders = resp.StatusCode, resp.Header.Clone() -} - -func (r *openAIReasoningRecoveryState) MarkAttemptDispatched() { - if r == nil { - return - } - r.idRepairUnsent = false - if r.retryUsed { - r.retryDispatched = true - r.diagnosticState = "retry_attempted" - } -} - -// ObserveFailure is observation only. A bare SSE error can be superseded by a -// completed response; neither the retry budget nor rejection cache changes here. -func (r *openAIReasoningRecoveryState) ObserveFailure(payload []byte, semanticCommitted bool) { - if r == nil { - return - } - r.failurePayload = bytes.Clone(payload) - r.semanticCommitted = semanticCommitted - r.failureTerminalForwarded = false - if r.c != nil && r.c.Writer != nil { - r.failureObservedBytes = max(0, OpenAICompactKeepaliveAdjustedWrittenSize(r.c)) - } -} - -// Called only after a stream parser has dispatched and flushed a complete -// failure terminal without a write error. Semantic output or an HTTP header -// alone is not evidence that the client has received the failure itself. -func markOpenAIReasoningFailureTerminalForwarded(c *gin.Context) { - r := openAIReasoningRecoveryStateFromContext(c) - if r == nil || r.diagnosticIncoming == nil || len(r.failurePayload) == 0 || c.Writer == nil { - return - } - if max(0, OpenAICompactKeepaliveAdjustedWrittenSize(c)) > r.failureObservedBytes { - r.failureTerminalForwarded = true - } -} - -// buildOpenAIReasoningScope identifies the source whose rejections are remembered. -// Credentials and route identity enter only a digest; neither is stored or logged. -func buildOpenAIReasoningScope(c *gin.Context, account *Account, req *http.Request, wireBody []byte) (OpenAIReasoningCacheScope, error) { - if c == nil || account == nil || req == nil || req.URL == nil { - return OpenAIReasoningCacheScope{}, errors.New("reasoning cache source unavailable") - } - key := getAPIKeyFromContext(c) - if key == nil { - return OpenAIReasoningCacheScope{}, errors.New("reasoning cache tenant unavailable") - } - groupID := int64(0) - if key.GroupID != nil { - groupID = *key.GroupID - } - proxyURL := "" - if account.ProxyID != nil && account.Proxy != nil { - proxyURL = account.Proxy.URL() - } - identity := openAIReasoningRequestIdentity(account, req, proxyURL, "") - return BuildOpenAIReasoningCacheScope(OpenAIReasoningScopeInput{ - UserID: key.UserID, APIKeyID: key.ID, GroupID: groupID, AccountID: account.ID, - SourceIdentityHash: identity, Endpoint: req.URL.String(), - Model: gjson.GetBytes(wireBody, "model").String(), - Reasoning: json.RawMessage(gjson.GetBytes(wireBody, "reasoning").Raw), - }) -} - -func openAIReasoningRequestIdentity(account *Account, req *http.Request, proxyURL, token string) string { - identity := struct { - AccountID int64 - AccountType, Endpoint, Host, Method, Proxy, Token string - Authorization, Organization, Project, ChatGPTAccount string - CredentialAccount, CredentialOrganization, CredentialProject string - }{ - AccountID: account.ID, AccountType: account.Type, Endpoint: req.URL.String(), - Host: req.Host, Method: req.Method, Proxy: proxyURL, Token: token, - Authorization: req.Header.Get("Authorization"), Organization: req.Header.Get("OpenAI-Organization"), - Project: req.Header.Get("OpenAI-Project"), ChatGPTAccount: req.Header.Get("ChatGPT-Account-ID"), - CredentialAccount: account.GetCredential("account_id"), CredentialOrganization: account.GetCredential("organization_id"), - CredentialProject: account.GetCredential("project_id"), - } - encoded, err := json.Marshal(identity) - if err != nil { - return "" - } - return openAIReasoningDigest(encoded) -} - -func openAIReasoningDigest(b []byte) string { - h := sha256.Sum256(b) - return hex.EncodeToString(h[:]) -} - -// PrepareRequest is called at the actual HTTP send boundary, after all normal -// transformations. Recovery compares the entire final body and source, so a -// later builder cannot silently remap or reinject state. -func (r *openAIReasoningRecoveryState) PrepareRequest(req *http.Request, body []byte, proxyURL string) (*http.Request, []byte, error) { - if r == nil || !r.enabled { - return req, body, nil - } - if req == nil || req.URL == nil || req.Method != http.MethodPost || - (!strings.HasSuffix(req.URL.Path, "/responses") && !strings.HasSuffix(req.URL.Path, "/responses/compact")) { - if r.retryUsed { - r.diagnosticStopReason = "endpoint_changed" - return nil, nil, r.StopError(errors.New("reasoning recovery endpoint changed")) - } - r.enabled = false - return req, body, nil - } - // NewRequest's GetBody reflects endpoint adaptations in the builder, not the - // caller's earlier request. Never infer the final wire input from that caller. - if req.GetBody != nil { - reader, err := req.GetBody() - if err != nil { - return nil, nil, errors.New("reasoning recovery request snapshot unavailable") - } - body, err = io.ReadAll(reader) - _ = reader.Close() - if err != nil { - return nil, nil, errors.New("reasoning recovery request snapshot unavailable") - } - } - identity := openAIReasoningRequestIdentity(r.account, req, proxyURL, r.token) - if identity == "" { - return nil, nil, errors.New("reasoning recovery source identity unavailable") - } - if r.retryUsed { - if r.ctx.Err() != nil { - r.diagnosticStopReason = "request_cancelled" - return nil, nil, r.StopError(r.ctx.Err()) - } - if identity != r.identity || !bytes.Equal(body, r.retryBody) { - r.diagnosticStopReason = "source_changed" - return nil, nil, r.StopError(errors.New("reasoning recovery source changed")) - } - // The normal gateway may detach cancellation to drain usage. An extra - // generation has no such permission: preserve the original deadline and - // cancellation while retaining transport-profile values on req.Context(). - retryCtx, cancel := context.WithCancel(req.Context()) - if deadline, ok := r.ctx.Deadline(); ok { - var deadlineCancel context.CancelFunc - retryCtx, deadlineCancel = context.WithDeadline(retryCtx, deadline) - r.retryCleanups = append(r.retryCleanups, deadlineCancel) - } - stop := context.AfterFunc(r.ctx, cancel) - r.retryCleanups = append(r.retryCleanups, func() { stop(); cancel() }) - req = req.Clone(retryCtx) - } else { - if r.original == nil { - r.original = bytes.Clone(body) - } - r.identity = identity - r.scope, _ = buildOpenAIReasoningScope(r.c, r.account, req, body) - body = r.skipRejectedHistory(body) - } - r.wire = bytes.Clone(body) - r.attemptUsage = nil - req = req.WithContext(WithHTTPUpstreamRedirectsDisabled(req.Context())) - if req.Body != nil { - _ = req.Body.Close() - } - req.Body = io.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - // The context flag blocks all redirect statuses, including 301/302/303's - // POST-to-GET conversion. Clearing GetBody also disallows transparent POST - // replay on a reused-connection error. - req.GetBody = nil - r.diagnosticRequest = req - return req, body, nil -} - -func (r *openAIReasoningRecoveryState) skipRejectedHistory(body []byte) []byte { - if r.store == nil || r.scope.ScopeHash == "" { - return body - } - if !openAIReasoningToolHistoryAllowsRecovery(body) { - return body - } - items := openAIReasoningCipherItems(body) - if len(items) == 0 { - return body - } - if _, err := canonicalReasoningCacheJSON(body); err != nil { - return body - } - hashes := make([]string, 0, len(items)) - for _, item := range items { - hashes = append(hashes, item.hash) - } - // The store takes a bounded number of hashes per call; a long history is - // looked up in batches inside the one I/O budget. - rejected := make(map[string]OpenAIRejectedReasoning, len(hashes)) - err := r.budget.Do(r.ctx, func(ioCtx context.Context) error { - for batch := range slices.Chunk(hashes, OpenAIReasoningStateMaxLookupEntries) { - found, readErr := r.store.GetOpenAIRejectedReasoning(ioCtx, r.scope, batch) - if readErr != nil { - return readErr - } - maps.Copy(rejected, found) - } - return nil - }) - if err != nil { - return body - } - now := time.Now() - skipped := make([]openAIReasoningCipherItem, 0, len(items)) - for _, item := range items { - if old, ok := rejected[item.hash]; ok && !old.RejectedAt.IsZero() && - !old.RejectedAt.After(now) && now.Before(old.ExpiresAt) && now.Before(old.RejectedAt.Add(OpenAIReasoningStateTTL)) { - skipped = append(skipped, item) - } - } - if len(skipped) == 0 { - return body - } - stripped, err := r.strip(body, skipped) - if err != nil { - return body - } - r.cacheSkippedItems += len(skipped) - r.record("rejected_history_skipped", 0, "", nil, len(skipped)) - return stripped -} - -// strip removes the ciphertext of items from body and remembers them as -// stripped by this attempt. After the repair the id goes with the ciphertext. -func (r *openAIReasoningRecoveryState) strip(body []byte, items []openAIReasoningCipherItem) ([]byte, error) { - edit := openAIReasoningRecoveryEdit{} - var sent []gjson.Result - if r.dropItemIDs { - sent = gjson.GetBytes(body, "input").Array() - } - for _, item := range items { - edit.cipher = append(edit.cipher, item.index) - if r.dropItemIDs && item.id != "" && item.index < len(sent) && openAIReasoningItemStandsWithoutID(sent[item.index]) { - edit.ids = append(edit.ids, item.index) - } - } - stripped, err := applyOpenAIReasoningRecoveryEdit(body, edit) - if err != nil { - return nil, err - } - r.stripped = append(r.stripped, items...) - r.itemIDsRemoved += len(edit.ids) - return stripped, nil -} - -type openAIReasoningCipherItem struct { - index int - hash string - id string -} - -func openAIReasoningCipherItems(body []byte) []openAIReasoningCipherItem { - input := gjson.GetBytes(body, "input") - if !input.IsArray() { - return nil - } - var out []openAIReasoningCipherItem - for index, item := range input.Array() { - cipher := item.Get("encrypted_content") - if item.Get("type").String() == "reasoning" && cipher.Type == gjson.String && cipher.String() != "" { - entry := openAIReasoningCipherItem{index: index, hash: openAIReasoningDigest([]byte(cipher.String()))} - if id := item.Get("id"); id.Type == gjson.String { - entry.id = id.String() - } - out = append(out, entry) - } - } - return out -} - -func openAIReasoningToolHistoryAllowsRecovery(body []byte) bool { - var input []any - if err := decodeOpenAIJSONUseNumber([]byte(gjson.GetBytes(body, "input").Raw), &input); err != nil { - return false - } - conversation := gjson.GetBytes(body, "conversation") - hasServerContext := gjson.GetBytes(body, "previous_response_id").String() != "" || (conversation.Exists() && conversation.Type != gjson.Null) - return validateOpenAIResponsesToolOutputs(input, hasServerContext) == nil -} - -func openAIReasoningRejectedIndices(body []byte, rejection openAIReasoningRejection) ([]int, []string) { - indices, _ := selectOpenAIRecoveryIndices(body, rejection) - if len(indices) == 0 { - return nil, nil - } - byIndex := map[int]string{} - for _, item := range openAIReasoningCipherItems(body) { - byIndex[item.index] = item.hash - } - hashes := make([]string, 0, len(indices)) - seen := map[string]bool{} - for _, index := range indices { - hash := byIndex[index] - if !seen[hash] { - hashes = append(hashes, hash) - seen[hash] = true - } - } - return indices, hashes -} - -// Evidence preserves numeric field presence, not default-zero struct members. -func openAIReasoningUsageEvidence(payload []byte) map[string]int64 { - var result map[string]int64 - for _, prefix := range []string{"usage", "response.usage"} { - usage := gjson.GetBytes(payload, prefix) - if !usage.IsObject() { - continue - } - for field, path := range map[string]string{ - "input_tokens": "input_tokens", "output_tokens": "output_tokens", "total_tokens": "total_tokens", - "cached_tokens": "input_tokens_details.cached_tokens", "reasoning_tokens": "output_tokens_details.reasoning_tokens", - "cache_creation_input_tokens": "cache_creation_input_tokens", "cache_read_input_tokens": "cache_read_input_tokens", - } { - value := usage.Get(path) - if value.Type != gjson.Number { - continue - } - count, err := strconv.ParseInt(value.Raw, 10, 64) - if err != nil || count < 0 { - continue - } - if result == nil { - result = make(map[string]int64) - } - result[field] = count - } - } - return result -} - -func observeOpenAIReasoningAttemptUsage(c *gin.Context, payload []byte) { - if c == nil { - return - } - value, _ := c.Get(openAIReasoningRecoveryContextKey) - state, _ := value.(*openAIReasoningRecoveryState) - if state == nil || !state.enabled { - return - } - if usage := openAIReasoningUsageEvidence(payload); len(usage) > 0 { - state.attemptUsage = usage - } -} - -func countOpenAIEncryptedFields(value any) int { - count := 0 - switch v := value.(type) { - case map[string]any: - for key, child := range v { - if key == "encrypted_content" && child != nil { - count++ - } - count += countOpenAIEncryptedFields(child) - } - case []any: - for _, child := range v { - count += countOpenAIEncryptedFields(child) - } - } - return count -} - -func stripOpenAIReasoningCipherIndices(body []byte, indices []int) ([]byte, error) { - return deleteOpenAIReasoningItemField(body, indices, "encrypted_content") -} - -// deleteOpenAIReasoningItemField deletes one field from the reasoning items at -// the listed input positions. A long history holds hundreds of them, and -// editing the whole body once for each costs seconds, so the items are edited -// on their own and the body is written once. For what its callers pass, valid -// JSON with one input and positions of that input, the result is byte for byte -// that of deleting the fields one by one. -func deleteOpenAIReasoningItemField(body []byte, indices []int, field string) ([]byte, error) { - input := gjson.GetBytes(body, "input") - if len(indices) == 0 || !input.IsArray() { - return deleteOpenAIReasoningItemFieldOneByOne(body, indices, field) - } - listed := make(map[int]bool, len(indices)) - for _, index := range indices { - if listed[index] { - // Listed twice, an item is edited twice; no caller does that. - return deleteOpenAIReasoningItemFieldOneByOne(body, indices, field) - } - listed[index] = true - } - out := make([]byte, 0, len(body)) - position, copied, met, located := 0, 0, 0, true - var failed error - input.ForEach(func(_, item gjson.Result) bool { - if listed[position] { - if item.Get("type").String() != "reasoning" { - failed = errors.New("reasoning recovery target mismatch") - return false - } - end := item.Index + len(item.Raw) - if item.Index < copied || end > len(body) || string(body[item.Index:end]) != item.Raw { - // The item is not where its offset says: edit the slow way. - located = false - return false - } - edited, err := sjson.Delete(item.Raw, field) - if err != nil { - failed = err - return false - } - out = append(out, body[copied:item.Index]...) - out = append(out, edited...) - copied = end - met++ - } - position++ - return true - }) - switch { - case failed != nil: - return nil, failed - case !located: - return deleteOpenAIReasoningItemFieldOneByOne(body, indices, field) - case met != len(listed): - return nil, errors.New("reasoning recovery target mismatch") - } - return append(out, body[copied:]...), nil -} - -func deleteOpenAIReasoningItemFieldOneByOne(body []byte, indices []int, field string) ([]byte, error) { - out := bytes.Clone(body) - for _, index := range indices { - path := fmt.Sprintf("input.%d", index) - if gjson.GetBytes(out, path+".type").String() != "reasoning" { - return nil, errors.New("reasoning recovery target mismatch") - } - var err error - out, err = sjson.DeleteBytes(out, path+"."+field) - if err != nil { - return nil, err - } - } - return out, nil -} - -// openAIReasoningRecoveryEdit is everything recovery may do to a request: at -// the listed input positions, all reasoning items, delete encrypted_content -// and delete id. Positions never move. -type openAIReasoningRecoveryEdit struct{ cipher, ids []int } - -// applyOpenAIReasoningRecoveryEdit makes the edit in one fixed order, so that -// the same edit yields the same bytes wherever it is repeated. -func applyOpenAIReasoningRecoveryEdit(body []byte, edit openAIReasoningRecoveryEdit) ([]byte, error) { - out, err := deleteOpenAIReasoningItemField(body, edit.cipher, "encrypted_content") - if err != nil { - return nil, err - } - return deleteOpenAIReasoningItemField(out, edit.ids, "id") -} - -// openAIReasoningItemHasCipher is the test openAIReasoningCipherItems applies. -func openAIReasoningItemHasCipher(item gjson.Result) bool { - cipher := item.Get("encrypted_content") - return cipher.Type == gjson.String && cipher.String() != "" -} - -// openAIReasoningItemStandsWithoutID reports an item the upstream takes as it -// is once its id is gone. An item without a summary is refused then, so such -// an item keeps its id. -func openAIReasoningItemStandsWithoutID(item gjson.Result) bool { - return item.Get("summary").IsArray() -} - -// Signal is deliberately pure. A bare SSE error may be superseded by a later -// authoritative failed/completed event and must not consume retry or cache state. -func openAIReasoningRecoverySignal(c *gin.Context, payload []byte, semanticCommitted bool) error { - if c == nil || semanticCommitted { - return nil - } - v, _ := c.Get(openAIReasoningRecoveryContextKey) - r, _ := v.(*openAIReasoningRecoveryState) - if r == nil || !r.enabled || r.retryUsed || r.ctx.Err() != nil { - return nil - } - rejection, ok := parseOpenAIReasoningRejection(payload) - if !ok { - return nil - } - indices, _ := openAIReasoningRejectedIndices(r.wire, rejection) - if len(indices) == 0 { - return nil - } - return &openAIReasoningRecoverySignalError{payload: bytes.Clone(payload)} -} - -type openAIReasoningRecoverySignalError struct{ payload []byte } - -func (*openAIReasoningRecoverySignalError) Error() string { - return "reasoning signature recovery eligible before semantic output" -} - -func (r *openAIReasoningRecoveryState) TryRecoverError(err error) ([]byte, bool) { - var signal *openAIReasoningRecoverySignalError - if r == nil || !errors.As(err, &signal) { - return nil, false - } - return r.TryRecover(r.upstreamStatus(http.StatusBadRequest), r.responseHeaders, signal.payload, false) -} - -func (r *openAIReasoningRecoveryState) TryRecover(status int, headers http.Header, payload []byte, semanticCommitted bool) ([]byte, bool) { - if r != nil { - r.ObserveFailure(payload, semanticCommitted) - if r.responseStatus == 0 { - r.responseStatus, r.responseHeaders = status, headers.Clone() - } - } - if r == nil || !r.enabled || r.retryUsed || semanticCommitted || r.ctx.Err() != nil || openAIReasoningRecoveryProtectedStatus(status) { - return nil, false - } - rejection, ok := parseOpenAIReasoningRejection(payload) - if !ok { - return nil, false - } - indices, hashes := openAIReasoningRejectedIndices(r.wire, rejection) - if len(indices) == 0 { - return nil, false - } - selected := make([]openAIReasoningCipherItem, 0, len(indices)) - for _, item := range openAIReasoningCipherItems(r.wire) { - if slices.Contains(indices, item.index) { - selected = append(selected, item) - } - } - if len(selected) != len(indices) { - return nil, false - } - stripped, err := r.strip(r.wire, selected) - if err != nil { - return nil, false - } - r.retryUsed = true - r.retryBody = bytes.Clone(stripped) - r.diagnosticState = "retry_prepared" - if r.store != nil && r.scope.ScopeHash != "" { - _ = r.budget.Do(r.ctx, func(ioCtx context.Context) error { - for batch := range slices.Chunk(hashes, OpenAIReasoningStateMaxLookupEntries) { - if err := r.store.PutOpenAIRejectedReasoning(ioCtx, r.scope, batch); err != nil { - return err - } - } - return nil - }) - } - r.record("retry_without_encrypted_content", status, rejection.code, payload, len(indices)) - return stripped, true -} - -// TryRepairUnfoundItemIDs undoes a side effect of this attempt's own strip. A -// reasoning item left without ciphertext still carries its id, and an upstream -// may take such an item for a reference to one it stores. When it does not -// find it, it says so and names the id (officially with status 404); without -// the id it takes the item as it is. The repair is made only for that answer -// naming an item this attempt stripped, removes the id of every such item, and -// is re-sent once. It is not the stripped retry and does not spend it. -// -// Nothing is remembered: every attempt that meets the answer repairs again. -func (r *openAIReasoningRecoveryState) TryRepairUnfoundItemIDs(status int, headers http.Header, payload []byte) ([]byte, bool) { - if r == nil || !r.enabled || r.idRepairUsed || len(r.stripped) == 0 || r.ctx.Err() != nil || - status < http.StatusBadRequest || openAIReasoningRecoveryProtectedStatus(status) { - return nil, false - } - // The official answer to an item looked up by id and not found. A relay may - // pass it on under another status; the named id is what identifies it. - message := extractUpstreamErrorMessage(payload) - if !strings.Contains(message, "Item with id '") { - return nil, false - } - input := gjson.GetBytes(r.wire, "input") - if !input.IsArray() { - return nil, false - } - items := input.Array() - found := false - ids := make([]int, 0, len(r.stripped)) - for _, item := range r.stripped { - if item.index >= len(items) { - continue - } - // What was stripped is checked against what was sent. - sent := items[item.index] - id := sent.Get("id") - if item.id == "" || sent.Get("type").String() != "reasoning" || openAIReasoningItemHasCipher(sent) || - id.Type != gjson.String || id.String() != item.id || !openAIReasoningItemStandsWithoutID(sent) || - slices.Contains(ids, item.index) { - continue - } - ids = append(ids, item.index) - found = found || strings.Contains(message, "Item with id '"+item.id+"' not found") - } - if !found { - return nil, false - } - repaired, err := applyOpenAIReasoningRecoveryEdit(r.wire, openAIReasoningRecoveryEdit{ids: ids}) - if err != nil { - return nil, false - } - r.idRepairUsed, r.idRepairUnsent, r.dropItemIDs = true, true, true - r.itemIDsRemoved += len(ids) - if r.retryUsed { - // The stripped retry was already sent; the send boundary accepts only - // the body it knows. - r.retryBody = bytes.Clone(repaired) - } - // The answer is recorded with its own status and request id, and leaves - // neither behind for a caller that does not observe each response. - observedStatus, observedHeaders := r.responseStatus, r.responseHeaders - r.responseStatus, r.responseHeaders = status, headers.Clone() - r.record("retry_without_item_ids", status, "", payload, len(ids)) - r.responseStatus, r.responseHeaders = observedStatus, observedHeaders - return repaired, true -} - -func openAIReasoningRecoveryProtectedStatus(status int) bool { - switch status { - case http.StatusUnauthorized, http.StatusPaymentRequired, http.StatusForbidden, http.StatusProxyAuthRequired, http.StatusTooManyRequests: - return true - default: - return false - } -} - -// OpenAIReasoningRecoveryTerminalError preserves classification for rendering -// and account-health accounting, but deliberately does not implement Unwrap. -// A caller must handle this terminal explicitly, never feed Failure back into -// scheduling: the single same-source recovery budget has already been spent. -// Two outcomes are not terminals, and StopError returns the attempt's failure -// marked for the next account instead: an account mismatch -// (openAICiphertextAccountMismatch) and a stripped retry that failed for a -// reason of the account's own (openAIRecoveryRetryAccountFailure). -type OpenAIReasoningRecoveryTerminalError struct { - Failure *UpstreamFailoverError - // FailureTerminalForwarded is explicit parser/write evidence, not a claim - // inferred from semantic output or Writer.Written. Callers still finalize - // accounting and health, but must not append a second response.failed. - FailureTerminalForwarded bool -} - -func (*OpenAIReasoningRecoveryTerminalError) Error() string { - return "reasoning signature recovery stopped; no further upstream retry permitted" -} - -func newOpenAIReasoningRecoveryTerminalError(failure *UpstreamFailoverError) *OpenAIReasoningRecoveryTerminalError { - if failure == nil { - failure = &UpstreamFailoverError{StatusCode: http.StatusBadGateway} - } - copyFailure := *failure - copyFailure.ResponseBody = nil - copyFailure.ResponseHeaders = failure.ResponseHeaders.Clone() - copyFailure.NextAccountAction = NextAccountStop - copyFailure.RetryableOnSameAccount = false - copyFailure.SameAccountRetryDelay = 0 - copyFailure.SameAccountRetryDeadline = time.Time{} - copyFailure.SameAccountRetryMax = 0 - return &OpenAIReasoningRecoveryTerminalError{Failure: ©Failure} -} - -// FailureForResponse preserves the rejection's existing request/provider -// classification without executing another compatibility retry or scheduler. -func (r *openAIReasoningRecoveryState) FailureForResponse(status int, headers http.Header, payload []byte) *UpstreamFailoverError { - message := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(payload))) - if classifyOpenAIContinuationStateError(message, payload) != openAIContinuationStateErrorNone { - return NewOpenAIContinuationStateUnavailableError(status, headers, bytes.Clone(payload)) - } - if classifyOpenAIRequestRejection(status, message, payload) != "" { - return NewOpenAIRequestRejectedError(status, headers) - } - return newOpenAIUpstreamFailoverError(status, headers, bytes.Clone(payload), message, false) -} - -// StopError never erases request-level semantics in order to forbid retries. -// It also captures signature exits where no safe recovery was possible; those -// used to return only the fixed client message and lose the actual rejection. -func (r *openAIReasoningRecoveryState) StopError(err error) error { - if err == nil || r == nil { - return err - } - if IsCodexGatewayBorrowRequestFailure(err) { - return err - } - if r.diagnosticIncoming == nil { - // Protocol-conversion callers have their own error contract. This repair - // opts in only at native Responses/passthrough send boundaries. - if !r.retryUsed { - return err - } - if !r.stopRecorded { - r.stopRecorded = true - r.record("recovery_failed", 0, "", nil, 0) - } - return errors.New("reasoning signature recovery failed; no further upstream retry permitted") - } - var stopped *OpenAIReasoningRecoveryTerminalError - if errors.As(err, &stopped) { - return err - } - // returned is the failure the attempt itself produced, if any; failure may - // be replaced below by one built from the observed payload. - var returned *UpstreamFailoverError - errors.As(err, &returned) - var failure *UpstreamFailoverError - var signal *openAIReasoningRecoverySignalError - if errors.As(err, &signal) { - r.ObserveFailure(signal.payload, false) - failure = NewOpenAIContinuationStateUnavailableError(r.upstreamStatus(http.StatusBadRequest), r.responseHeaders, signal.payload) - } else if errors.As(err, &failure) && len(failure.ResponseBody) > 0 && len(r.failurePayload) == 0 { - r.ObserveFailure(failure.ResponseBody, r.semanticCommitted) - } - rejection, signatureRejected := parseOpenAIReasoningRejection(r.failurePayload) - if failure == nil { - failure = r.requestRejectionFromStream(r.failurePayload) - } - cacheSkipRejected := r.cacheSkippedItems > 0 && failure != nil && failure.IsOpenAIRequestRejected() - r.accountMismatch = r.anotherAccountMayAccept(failure, signatureRejected) - r.accountFailure = !r.accountMismatch && r.accountFailedAfterRetry(returned) - if !r.retryUsed && !signatureRejected && !cacheSkipRejected && !r.accountMismatch { - if r.idRepairUnsent && !r.stopRecorded { - // A cached pre-strip needs no stripped retry. Keep the builder's - // error contract, but record that its prepared id repair was not sent. - r.stopRecorded = true - r.record("recovery_not_attempted", 0, "", nil, 0) - } - return err - } - if failure == nil { - if signatureRejected { - failure = NewOpenAIContinuationStateUnavailableError(r.upstreamStatus(http.StatusBadRequest), r.responseHeaders, nil) - } else { - failure = &UpstreamFailoverError{StatusCode: r.upstreamStatus(http.StatusBadGateway), ResponseHeaders: r.responseHeaders.Clone()} - } - } - if !r.stopRecorded { - r.stopRecorded = true - action := "recovery_not_attempted" - r.diagnosticState = "not_attempted" - if r.retryUsed { - action = "recovery_failed" - r.diagnosticState = "budget_exhausted" - } - r.record(action, r.upstreamStatus(failure.StatusCode), rejection.code, r.failurePayload, 0) - } - if r.accountMismatch { - return openAICiphertextAccountMismatch(failure) - } - if r.accountFailure { - handOff := openAIRecoveryRetryAccountFailure(returned) - if r.redactPayload != nil && len(handOff.ResponseBody) > 0 { - handOff.ResponseBody = r.redactPayload(handOff.ResponseBody) - } - return handOff - } - if !r.retryUsed && !r.semanticCommitted { - return failure - } - terminal := newOpenAIReasoningRecoveryTerminalError(failure) - terminal.FailureTerminalForwarded = r.failureTerminalForwarded - return terminal -} - -// anotherAccountMayAccept decides whether this attempt ended as an account -// mismatch: the upstream said it could not decrypt or verify ciphertext in the -// request, on the first send or on the stripped one, nothing on this account -// repaired the request, and no output or failure reached the client. Any other -// rejection of the request would be the same on every account. The handler -// still confirms the client-side state before it switches. -func (r *openAIReasoningRecoveryState) anotherAccountMayAccept(failure *UpstreamFailoverError, signatureRejected bool) bool { - if !r.switchAccounts || r.semanticCommitted || r.failureTerminalForwarded || r.ctx.Err() != nil || - openAIReasoningRecoveryProtectedStatus(r.responseStatus) || - // wire is absent when this account type has no stripped retry. - openAIRequestHoldsServerContext(r.wire) || openAIRequestHoldsServerContext(r.diagnosticIncoming) { - return false - } - if (r.retryUsed && !r.retryDispatched) || r.idRepairUnsent { - // The stripped retry, or the repair after it, was prepared but never - // sent. That is a fault in how this request was built, not evidence - // about the account. - return false - } - switch { - case failure == nil: - return signatureRejected - case !failure.IsOpenAIContinuationStateUnavailable() && !failure.IsOpenAIRequestRejected(): - return false - default: - return signatureRejected || openAIUpstreamReportsUndecryptableCiphertext(r.failurePayload) - } -} - -// accountFailedAfterRetry reports that the stripped retry was sent and the -// attempt then ended, before anything reached the client, with a failure of -// the account itself: one the attempt returned as a failover to the next -// account, which is how the same answer is treated on a first send. A request -// rejection, continuation state and a refusal are not the account's, and an -// error the attempt did not classify as a failover ends the request as before. -func (r *openAIReasoningRecoveryState) accountFailedAfterRetry(returned *UpstreamFailoverError) bool { - if returned == nil || !r.switchAccounts || !r.retryUsed || !r.retryDispatched || r.idRepairUnsent || - r.semanticCommitted || r.failureTerminalForwarded || r.ctx.Err() != nil { - return false - } - return returned.ShouldRetryNextAccount() && !returned.IsOpenAIRequestRejected() && - !returned.IsOpenAIContinuationStateUnavailable() && !returned.IsOpenAIRefusalRecovery() -} - -// ResetOpenAIReasoningRecoveryAttempt detaches the state of a rejected attempt -// before the request moves to another account, so that a forwarder which -// creates no state of its own does not read this one. The response header that -// names the last recovery action is left as it is. -func ResetOpenAIReasoningRecoveryAttempt(c *gin.Context) { - if c != nil { - c.Set(openAIReasoningRecoveryContextKey, (*openAIReasoningRecoveryState)(nil)) - } -} - -func (r *openAIReasoningRecoveryState) upstreamStatus(fallback int) int { - if r != nil && r.responseStatus > 0 { - return r.responseStatus - } - return fallback -} - -func (r *openAIReasoningRecoveryState) requestRejectionFromStream(payload []byte) *UpstreamFailoverError { - if r == nil || r.diagnosticIncoming == nil || (!r.retryUsed && r.cacheSkippedItems == 0) || !gjson.ValidBytes(payload) || openAIHTTPResponseTerminalError(payload) == nil { - return nil - } - message := extractOpenAISSEErrorMessage(payload) - semanticStatus := openAIStreamFailedEventSemanticStatus(payload, message) - if classifyOpenAIRequestRejection(semanticStatus, message, payload) == "" { - return nil - } - return NewOpenAIRequestRejectedError(r.upstreamStatus(semanticStatus), r.responseHeaders) -} - -func (r *openAIReasoningRecoveryState) continuationDiagnosticRecovery(payload []byte) *openAIContinuationRecoveryShape { - shape := &openAIContinuationRecoveryShape{ - CacheSkippedItems: r.cacheSkippedItems, - ItemIDsRemoved: r.itemIDsRemoved, - RetryAttempted: r.retryDispatched, - Disposition: r.diagnosticState, - AccountMismatch: r.accountMismatch, - AccountFailure: r.accountFailure, - } - if shape.Disposition == "" { - shape.Disposition = "not_attempted" - } - if r.stopRecorded && (r.idRepairUnsent || (r.retryUsed && !r.retryDispatched)) { - // The last re-send prepared, stripped retry or repair, was not sent. - shape.NotAttemptedReason = r.diagnosticStopReason - if shape.NotAttemptedReason == "" { - shape.NotAttemptedReason = "recovery_not_dispatched" - } - } else if shape.Disposition == "not_attempted" { - shape.NotAttemptedReason = r.recoveryNotAttemptedReason(payload) - } - return shape -} - -func (r *openAIReasoningRecoveryState) recoveryNotAttemptedReason(payload []byte) string { - switch { - case !r.enabled: - return "disabled" - case r.semanticCommitted: - return "semantic_output_committed" - case r.ctx.Err() != nil: - return "request_cancelled" - case openAIReasoningRecoveryProtectedStatus(r.responseStatus): - return "protected_status" - } - rejection, ok := parseOpenAIReasoningRejection(payload) - if !ok { - return "not_signature_rejection" - } - indices, reason := selectOpenAIRecoveryIndices(r.wire, rejection) - if len(indices) > 0 { - if _, err := stripOpenAIReasoningCipherIndices(r.wire, indices); err != nil { - return "rewrite_failed" - } - } - return reason -} - -func (r *openAIReasoningRecoveryState) diagnosticClassification(payload []byte) string { - message := extractOpenAISSEErrorMessage(payload) - if kind := classifyOpenAIContinuationStateError(message, payload); kind != openAIContinuationStateErrorNone { - return string(kind) - } - status := r.responseStatus - if status < 400 { - status = openAIStreamFailedEventSemanticStatus(payload, message) - } - return classifyOpenAIRequestRejection(status, message, payload) -} - -func (r *openAIReasoningRecoveryState) record(action string, status int, code string, payload []byte, count int) { - if r.c == nil { - return - } - if !r.c.Writer.Written() { - r.c.Header(openAIReasoningRecoveryHeader, action) - } - detail := map[string]any{"action": action, "items": count, "usage_status": "unavailable"} - usage := openAIReasoningUsageEvidence(payload) - recordedPayload := payload - if r.redactPayload != nil && len(payload) > 0 { - recordedPayload = r.redactPayload(payload) - } - upstreamMessage := "" - if len(payload) > 0 { - detail["payload_sha256"] = openAIReasoningDigest(payload) - upstreamMessage = extractOpenAISSEErrorMessage(recordedPayload) - } - if len(usage) == 0 && action == "retry_without_encrypted_content" { - usage = r.attemptUsage - if len(usage) > 0 { - detail["usage_source"] = "earlier_stream_event" - } - } - if len(usage) > 0 { - detail["usage_status"] = "available" - detail["usage"] = usage - } - encoded := encodeOpenAIReasoningRecoveryDetail(detail, upstreamMessage, recordedPayload) - logger.FromContext(r.ctx).Info("openai.reasoning_recovery", - zap.Int64("account_id", r.account.ID), zap.String("action", action), - zap.String("code", code), zap.Int("upstream_status", status), - zap.String("evidence", string(encoded))) - appendOpsUpstreamError(r.c, OpsUpstreamErrorEvent{ - Platform: r.account.Platform, AccountID: r.account.ID, - ProxyID: opsUpstreamProxyID(r.account), ProxyName: opsUpstreamProxyName(r.account), - Kind: "reasoning_recovery", Stage: "inference", Scope: "request", Reason: code, - UpstreamStatusCode: status, Message: action, Detail: string(encoded), - UpstreamRequestID: r.responseHeaders.Get("x-request-id"), - ContinuationDiagnostic: r.diagnosticForRecordedAttempt(action, recordedPayload), - }) -} - -// encodeOpenAIReasoningRecoveryDetail adds the upstream message and the rejected -// payload to the essential detail, halving their bounds until the encoded -// detail fits openAIReasoningRecoveryDetailEncodedLimit. -func encodeOpenAIReasoningRecoveryDetail(detail map[string]any, upstreamMessage string, payload []byte) []byte { - essential, _ := json.Marshal(detail) - if upstreamMessage == "" && len(payload) == 0 { - return essential - } - messageLimit, payloadLimit := openAIReasoningRecoveryMessageDetailLimit, openAIReasoningRecoveryPayloadDetailLimit - for messageLimit > 0 || payloadLimit > 0 { - candidate := make(map[string]any, len(detail)+5) - for key, value := range detail { - candidate[key] = value - } - if upstreamMessage != "" && messageLimit > 0 { - text, truncated := continuationDiagnosticBoundValue(upstreamMessage, messageLimit) - candidate["upstream_message"] = text - if truncated { - candidate["upstream_message_truncated"] = true - } - } - if len(payload) > 0 { - if payloadLimit > 0 { - text, truncated := continuationDiagnosticBoundValue(string(payload), payloadLimit) - candidate["payload"] = text - if truncated { - candidate["payload_truncated"] = true - } - } else { - candidate["payload_truncated"] = true - } - candidate["payload_bytes"] = len(payload) - } - if encoded, err := json.Marshal(candidate); err == nil && len(encoded) <= openAIReasoningRecoveryDetailEncodedLimit { - return encoded - } - messageLimit, payloadLimit = messageLimit/2, payloadLimit/2 - if messageLimit < 64 { - messageLimit = 0 - } - if payloadLimit < 64 { - payloadLimit = 0 - } - } - return essential -} - -func (r *openAIReasoningRecoveryState) diagnosticForRecordedAttempt(action string, payload []byte) *OpenAIContinuationDiagnostic { - if r.diagnosticIncoming == nil || action == "rejected_history_skipped" { - // Called before PrepareRequest freezes the stripped wire. A later failure - // will report the final snapshot and skipped-item count together. - return nil - } - return buildOpenAIContinuationDiagnostic(r.c, r.diagnosticIncoming, r.diagnosticRequest, r.wire, payload, r.diagnosticClassification(payload)) -} diff --git a/backend/internal/service/openai_reasoning_recovery_integration_test.go b/backend/internal/service/openai_reasoning_recovery_integration_test.go deleted file mode 100644 index 4aed45686e10..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_integration_test.go +++ /dev/null @@ -1,261 +0,0 @@ -//go:build unit - -package service - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" -) - -func assertReasoningRecoveryUsageEvidence(t *testing.T, c *gin.Context, input int64, output *int64) { - t.Helper() - value, ok := c.Get(OpsUpstreamErrorsKey) - require.True(t, ok) - found := false - for _, event := range value.([]*OpsUpstreamErrorEvent) { - if event.Kind != "reasoning_recovery" || event.Message != "retry_without_encrypted_content" { - continue - } - found = true - detail := gjson.Parse(event.Detail) - require.Equal(t, "available", detail.Get("usage_status").String()) - require.Equal(t, input, detail.Get("usage.input_tokens").Int()) - if output == nil { - require.False(t, detail.Get("usage.output_tokens").Exists(), "missing output usage must remain unavailable, not zero") - } else { - require.Equal(t, *output, detail.Get("usage.output_tokens").Int()) - } - } - require.True(t, found) -} - -func TestOpenAIReasoningRecoveryLegacyDoneAndEarlierUsage(t *testing.T) { - for _, mode := range []string{"native", "native_async", "passthrough", "buffered", "passthrough_buffered"} { - for _, terminal := range []string{"response.failed", "response.done"} { - t.Run(mode+"/"+terminal, func(t *testing.T) { - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - progress := `{"type":"response.in_progress","response":{"usage":{"input_tokens":3}}}` - failed := fmt.Sprintf(`{"type":%q,"response":{"status":"failed","error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}}`, terminal) - _, err := runOpenAIHTTPTerminalHandler(mode, state.c, io.NopCloser(strings.NewReader("data: "+progress+"\n\ndata: "+failed+"\n\n")), "text/event-stream") - var signal *openAIReasoningRecoverySignalError - require.ErrorAs(t, err, &signal) - _, recovered := state.TryRecoverError(err) - require.True(t, recovered) - assertReasoningRecoveryUsageEvidence(t, state.c, 3, nil) - }) - } - } -} - -// A Chat message has no field for reasoning ciphertext. On the Chat Completions -// endpoint only a Responses-shaped body carries it to the Responses upstream. -const reasoningRecoveryChatFixture = `{"model":"gpt-5.6-sol","reasoning":{"effort":"xhigh"},"store":false,"input":[{"role":"user","content":"constraints"},{"type":"reasoning","summary":[],"encrypted_content":"opaque-old"},{"role":"user","content":"continue"}]}` - -const reasoningRecoveryChatCompleted = `{"type":"response.completed","response":{"id":"resp_final","model":"gpt-5.6-sol","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":4,"output_tokens":6}}}` - -func reasoningRecoveryChatBody(t *testing.T, stream bool) []byte { - t.Helper() - body, err := sjson.SetBytes([]byte(reasoningRecoveryChatFixture), "stream", stream) - require.NoError(t, err) - return body -} - -func reasoningRecoveryChatContext(t *testing.T, body []byte) (*gin.Context, *httptest.ResponseRecorder) { - t.Helper() - gin.SetMode(gin.TestMode) - recorder := httptest.NewRecorder() - c, _ := gin.CreateTestContext(recorder) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) - group := int64(4) - c.Set("api_key", &APIKey{ID: 71, UserID: 9, GroupID: &group}) - return c, recorder -} - -func reasoningRecoverySSEResponse(payload string, status int) *http.Response { - return &http.Response{StatusCode: status, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(payload))} -} - -func TestOpenAIReasoningRecoveryChatPathWaitsForAuthoritativeRejection(t *testing.T) { - for _, stream := range []bool{false, true} { - for _, terminal := range []string{"response.failed", "response.done", "response.completed", "eof"} { - t.Run(fmt.Sprintf("stream_%t/%s", stream, terminal), func(t *testing.T) { - store := &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}} - account := newOpenAIRejectedFieldTestAccount() - progress := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_rejected\"}}\n\n" + - "data: {\"type\":\"response.in_progress\",\"response\":{\"usage\":{\"input_tokens\":3}}}\n\n" - bare := "data: {\"type\":\"error\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}}\n\n" - completed := "data: " + reasoningRecoveryChatCompleted + "\n\n" - firstFailure := progress + bare - switch terminal { - case "response.completed": - firstFailure += completed - case "response.failed", "response.done": - firstFailure += fmt.Sprintf("data: {\"type\":%q,\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}}}\n\n", terminal) - } - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - reasoningRecoverySSEResponse(firstFailure, http.StatusOK), - reasoningRecoverySSEResponse(completed, http.StatusOK), - }} - svc := newOpenAIRejectedFieldTestService(upstream) - svc.cache = store - body := reasoningRecoveryChatBody(t, stream) - c, writer := reasoningRecoveryChatContext(t, body) - result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") - require.NoError(t, err) - require.Equal(t, 4, result.Usage.InputTokens) - require.Equal(t, 6, result.Usage.OutputTokens) - if terminal == "response.completed" { - require.Len(t, upstream.requests, 1) - require.Empty(t, store.values) - } else { - require.NotContains(t, writer.Body.String(), "resp_rejected") - require.Len(t, upstream.requests, 2) - require.Len(t, store.values, 1) - assertReasoningRecoveryUsageEvidence(t, c, 3, nil) - } - }) - } - } -} - -func TestOpenAIReasoningRecoveryChatPathRecoversOnce(t *testing.T) { - for _, failure := range []string{"http", "sse_before_output", "sse_after_output", "recovery_fails"} { - t.Run(failure, func(t *testing.T) { - store := &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}} - account := newOpenAIRejectedFieldTestAccount() - account.Extra["same_account_retry_count"] = 0 - rejection := reasoningRecoverySSEResponse(`{"error":{"code":"invalid_encrypted_content","param":"input[1].encrypted_content"}}`, http.StatusBadRequest) - if strings.HasPrefix(failure, "sse_") { - payload := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"first\"}}\n\n" - if failure == "sse_after_output" { - payload += "data: {\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"already thinking\"}\n\n" - } - payload += `data: {"type":"response.failed","response":{"status":"failed","error":{"code":"invalid_encrypted_content","param":"input[1].encrypted_content"}}}` + "\n\n" - rejection = reasoningRecoverySSEResponse(payload, http.StatusOK) - } - final := reasoningRecoverySSEResponse("data: "+reasoningRecoveryChatCompleted+"\n\n", http.StatusOK) - if failure == "recovery_fails" { - final = reasoningRecoverySSEResponse(`{"error":{"code":"server_error"}}`, http.StatusInternalServerError) - } - upstream := &httpUpstreamRecorder{responses: []*http.Response{rejection, final}} - svc := newOpenAIRejectedFieldTestService(upstream) - svc.cache = store - body := reasoningRecoveryChatBody(t, true) - c, _ := reasoningRecoveryChatContext(t, body) - _, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") - if failure == "sse_after_output" { - require.Error(t, err) - require.Len(t, upstream.requests, 1) - require.Empty(t, store.values) - return - } - if failure == "recovery_fails" { - require.Error(t, err) - var failover *UpstreamFailoverError - require.False(t, errors.As(err, &failover), "one recovery cannot reenter the account pool") - } else { - require.NoError(t, err) - } - require.Len(t, upstream.requests, 2) - require.Len(t, store.values, 1) - require.Equal(t, "opaque-old", gjson.GetBytes(upstream.bodies[0], "input.1.encrypted_content").String()) - expected, err := sjson.DeleteBytes(upstream.bodies[0], "input.1.encrypted_content") - require.NoError(t, err) - require.JSONEq(t, string(expected), string(upstream.bodies[1])) - require.Equal(t, upstream.requests[0].Header, upstream.requests[1].Header) - require.Equal(t, upstream.requests[0].URL, upstream.requests[1].URL) - require.Nil(t, upstream.requests[1].GetBody) - }) - } -} - -type reasoningPartialWriter struct{ gin.ResponseWriter } - -func (w reasoningPartialWriter) Write(p []byte) (int, error) { - n, _ := w.ResponseWriter.Write(p[:min(len(p), 4)]) - return n, io.ErrClosedPipe -} - -func TestOpenAIReasoningRecoveryChatPartialWriteCannotRecover(t *testing.T) { - state, _, writer := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - state.c.Writer = reasoningPartialWriter{state.c.Writer} - body := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_partial\",\"status\":\"in_progress\"}}\n\n" + - "data: {\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"visible\"}\n\n" + - "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}}}\n\n" - svc := newOpenAIRejectedFieldTestService(&httpUpstreamRecorder{}) - _, err := svc.handleChatStreamingResponse(reasoningRecoverySSEResponse(body, 200), state.c, state.account, "gpt-5.6-sol", "gpt-5.6-sol", "gpt-5.6-sol", time.Now(), 32) - require.Error(t, err) - require.NotEmpty(t, writer.Body.String()) - var signal *openAIReasoningRecoverySignalError - require.False(t, errors.As(err, &signal)) - _, recovered := state.TryRecoverError(err) - require.False(t, recovered) -} - -// A tool round on the Chat Completions endpoint is forwarded as the plain -// conversion of the client's messages: the upstream reasoning of the first turn -// is neither kept nor put back, and no reasoning state is read or written. -func TestOpenAIReasoningRecoveryChatToolRoundKeepsNoReasoningState(t *testing.T) { - const firstOutput = `[{"type":"reasoning","id":"rs_original","status":"completed","summary":[{"type":"summary_text","text":"checking carefully "}],"encrypted_content":"opaque-original"},{"type":"function_call","id":"fc_original","status":"completed","call_id":"call_orders","name":"load_orders","arguments":"{}"}]` - messages := []apicompat.ChatMessage{{Role: "user", Content: json.RawMessage(`"Use the orders tool."`)}} - chatBody := func(stream bool) []byte { - body, err := json.Marshal(map[string]any{ - "model": "gpt-5.6-sol", "messages": messages, "stream": stream, - "tools": []any{map[string]any{"type": "function", "function": map[string]any{"name": "load_orders", "parameters": map[string]any{"type": "object"}}}}, - }) - require.NoError(t, err) - return body - } - for _, stream := range []bool{false, true} { - t.Run(fmt.Sprintf("stream_%t", stream), func(t *testing.T) { - messages = messages[:1] - store := &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}} - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - reasoningRecoverySSEResponse("data: "+strings.Replace(reasoningRecoveryChatCompleted, `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}]`, firstOutput, 1)+"\n\n", http.StatusOK), - reasoningRecoverySSEResponse("data: "+reasoningRecoveryChatCompleted+"\n\n", http.StatusOK), - }} - svc := newOpenAIRejectedFieldTestService(upstream) - svc.cache = store - account := newOpenAIRejectedFieldTestAccount() - first := chatBody(stream) - c, writer := reasoningRecoveryChatContext(t, first) - _, err := svc.ForwardAsChatCompletions(context.Background(), c, account, first, "", "") - require.NoError(t, err) - require.Contains(t, writer.Body.String(), "call_orders") - - messages = append(messages, - apicompat.ChatMessage{Role: "assistant", ReasoningContent: "checking carefully ", ToolCalls: []apicompat.ChatToolCall{{ID: "call_orders", Type: "function", Function: apicompat.ChatFunctionCall{Name: "load_orders", Arguments: "{}"}}}}, - apicompat.ChatMessage{Role: "tool", ToolCallID: "call_orders", Content: json.RawMessage(`"order data"`)}) - second := chatBody(stream) - c, _ = reasoningRecoveryChatContext(t, second) - _, err = svc.ForwardAsChatCompletions(context.Background(), c, account, second, "", "") - require.NoError(t, err) - - require.Len(t, upstream.bodies, 2) - var request apicompat.ChatCompletionsRequest - require.NoError(t, json.Unmarshal(second, &request)) - converted, err := apicompat.ChatCompletionsToResponses(&request) - require.NoError(t, err) - require.JSONEq(t, string(converted.Input), gjson.GetBytes(upstream.bodies[1], "input").Raw) - require.NotContains(t, string(upstream.bodies[1]), "opaque-original") - require.NotContains(t, string(upstream.bodies[1]), "rs_original") - require.Zero(t, store.gets) - require.Zero(t, store.puts) - }) - } -} diff --git a/backend/internal/service/openai_reasoning_recovery_item_id_test.go b/backend/internal/service/openai_reasoning_recovery_item_id_test.go deleted file mode 100644 index 2d77b931b657..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_item_id_test.go +++ /dev/null @@ -1,668 +0,0 @@ -//go:build unit - -package service - -import ( - "bytes" - "context" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" -) - -// An id in the form a relay mints. The official upstream takes a reasoning -// item that carries such an id and no ciphertext for a reference to an item it -// stores, and does not find it. -const reasoningRecoveryRelayItemID = "rs_0123456789abcdef0123456789abcdef" - -var reasoningRecoveryRelayFixture = strings.Replace(reasoningRecoveryFixture, "rs_old", reasoningRecoveryRelayItemID, 1) - -func reasoningRecoveryJSON(status int, body string) *http.Response { - resp := newJSONResponse(status, body) - resp.Header.Set("Content-Type", "application/json") - return resp -} - -// The official answer, as recorded in production. -func reasoningRecoveryUnfoundItem(id string) *http.Response { - return reasoningRecoveryJSON(http.StatusNotFound, `{"error":{"message":"Item with id '`+id+`' not found. Items are not persisted when `+"`store`"+` is set to false. Try again with `+"`store`"+` set to true, or remove this item from your input.","type":"invalid_request_error","param":"input","code":null}}`) -} - -func reasoningRecoveryCipherRejected() *http.Response { - return reasoningRecoveryJSON(http.StatusBadRequest, `{"error":{"message":"The encrypted content could not be verified.","type":"invalid_request_error","param":null,"code":"invalid_encrypted_content"}}`) -} - -func reasoningRecoveryCompleted() *http.Response { - return reasoningRecoveryJSON(http.StatusOK, `{"id":"resp_final","status":"completed","model":"gpt-5.6-sol","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":11,"output_tokens":7}}`) -} - -// reasoningRecoveryItemIDHarness forwards requests of one key to one account -// over a scripted upstream and a rejection memory that outlives a request. -type reasoningRecoveryItemIDHarness struct { - svc *OpenAIGatewayService - store *reasoningRecoveryMemoryStore - upstream *httpUpstreamRecorder - account *Account -} - -func newReasoningRecoveryItemIDHarness(passthrough bool) *reasoningRecoveryItemIDHarness { - upstream := &httpUpstreamRecorder{} - h := &reasoningRecoveryItemIDHarness{ - svc: newOpenAIImageGenerationControlTestService(upstream), upstream: upstream, - store: &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}}, account: newOpenAIImageGenerationControlTestAccount(), - } - h.svc.cache = h.store - h.account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - return h -} - -type reasoningRecoveryItemIDAttempt struct { - c *gin.Context - recorder *httptest.ResponseRecorder - bodies [][]byte - err error -} - -// forward sends body once; the upstream answers with responses in order. -func (h *reasoningRecoveryItemIDHarness) forward(t *testing.T, body string, responses ...*http.Response) reasoningRecoveryItemIDAttempt { - t.Helper() - c, recorder := newOpenAIImageGenerationControlTestContext(false, "test-client") - getAPIKeyFromContext(c).UserID = 77 - sent := len(h.upstream.bodies) - h.upstream.responses = responses - _, err := h.svc.Forward(context.Background(), c, h.account, []byte(body)) - require.Empty(t, h.upstream.responses, "every scripted answer was asked for") - return reasoningRecoveryItemIDAttempt{c: c, recorder: recorder, bodies: h.upstream.bodies[sent:], err: err} -} - -// learn makes the memory hold the ciphertext of body's reasoning items: the -// upstream rejects it once and accepts the stripped request. -func (h *reasoningRecoveryItemIDHarness) learn(t *testing.T, body string) { - t.Helper() - attempt := h.forward(t, body, reasoningRecoveryCipherRejected(), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 2) - require.Equal(t, 1, h.store.puts) -} - -func (a reasoningRecoveryItemIDAttempt) recoveryActions() []string { - events, _ := a.c.Get(OpsUpstreamErrorsKey) - var actions []string - recorded, _ := events.([]*OpsUpstreamErrorEvent) - for _, event := range recorded { - if event.Kind == "reasoning_recovery" { - actions = append(actions, event.Message) - } - } - return actions -} - -func forEachReasoningRecoveryPath(t *testing.T, run func(t *testing.T, h *reasoningRecoveryItemIDHarness)) { - for _, passthrough := range []bool{false, true} { - t.Run(map[bool]string{true: "passthrough", false: "native"}[passthrough], func(t *testing.T) { - run(t, newReasoningRecoveryItemIDHarness(passthrough)) - }) - } -} - -// The upstream cannot find an item the gateway left behind without ciphertext. -// The request is sent once more with the ids of the stripped items removed and -// nothing else changed. -func TestOpenAIReasoningRecoveryRepairsAStrippedItemTheUpstreamCannotFind(t *testing.T) { - t.Run("pre_stripped_first_send", func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - - attempt := h.forward(t, reasoningRecoveryRelayFixture, reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 2, "the pre-stripped send and the repair") - require.False(t, gjson.GetBytes(attempt.bodies[0], "input.1.encrypted_content").Exists()) - require.Equal(t, reasoningRecoveryRelayItemID, gjson.GetBytes(attempt.bodies[0], "input.1.id").String()) - withoutID, err := sjson.DeleteBytes(attempt.bodies[0], "input.1.id") - require.NoError(t, err) - require.Equal(t, string(withoutID), string(attempt.bodies[1]), "only the id of the stripped item is gone") - require.Equal(t, "visible summary", gjson.GetBytes(attempt.bodies[1], "input.1.summary.0.text").String(), "the item itself stays") - require.Equal(t, "fc_one", gjson.GetBytes(attempt.bodies[1], "input.2.id").String(), "no other item loses its id") - require.Contains(t, attempt.recorder.Body.String(), "resp_final") - require.Equal(t, []string{"rejected_history_skipped", "retry_without_item_ids"}, attempt.recoveryActions()) - require.Equal(t, "retry_without_item_ids", attempt.recorder.Header().Get(openAIReasoningRecoveryHeader)) - require.Equal(t, 1, h.store.puts, "the repair writes nothing to the memory") - }) - }) - t.Run("stripped_retry", func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - attempt := h.forward(t, reasoningRecoveryRelayFixture, - reasoningRecoveryCipherRejected(), reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 3, "the first send, the stripped retry and the repair") - require.Equal(t, "opaque-old", gjson.GetBytes(attempt.bodies[0], "input.1.encrypted_content").String()) - stripped, err := sjson.DeleteBytes(attempt.bodies[0], "input.1.encrypted_content") - require.NoError(t, err) - require.Equal(t, string(stripped), string(attempt.bodies[1])) - withoutID, err := sjson.DeleteBytes(stripped, "input.1.id") - require.NoError(t, err) - require.Equal(t, string(withoutID), string(attempt.bodies[2])) - require.Contains(t, attempt.recorder.Body.String(), "resp_final") - require.Equal(t, []string{"retry_without_encrypted_content", "retry_without_item_ids"}, attempt.recoveryActions()) - require.Equal(t, 1, h.store.puts, "the rejected ciphertext only") - }) - }) - // A relay may pass the official answer on under a status of its own. - t.Run("answered_with_another_status", func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - wrapped := reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID) - wrapped.StatusCode = http.StatusBadRequest - - attempt := h.forward(t, reasoningRecoveryRelayFixture, wrapped, reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 2) - require.False(t, gjson.GetBytes(attempt.bodies[1], "input.1.id").Exists()) - }) - }) -} - -// Nothing about the repair is remembered: the next request of the conversation -// is sent as before, meets the same answer and is repaired again. -func TestOpenAIReasoningRecoveryRepairIsMadeAgainOnTheNextRequest(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - first := h.forward(t, reasoningRecoveryRelayFixture, reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCompleted()) - require.NoError(t, first.err) - - next := h.forward(t, reasoningRecoveryRelayFixture, reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCompleted()) - require.NoError(t, next.err) - require.Len(t, next.bodies, 2) - require.Equal(t, string(first.bodies[0]), string(next.bodies[0]), "the id is still sent first") - require.Equal(t, string(first.bodies[1]), string(next.bodies[1])) - }) -} - -// The upstream names the first item it cannot find. Every stripped item loses -// its id in the one repair, so the next one is not reported in turn. -func TestOpenAIReasoningRecoveryRepairRemovesTheIDOfEveryStrippedItem(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - body := strings.Replace(reasoningRecoveryRelayFixture, `"output":"[]"}`, - `"output":"[]"},{"type":"reasoning","id":"rs_fedcba9876543210fedcba9876543210","summary":[],"encrypted_content":"opaque-second"}`, 1) - h.learn(t, body) - - attempt := h.forward(t, body, reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 2) - for _, index := range []string{"1", "4"} { - require.True(t, gjson.GetBytes(attempt.bodies[0], "input."+index+".id").Exists()) - require.False(t, gjson.GetBytes(attempt.bodies[0], "input."+index+".encrypted_content").Exists()) - require.Equal(t, "reasoning", gjson.GetBytes(attempt.bodies[1], "input."+index+".type").String()) - require.False(t, gjson.GetBytes(attempt.bodies[1], "input."+index+".id").Exists()) - } - }) -} - -// A repair that happens before any stripped retry leaves that retry available: -// ciphertext the upstream then rejects is still stripped once, and its item -// loses the id with it. -func TestOpenAIReasoningRecoveryRepairLeavesTheStrippedRetryAvailable(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - body := strings.Replace(reasoningRecoveryRelayFixture, `"output":"[]"}`, - `"output":"[]"},{"type":"reasoning","id":"rs_fedcba9876543210fedcba9876543210","summary":[],"encrypted_content":"opaque-new"}`, 1) - - attempt := h.forward(t, body, - reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCipherRejected(), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 3) - require.Equal(t, "opaque-new", gjson.GetBytes(attempt.bodies[1], "input.4.encrypted_content").String()) - require.False(t, gjson.GetBytes(attempt.bodies[1], "input.1.id").Exists()) - for _, index := range []string{"1", "4"} { - require.Equal(t, "reasoning", gjson.GetBytes(attempt.bodies[2], "input."+index+".type").String()) - require.False(t, gjson.GetBytes(attempt.bodies[2], "input."+index+".encrypted_content").Exists()) - require.False(t, gjson.GetBytes(attempt.bodies[2], "input."+index+".id").Exists(), "after the repair a strip takes the id too") - } - require.Equal(t, []string{"rejected_history_skipped", "retry_without_item_ids", "retry_without_encrypted_content"}, attempt.recoveryActions()) - }) -} - -// The repair is made once in an attempt. -func TestOpenAIReasoningRecoveryRepairsOnce(t *testing.T) { - validation := func() *http.Response { - return reasoningRecoveryJSON(http.StatusBadRequest, `{"error":{"message":"Missing required parameter: 'input[1].id'.","type":"invalid_request_error","param":"input[1].id","code":"missing_required_parameter"}}`) - } - for _, test := range []struct { - name string - answer func() *http.Response - }{ - {name: "not_found_again", answer: func() *http.Response { return reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID) }}, - {name: "rejected", answer: validation}, - } { - t.Run(test.name, func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - - attempt := h.forward(t, reasoningRecoveryRelayFixture, reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), test.answer()) - require.Error(t, attempt.err) - require.Len(t, attempt.bodies, 2, "no second repair") - require.Equal(t, 1, h.store.puts) - }) - }) - } - t.Run("after_the_stripped_retry", func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - attempt := h.forward(t, reasoningRecoveryRelayFixture, reasoningRecoveryCipherRejected(), - reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID)) - require.Len(t, attempt.bodies, 3) - var stopped *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, attempt.err, &stopped, "the stripped retry is spent: the request ends on this account") - require.Equal(t, http.StatusNotFound, stopped.Failure.StatusCode) - recovery := lastReasoningRecoveryDiagnostic(t, attempt.c).ContinuationDiagnostic.Recovery - require.Equal(t, 1, recovery.ItemIDsRemoved) - require.True(t, recovery.RetryAttempted, "the repaired body was sent") - }) - }) -} - -// Only the answer naming an item this attempt stripped is repaired. -func TestOpenAIReasoningRecoveryLeavesEveryOtherMissingItemAlone(t *testing.T) { - const clientItemID = "rs_ffffffffffffffffffffffffffffffff" - withClientItem := strings.Replace(reasoningRecoveryRelayFixture, `"output":"[]"}`, - `"output":"[]"},{"type":"reasoning","id":"`+clientItemID+`","summary":[]}`, 1) - withoutSummary := strings.Replace(reasoningRecoveryRelayFixture, `"summary":[{"type":"summary_text","text":"visible summary"}],`, ``, 1) - for _, test := range []struct { - name string - learn string - body string - answer func() *http.Response - }{ - { - name: "an_item_the_client_sent_without_ciphertext", learn: reasoningRecoveryRelayFixture, body: withClientItem, - answer: func() *http.Response { return reasoningRecoveryUnfoundItem(clientItemID) }, - }, - { - name: "an_id_that_is_not_in_the_request", learn: reasoningRecoveryRelayFixture, body: reasoningRecoveryRelayFixture, - answer: func() *http.Response { return reasoningRecoveryUnfoundItem("rs_00000000000000000000000000000000") }, - }, - { - name: "nothing_was_stripped", body: reasoningRecoveryRelayFixture, - answer: func() *http.Response { return reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID) }, - }, - { - // Without a summary the upstream refuses the item once its id is gone. - name: "an_item_without_a_summary", learn: withoutSummary, body: withoutSummary, - answer: func() *http.Response { return reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID) }, - }, - { - name: "a_status_that_says_something_about_the_account", learn: reasoningRecoveryRelayFixture, body: reasoningRecoveryRelayFixture, - answer: func() *http.Response { - forbidden := reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID) - forbidden.StatusCode = http.StatusForbidden - return forbidden - }, - }, - { - name: "another_message", learn: reasoningRecoveryRelayFixture, body: reasoningRecoveryRelayFixture, - answer: func() *http.Response { - return reasoningRecoveryJSON(http.StatusNotFound, `{"error":{"message":"The model `+"`"+reasoningRecoveryRelayItemID+"`"+` does not exist.","type":"invalid_request_error","param":"input","code":null}}`) - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - if test.learn != "" { - h.learn(t, test.learn) - } - attempt := h.forward(t, test.body, test.answer()) - require.Error(t, attempt.err) - require.Len(t, attempt.bodies, 1, "the request is not sent again") - require.NotContains(t, attempt.recoveryActions(), "retry_without_item_ids") - }) - }) - } -} - -// An item without a summary keeps its id also when a strip follows the repair. -func TestOpenAIReasoningRecoveryKeepsTheIDOfAnItemWithoutSummary(t *testing.T) { - forEachReasoningRecoveryPath(t, func(t *testing.T, h *reasoningRecoveryItemIDHarness) { - h.learn(t, reasoningRecoveryRelayFixture) - const bareID = "rs_fedcba9876543210fedcba9876543210" - body := strings.Replace(reasoningRecoveryRelayFixture, `"output":"[]"}`, - `"output":"[]"},{"type":"reasoning","id":"`+bareID+`","encrypted_content":"opaque-new"}`, 1) - - attempt := h.forward(t, body, - reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), reasoningRecoveryCipherRejected(), reasoningRecoveryCompleted()) - require.NoError(t, attempt.err) - require.Len(t, attempt.bodies, 3) - require.False(t, gjson.GetBytes(attempt.bodies[2], "input.1.id").Exists()) - require.False(t, gjson.GetBytes(attempt.bodies[2], "input.4.encrypted_content").Exists()) - require.Equal(t, bareID, gjson.GetBytes(attempt.bodies[2], "input.4.id").String()) - }) -} - -// The repair acts on the body that was sent: an item it stripped must be there, -// at its place, with its id and without ciphertext. -func TestOpenAIReasoningRecoveryRepairChecksWhatItStrippedAgainstWhatWasSent(t *testing.T) { - unfound := []byte(`{"error":{"message":"Item with id '` + reasoningRecoveryRelayItemID + `' not found."}}`) - prepared := func(t *testing.T) *openAIReasoningRecoveryState { - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryRelayFixture) - _, repaired := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, unfound) - require.False(t, repaired, "nothing has been stripped") - _, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - return state - } - - t.Run("the_stripped_body_was_not_sent", func(t *testing.T) { - state := prepared(t) - _, repaired := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, unfound) - require.False(t, repaired, "the body last sent still holds the ciphertext") - }) - for name, sent := range map[string]func(wire []byte) []byte{ - "another_id_at_its_place": func(wire []byte) []byte { - out, _ := sjson.SetBytes(wire, "input.1.id", "rs_ffffffffffffffffffffffffffffffff") - return out - }, - "another_item_at_its_place": func(wire []byte) []byte { - out, _ := sjson.SetBytes(wire, "input.1.type", "message") - return out - }, - } { - t.Run(name, func(t *testing.T) { - state := prepared(t) - state.wire = sent(state.retryBody) - _, repaired := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, unfound) - require.False(t, repaired) - }) - } - t.Run("an_entry_that_no_longer_holds_does_not_block_the_others", func(t *testing.T) { - const second = "rs_fedcba9876543210fedcba9876543210" - body := strings.Replace(reasoningRecoveryRelayFixture, `"output":"[]"}`, - `"output":"[]"},{"type":"reasoning","id":"`+second+`","summary":[],"encrypted_content":"opaque-second"}`, 1) - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), body) - _, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - state.wire, _ = sjson.SetBytes(state.retryBody, "input.1.type", "message") - - repaired, ok := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, []byte(`{"error":{"message":"Item with id '`+second+`' not found."}}`)) - require.True(t, ok) - require.Equal(t, reasoningRecoveryRelayItemID, gjson.GetBytes(repaired, "input.1.id").String()) - require.False(t, gjson.GetBytes(repaired, "input.4.id").Exists()) - }) - t.Run("sent_as_stripped", func(t *testing.T) { - state := prepared(t) - state.MarkAttemptDispatched() - state.wire = bytes.Clone(state.retryBody) - state.responseStatus, state.responseHeaders = 0, nil - body, repaired := state.TryRepairUnfoundItemIDs(http.StatusNotFound, http.Header{"X-Request-Id": []string{"req_unfound"}}, unfound) - require.True(t, repaired) - require.False(t, gjson.GetBytes(body, "input.1.id").Exists()) - require.Equal(t, string(body), string(state.retryBody), "the send boundary accepts the repaired body") - require.True(t, state.retryDispatched, "the stripped retry stays recorded as sent") - require.Equal(t, "retry_attempted", state.diagnosticState) - require.Zero(t, state.responseStatus, "the repaired answer is not left behind as the attempt's status") - require.Nil(t, state.responseHeaders) - _, again := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, unfound) - require.False(t, again) - }) -} - -// The repair's event carries the status and the request id of the answer it -// repairs, whatever an earlier answer left in the state, and a repaired body -// that is then not sent is reported with the reason. -func TestOpenAIReasoningRecoveryRepairIsRecordedWithItsOwnAnswer(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryRelayFixture) - state.BindDiagnosticRequest([]byte(reasoningRecoveryRelayFixture), req) - stripped, retry := state.TryRecover(http.StatusBadRequest, http.Header{"X-Request-Id": []string{"req_cipher_rejected"}}, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - retried, _, err := state.PrepareRequest(req.Clone(context.Background()), stripped, "") - require.NoError(t, err) - state.MarkAttemptDispatched() - - unfound := []byte(`{"error":{"message":"Item with id '` + reasoningRecoveryRelayItemID + `' not found."}}`) - repaired, ok := state.TryRepairUnfoundItemIDs(http.StatusNotFound, http.Header{"X-Request-Id": []string{"req_item_not_found"}}, unfound) - require.True(t, ok) - events, _ := state.c.Get(OpsUpstreamErrorsKey) - recorded := events.([]*OpsUpstreamErrorEvent) - last := recorded[len(recorded)-1] - require.Equal(t, "retry_without_item_ids", last.Message) - require.Equal(t, http.StatusNotFound, last.UpstreamStatusCode) - require.Equal(t, "req_item_not_found", last.UpstreamRequestID) - require.True(t, last.ContinuationDiagnostic.Recovery.RetryAttempted, "the stripped retry was sent") - - // The send boundary refuses a body other than the repaired one. - _, _, err = state.PrepareRequest(retried.Clone(context.Background()), stripped, "") - var stopped *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, err, &stopped) - blocked := lastReasoningRecoveryDiagnostic(t, state.c).ContinuationDiagnostic.Recovery - require.Equal(t, "source_changed", blocked.NotAttemptedReason, "the repaired body was never sent") - require.True(t, blocked.RetryAttempted, "the stripped retry stays recorded as sent") - require.NotEqual(t, string(stripped), string(repaired)) -} - -// A repair prepared after either a cached strip or a sent stripped retry may -// fail before sending. Report that fact without handing the old answer to -// another account. The cached path keeps the builder's ordinary error contract. -func TestOpenAIReasoningRecoveryUnsentRepairKeepsItsDiagnostic(t *testing.T) { - for _, test := range []struct { - name string - cached bool - prior func() error - }{ - {name: "cached_snapshot_failure", cached: true}, - {name: "stripped_retry_snapshot_failure"}, - {name: "stripped_retry_old_account_failure", prior: func() error { - return &UpstreamFailoverError{StatusCode: http.StatusBadGateway, ResponseBody: []byte(`{"error":{"code":"credit_balance_exhausted"}}`)} - }}, - {name: "stripped_retry_old_cipher_rejection", prior: func() error { - return NewOpenAIContinuationStateUnavailableError(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`)) - }}, - } { - t.Run(test.name, func(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryRelayFixture) - var body []byte - if test.cached { - store := &reasoningRecoveryMemoryStore{} - learner, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryRelayFixture) - learner.store = store - _, retry := learner.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - state.store = store - body = []byte(reasoningRecoveryRelayFixture) - } else { - var retry bool - body, retry = state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - } - retried, _, err := state.PrepareRequest(req.Clone(context.Background()), body, "") - require.NoError(t, err) - state.BindDiagnosticRequest([]byte(reasoningRecoveryRelayFixture), retried) - state.MarkAttemptDispatched() - repaired, ok := state.TryRepairUnfoundItemIDs(http.StatusNotFound, nil, []byte(`{"error":{"message":"Item with id '`+reasoningRecoveryRelayItemID+`' not found."}}`)) - require.True(t, ok) - - var attempt error - if test.prior != nil { - attempt = test.prior() - } else { - unsent := retried.Clone(context.Background()) - unsent.GetBody = func() (io.ReadCloser, error) { return nil, errors.New("snapshot unavailable") } - _, _, attempt = state.PrepareRequest(unsent, repaired, "") - require.Error(t, attempt) - } - stopped := state.StopError(attempt) - if test.cached { - require.Same(t, attempt, stopped) - } else { - var terminal *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, stopped, &terminal) - require.False(t, terminal.Failure.ShouldRetryNextAccount()) - } - diagnostic := lastReasoningRecoveryDiagnostic(t, state.c).ContinuationDiagnostic.Recovery - require.Equal(t, "recovery_not_dispatched", diagnostic.NotAttemptedReason) - require.Equal(t, !test.cached, diagnostic.RetryAttempted) - require.Equal(t, 1, diagnostic.ItemIDsRemoved) - require.False(t, diagnostic.AccountMismatch) - require.False(t, diagnostic.AccountFailure) - }) - } -} - -// The fields are deleted in one pass over the body. For valid JSON with one -// input, which is all the callers pass, the bytes are those of deleting them -// one whole-body edit at a time, which is what a request was sent with before, -// and the same targets are refused. -func TestOpenAIReasoningRecoveryDeletesFieldsInOnePassAsOneByOne(t *testing.T) { - const ( - compact = `{"type":"reasoning","id":"rs_a","summary":[{"type":"summary_text","text":"id \"quoted\" \\ é"}],"encrypted_content":"a"}` - idFirst = `{"id":"rs_b","type":"reasoning","encrypted_content":"b","summary":[]}` - idLast = `{"type":"reasoning","summary":[],"encrypted_content":"c","id":"rs_c"}` - idOnly = `{"type":"reasoning","id":"rs_d"}` - bare = `{"type":"reasoning","summary":[{"id":"nested","encrypted_content":"nested"}]}` - spaced = "{\n \"type\" : \"reasoning\" ,\n \"id\" : \"rs_e\" ,\n \"encrypted_content\" : \"e\" ,\n \"summary\" : [ ]\n }" - message = `{"type":"message","id":"msg_1","role":"user","content":"hello"}` - ) - bodies := map[string]string{ - "compact": `{"model":"m","input":[` + strings.Join([]string{message, compact, idFirst, message, idLast, idOnly, bare, spaced}, ",") + `],"store":false}`, - "spaced": "{\n \"input\" : [\n " + strings.Join([]string{message, compact, idFirst, message, idLast, idOnly, bare, spaced}, " ,\n ") + "\n ] ,\n \"model\" : \"m\"\n}", - "object": `{"input":{"0":` + compact + `}}`, - "absent": `{"model":"m"}`, - } - targets := map[string][]int{ - "none": nil, "one": {1}, "all": {1, 2, 4, 5, 6, 7}, "unsorted_and_repeated": {7, 2, 2, 1, 7}, - "a_message": {1, 3}, "out_of_range": {1, 8}, "negative": {-1}, "first": {0}, - } - for bodyName, body := range bodies { - for targetName, indices := range targets { - for _, field := range []string{"encrypted_content", "id"} { - t.Run(strings.Join([]string{bodyName, targetName, field}, "/"), func(t *testing.T) { - want, wantErr := deleteOpenAIReasoningItemFieldOneByOne([]byte(body), indices, field) - got, gotErr := deleteOpenAIReasoningItemField([]byte(body), indices, field) - require.Equal(t, wantErr == nil, gotErr == nil, "one refuses what the other refuses") - require.Equal(t, string(want), string(got)) - }) - } - } - } - // A history of the size seen in production stays one pass. - items := make([]string, 0, 400) - for index := range 400 { - items = append(items, strings.Replace(compact, "rs_a", "rs_"+strings.Repeat("0", 28)+string(rune('a'+index%26))+string(rune('a'+index/26)), 1)) - } - long := []byte(`{"model":"m","input":[` + strings.Join(items, ",") + `]}`) - all := make([]int, 400) - for index := range all { - all[index] = index - } - want, err := deleteOpenAIReasoningItemFieldOneByOne(long, all, "id") - require.NoError(t, err) - got, err := deleteOpenAIReasoningItemField(long, all, "id") - require.NoError(t, err) - require.Equal(t, string(want), string(got)) - require.NotContains(t, string(got), `"id":"rs_`) -} - -// A Responses-shaped body on the Chat Completions endpoint is stripped by the -// same rules and is repaired the same way. -func TestOpenAIReasoningRecoveryChatPathRepairsAStrippedItemTheUpstreamCannotFind(t *testing.T) { - store := &reasoningRecoveryMemoryStore{GatewayCache: &stubGatewayCache{}} - account := newOpenAIRejectedFieldTestAccount() - fixture := strings.Replace(reasoningRecoveryChatFixture, `{"type":"reasoning",`, `{"type":"reasoning","id":"`+reasoningRecoveryRelayItemID+`",`, 1) - body, err := sjson.SetBytes([]byte(fixture), "stream", true) - require.NoError(t, err) - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - reasoningRecoveryCipherRejected(), - reasoningRecoveryUnfoundItem(reasoningRecoveryRelayItemID), - reasoningRecoverySSEResponse("data: "+reasoningRecoveryChatCompleted+"\n\n", http.StatusOK), - }} - svc := newOpenAIRejectedFieldTestService(upstream) - svc.cache = store - c, _ := reasoningRecoveryChatContext(t, body) - - _, err = svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") - require.NoError(t, err) - require.Len(t, upstream.requests, 3, "the first send, the stripped retry and the repair") - require.Equal(t, reasoningRecoveryRelayItemID, gjson.GetBytes(upstream.bodies[1], "input.1.id").String()) - require.Equal(t, "reasoning", gjson.GetBytes(upstream.bodies[2], "input.1.type").String()) - require.False(t, gjson.GetBytes(upstream.bodies[2], "input.1.id").Exists()) - require.False(t, gjson.GetBytes(upstream.bodies[2], "input.1.encrypted_content").Exists()) -} - -// The edit is repeated on the clean body exactly, also where the wire body -// holds an item the clean body does not, and nothing else passes as an edit. -func TestOpenAIReasoningRecoveryProjectionRepeatsOnlyRecoveryEdits(t *testing.T) { - const ( - system = `{"type":"message","role":"developer","content":"system"}` - message = `{"type":"message","id":"msg_1","role":"user","content":"hello"}` - ) - body := func(items ...string) string { return `{"model":"m","input":[` + strings.Join(items, ",") + `]}` } - cipher := func(id, text string) string { - return `{"type":"reasoning","id":"` + id + `","summary":[],"encrypted_content":"` + text + `"}` - } - bare := func(id string) string { return `{"type":"reasoning","id":"` + id + `","summary":[]}` } - const idless = `{"type":"reasoning","summary":[]}` - - for _, test := range []struct { - name, clean, before, after, want string - }{ - { - name: "ciphertext_only", - clean: body(cipher("rs_a", "a"), message), before: body(system, cipher("rs_a", "a"), message), - after: body(system, bare("rs_a"), message), want: body(bare("rs_a"), message), - }, - { - name: "ciphertext_and_id", - clean: body(cipher("rs_a", "a"), cipher("rs_b", "b")), before: body(system, cipher("rs_a", "a"), cipher("rs_b", "b")), - after: body(system, idless, cipher("rs_b", "b")), want: body(idless, cipher("rs_b", "b")), - }, - { - name: "id_of_an_item_left_behind", - clean: body(bare("rs_a"), message, bare("rs_b")), before: body(system, bare("rs_a"), message, bare("rs_b")), - after: body(system, bare("rs_a"), message, idless), want: body(bare("rs_a"), message, idless), - }, - { - name: "second_of_two_items_with_one_id", - clean: body(bare("rs_a"), bare("rs_a")), before: body(system, bare("rs_a"), bare("rs_a")), - after: body(system, bare("rs_a"), idless), want: body(bare("rs_a"), idless), - }, - { - name: "the_id_of_another_item", - clean: body(bare("rs_a"), message), before: body(bare("rs_a"), message), - after: body(bare("rs_a"), `{"type":"message","role":"user","content":"hello"}`), - }, - { - name: "the_id_of_an_item_that_keeps_its_ciphertext", - clean: body(cipher("rs_a", "a")), before: body(cipher("rs_a", "a")), - after: body(`{"type":"reasoning","summary":[],"encrypted_content":"a"}`), - }, - { - name: "any_other_field", - clean: body(bare("rs_a")), before: body(bare("rs_a")), - after: body(`{"type":"reasoning","id":"rs_a"}`), - }, - { - name: "an_item_the_clean_body_does_not_hold", - clean: body(message), before: body(bare("rs_a"), message), - after: body(idless, message), - }, - { - name: "an_item_removed", - clean: body(message, cipher("rs_a", "a")), before: body(message, cipher("rs_a", "a")), - after: body(message), - }, - } { - t.Run(test.name, func(t *testing.T) { - projected, err := projectReasoningRecoveryEdits([]byte(test.clean), []byte(test.before), []byte(test.after)) - if test.want == "" { - require.ErrorIs(t, err, errReasoningWireProjection) - return - } - require.NoError(t, err) - require.Equal(t, test.want, string(projected)) - }) - } -} diff --git a/backend/internal/service/openai_reasoning_recovery_policy.go b/backend/internal/service/openai_reasoning_recovery_policy.go deleted file mode 100644 index 0f98980b3006..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_policy.go +++ /dev/null @@ -1,151 +0,0 @@ -package service - -import ( - "encoding/json" - "regexp" - "strconv" - "strings" - - "github.com/tidwall/gjson" -) - -// openAIReasoningRejection is an upstream error that rejects reasoning -// ciphertext of the request. message is the upstream error text; selection -// reads it only for the id of the rejected item. -type openAIReasoningRejection struct{ code, param, message string } - -// parseOpenAIReasoningRejection recognizes an upstream error, response.failed or -// failed response.done payload whose error code is thinking_signature_invalid -// or invalid_encrypted_content. A payload that carries a protected status is -// never one. This reads protocol framing only; whether the request can be -// repaired is decided by selectOpenAIRecoveryIndices. -func parseOpenAIReasoningRejection(payload []byte) (openAIReasoningRejection, bool) { - if _, err := canonicalReasoningCacheJSON(payload); err != nil { - return openAIReasoningRejection{}, false - } - root := gjson.ParseBytes(payload) - for _, path := range []string{"status_code", "error.status_code", "error.status", "response.error.status_code", "response.error.status"} { - if value := root.Get(path); (value.Type == gjson.Number || value.Type == gjson.String) && openAIReasoningRecoveryProtectedStatus(int(value.Int())) { - return openAIReasoningRejection{}, false - } - } - event, responseStatus := root.Get("type").String(), root.Get("response.status").String() - if event != "" && event != "error" && event != "response.failed" && event != "response.done" { - return openAIReasoningRejection{}, false - } - if event == "response.done" && responseStatus != "failed" { - return openAIReasoningRejection{}, false - } - if status := root.Get("status").String(); status != "" && status != "failed" { - return openAIReasoningRejection{}, false - } - var source gjson.Result - switch { - case root.Get("response.error").IsObject(): - if event != "response.failed" && responseStatus != "failed" { - return openAIReasoningRejection{}, false - } - source = root.Get("response.error") - case root.Get("error").IsObject(): - source = root.Get("error") - case event == "error": - source = root - default: - return openAIReasoningRejection{}, false - } - code := source.Get("code") - if code.Type != gjson.String || (code.String() != "thinking_signature_invalid" && code.String() != "invalid_encrypted_content") { - return openAIReasoningRejection{}, false - } - param := source.Get("param") - if param.Exists() && param.Type != gjson.String && param.Type != gjson.Null { - return openAIReasoningRejection{}, false - } - rejection := openAIReasoningRejection{code: code.String(), param: param.String()} - if message := source.Get("message"); message.Type == gjson.String { - rejection.message = message.String() - } - return rejection, true -} - -// openAIRecoveryMessageNamesItem reports whether the upstream message contains -// a Responses reasoning item id as a whole identifier. The upstream reports a -// ciphertext it cannot verify as "… for item rs_… could not be verified" with a -// null param; the id is the only place the rejected item is identified. -func openAIRecoveryMessageNamesItem(message, id string) bool { - if !strings.HasPrefix(id, "rs_") || len(id) == len("rs_") { - return false - } - identifier := func(b byte) bool { - return b == '_' || b == '-' || (b >= '0' && b <= '9') || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') - } - for offset := 0; ; { - at := strings.Index(message[offset:], id) - if at < 0 { - return false - } - start := offset + at - end := start + len(id) - if (start == 0 || !identifier(message[start-1])) && (end == len(message) || !identifier(message[end])) { - return true - } - offset = start + 1 - } -} - -var openAIReasoningErrorInputParam = regexp.MustCompile(`^input(?:\[(\d+)\]|\.(\d+))(?:\.encrypted_content)?$`) - -// selectOpenAIRecoveryIndices returns the reasoning items of the sent body that -// may lose their ciphertext for the rejection, and the reason the continuation -// diagnostic reports while no stripped retry has been sent: why nothing is -// selected, or recovery_not_dispatched when something is. -func selectOpenAIRecoveryIndices(body []byte, rejection openAIReasoningRejection) ([]int, string) { - if _, err := canonicalReasoningCacheJSON(body); err != nil { - return nil, "invalid_request_snapshot" - } - // Removing ciphertext cannot repair a locally provable orphan tool result. - if !openAIReasoningToolHistoryAllowsRecovery(body) { - return nil, "invalid_tool_history" - } - items := openAIReasoningCipherItems(body) - if len(items) == 0 { - return nil, "no_reasoning_ciphertext" - } - if match := openAIReasoningErrorInputParam.FindStringSubmatch(rejection.param); match != nil { - text := match[1] - if text == "" { - text = match[2] - } - if index, err := strconv.Atoi(text); err == nil { - for _, item := range items { - if item.index == index { - return []int{index}, "recovery_not_dispatched" - } - } - } - return nil, "target_not_reasoning_ciphertext" - } - if rejection.param != "" && rejection.param != "input" && rejection.param != "reasoning.encrypted_content" { - return nil, "unsupported_error_param" - } - if openAIRequestHoldsServerContext(body) { - return nil, "server_held_context" - } - indices := make([]int, 0, len(items)) - named := false - for _, item := range items { - indices = append(indices, item.index) - named = named || (rejection.message != "" && openAIRecoveryMessageNamesItem(rejection.message, item.id)) - } - encryptedFields := 0 - var parsed any - if json.Unmarshal(body, &parsed) == nil { - encryptedFields = countOpenAIEncryptedFields(parsed) - } - // Another ciphertext carrier (compaction, an inter-agent message) may be the - // rejected one, unless the upstream names a reasoning item itself. - if encryptedFields != len(items) && !named { - return nil, "ambiguous_encrypted_carriers" - } - return indices, "recovery_not_dispatched" -} diff --git a/backend/internal/service/openai_reasoning_recovery_policy_test.go b/backend/internal/service/openai_reasoning_recovery_policy_test.go deleted file mode 100644 index 44e6891b959a..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_policy_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build unit - -package service - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestOpenAIReasoningRecoveryRejectionExcludesProtectedStatuses(t *testing.T) { - _, ok := parseOpenAIReasoningRejection([]byte(`{"error":{"code":"thinking_signature_invalid"}}`)) - require.True(t, ok) - _, ok = parseOpenAIReasoningRejection([]byte(`{"error":{"status":429,"code":"thinking_signature_invalid"}}`)) - require.False(t, ok, "quota/protected failures cannot become recovery markers") -} - -func TestOpenAIReasoningRecoverySelectionRequiresUnambiguousLocalCipherScope(t *testing.T) { - const ( - history = `"input":[{"role":"user","content":"first"},{"type":"reasoning","id":"rs_one","encrypted_content":"a"},{"role":"user","content":"second"},{"type":"reasoning","id":"rs_three","encrypted_content":"b"}]` - local = `{` + history + `}` - held = `{"previous_response_id":"resp_old",` + history + `}` - carriers = `{"input":[{"role":"user","content":"first"},{"type":"reasoning","id":"rs_one","encrypted_content":"a"},{"type":"compaction","encrypted_content":"c"},{"type":"reasoning","id":"rs_three","encrypted_content":"b"}]}` - ) - rejection := func(payload string) openAIReasoningRejection { - t.Helper() - parsed, ok := parseOpenAIReasoningRejection([]byte(payload)) - require.True(t, ok, payload) - return parsed - } - unqualified := rejection(`{"error":{"code":"invalid_encrypted_content"}}`) - - indices, reason := selectOpenAIRecoveryIndices([]byte(local), unqualified) - require.Equal(t, []int{1, 3}, indices, "the unambiguous local set is selected whole") - require.Equal(t, "recovery_not_dispatched", reason) - - indices, reason = selectOpenAIRecoveryIndices([]byte(held), unqualified) - require.Empty(t, indices, "server-held context is left alone") - require.Equal(t, "server_held_context", reason) - - indices, _ = selectOpenAIRecoveryIndices([]byte(held), rejection(`{"error":{"code":"invalid_encrypted_content","param":"input[3].encrypted_content"}}`)) - require.Equal(t, []int{3}, indices, "an explicit target is selected alone") - - indices, reason = selectOpenAIRecoveryIndices([]byte(held), rejection(`{"error":{"code":"invalid_encrypted_content","param":"input[2].encrypted_content"}}`)) - require.Empty(t, indices, "an item without reasoning ciphertext is never selected") - require.Equal(t, "target_not_reasoning_ciphertext", reason) - - indices, reason = selectOpenAIRecoveryIndices([]byte(carriers), rejection(`{"error":{"code":"invalid_encrypted_content","param":"input"}}`)) - require.Empty(t, indices, "the compaction ciphertext may be the rejected one") - require.Equal(t, "ambiguous_encrypted_carriers", reason) - - indices, _ = selectOpenAIRecoveryIndices([]byte(carriers), rejection(`{"error":{"code":"invalid_encrypted_content","param":"input","message":"The encrypted content for item rs_three could not be verified."}}`)) - require.Equal(t, []int{1, 3}, indices, "a rejection that names a reasoning item is not ambiguous") -} diff --git a/backend/internal/service/openai_reasoning_recovery_terminal_test.go b/backend/internal/service/openai_reasoning_recovery_terminal_test.go deleted file mode 100644 index 754b752aa3d7..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_terminal_test.go +++ /dev/null @@ -1,321 +0,0 @@ -//go:build unit - -package service - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - "testing" - "time" - - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" -) - -type reasoningRecoverySeededNegativeCache struct { - GatewayCache - entry OpenAIRejectedReasoning - gets int - puts int -} - -func (s *reasoningRecoverySeededNegativeCache) GetOpenAIRejectedReasoning(_ context.Context, scope OpenAIReasoningCacheScope, hashes []string) (map[string]OpenAIRejectedReasoning, error) { - s.gets++ - result := make(map[string]OpenAIRejectedReasoning) - if scope.ScopeHash != "" { - for _, hash := range hashes { - if hash == openAIReasoningDigest([]byte("opaque-old")) { - result[hash] = s.entry - } - } - } - return result, nil -} - -func (s *reasoningRecoverySeededNegativeCache) PutOpenAIRejectedReasoning(context.Context, OpenAIReasoningCacheScope, []string) error { - s.puts++ - return nil -} - -func lastReasoningRecoveryDiagnostic(t *testing.T, c *gin.Context) *OpsUpstreamErrorEvent { - t.Helper() - value, ok := c.Get(OpsUpstreamErrorsKey) - require.True(t, ok) - var last *OpsUpstreamErrorEvent - for _, event := range value.([]*OpsUpstreamErrorEvent) { - if event.ContinuationDiagnostic != nil { - last = event - } - } - require.NotNil(t, last) - return last -} - -// Every outcome keeps its classification and diagnostic. Where the upstream -// said it could not verify ciphertext, the attempt ends as an account mismatch, -// a failure the handler may hand to another account; a validation rejection is -// the same on every account and ends where it happened. -func TestOpenAIReasoningRecoveryOutcomesPreserveClassificationAndDiagnostic(t *testing.T) { - const signature = `{"code":"thinking_signature_invalid","param":"input[1].encrypted_content","message":"private-signature-detail-8162"}` - const wrongTarget = `{"code":"thinking_signature_invalid","param":"input[2].encrypted_content","message":"private-signature-detail-8162"}` - const validation = `{"type":"invalid_request_error","code":"missing_required_parameter","param":"input[3].call_id","message":"private-validation-detail-7514"}` - for _, passthrough := range []bool{false, true} { - for _, transport := range []string{"http", "stream", "buffered_sse"} { - for _, behavior := range []string{"recovery_then_validation", "recovery_then_signature", "unrecoverable_signature", "cache_skip_then_validation"} { - name := fmt.Sprintf("passthrough_%t/%s/%s", passthrough, transport, behavior) - t.Run(name, func(t *testing.T) { - input := reasoningRecoveryFixture - if transport == "stream" { - input = strings.Replace(input, `"store":false`, `"store":false,"stream":true`, 1) - } - payload, classification := validation, "request_validation" - if behavior == "unrecoverable_signature" { - payload, classification = wrongTarget, "thinking_signature_invalid" - } else if behavior == "recovery_then_signature" { - payload, classification = signature, "thinking_signature_invalid" - } - status, contentType, terminal := http.StatusBadRequest, "application/json", `{"error":`+payload+`}` - if transport != "http" { - status, contentType = http.StatusOK, "text/event-stream" - terminal = "data: " + `{"type":"response.failed","response":{"status":"failed","error":` + payload + `}}` + "\n\n" - } - responses := []*http.Response{{ - StatusCode: status, - Header: http.Header{"Content-Type": []string{contentType}, "X-Request-Id": []string{"request-final-safe-id"}}, - Body: io.NopCloser(strings.NewReader(terminal)), - }} - retried := strings.HasPrefix(behavior, "recovery_then_") - if retried { - responses = append([]*http.Response{newJSONResponse(http.StatusBadRequest, `{"error":`+signature+`}`)}, responses...) - } - upstream := &httpUpstreamRecorder{responses: responses} - svc := newOpenAIImageGenerationControlTestService(upstream) - repo := &openAIPassthroughFailoverRepo{} - svc.rateLimitService = &RateLimitService{accountRepo: repo, cfg: svc.cfg} - c, _ := newOpenAIImageGenerationControlTestContext(false, "test-client") - getAPIKeyFromContext(c).UserID = 77 - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true, "pool_mode_enabled": true} - var negative *reasoningRecoverySeededNegativeCache - if behavior == "cache_skip_then_validation" { - negative = &reasoningRecoverySeededNegativeCache{ - GatewayCache: &stubGatewayCache{}, - entry: OpenAIRejectedReasoning{RejectedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(time.Hour)}, - } - svc.cache = negative - } - result, err := svc.Forward(context.Background(), c, account, []byte(input)) - require.Error(t, err) - require.Nil(t, result) - mismatch := classification != "request_validation" - var failure *UpstreamFailoverError - var stopped *OpenAIReasoningRecoveryTerminalError - if retried && !mismatch { - require.ErrorAs(t, err, &stopped) - require.False(t, errors.As(err, &failure), "a validation rejection of the stripped retry is a terminal") - failure = stopped.Failure - } else { - require.False(t, errors.As(err, &stopped), "an account mismatch is not a terminal") - require.ErrorAs(t, err, &failure) - } - if retried { - require.Len(t, upstream.requests, 2, "no third POST on this account") - } else { - require.Len(t, upstream.requests, 1) - } - require.Equal(t, status, failure.StatusCode, "retain actual HTTP status separately from client semantics") - require.Equal(t, http.StatusBadRequest, failure.ClientStatusCode) - require.Equal(t, classification == "request_validation", failure.IsOpenAIRequestRejected()) - require.Equal(t, classification != "request_validation", failure.IsOpenAIContinuationStateUnavailable()) - require.Equal(t, mismatch, failure.IsOpenAICiphertextAccountMismatch()) - require.Equal(t, mismatch, failure.ShouldRetryNextAccount()) - require.False(t, failure.RetryableOnSameAccount) - require.True(t, failure.SuppressAccountHealthPenalty) - require.False(t, failure.ShouldReportAccountScheduleFailure()) - require.Empty(t, repo.rateLimitCalls) - require.Empty(t, repo.overloadCalls) - for _, req := range upstream.requests { - require.Nil(t, req.GetBody) - } - for _, path := range []string{"reasoning", "input.0", "input.1.id", "input.1.phase", "input.1.summary", "input.1.unknown", "input.2", "input.3"} { - require.JSONEq(t, gjson.Get(input, path).Raw, gjson.GetBytes(upstream.lastBody, path).Raw, path) - } - event := lastReasoningRecoveryDiagnostic(t, c) - diagnostic := event.ContinuationDiagnostic - require.Equal(t, classification, diagnostic.Classification) - require.Equal(t, status, event.UpstreamStatusCode) - require.Equal(t, "request-final-safe-id", event.UpstreamRequestID) - require.Equal(t, "frozen_request", diagnostic.Wire.BodySource) - require.False(t, diagnostic.Wire.InspectionLimited) - require.Equal(t, len(upstream.lastBody), diagnostic.Wire.BodyBytes) - require.Equal(t, 1, diagnostic.Incoming.History.Encrypted) - require.NotNil(t, diagnostic.Recovery) - require.Equal(t, mismatch, diagnostic.Recovery.AccountMismatch) - require.Equal(t, retried, diagnostic.Recovery.RetryAttempted) - if retried { - require.Equal(t, "budget_exhausted", diagnostic.Recovery.Disposition) - require.Zero(t, diagnostic.Wire.History.Encrypted) - } else { - require.Equal(t, "not_attempted", diagnostic.Recovery.Disposition) - if negative != nil { - require.Equal(t, 1, diagnostic.Recovery.CacheSkippedItems) - require.Equal(t, "not_signature_rejection", diagnostic.Recovery.NotAttemptedReason) - require.Zero(t, diagnostic.Wire.History.Encrypted) - require.Equal(t, 1, negative.gets) - require.Zero(t, negative.puts, "a negative cache hit must not refresh rejection memory") - } else { - require.Equal(t, "target_not_reasoning_ciphertext", diagnostic.Recovery.NotAttemptedReason) - require.Equal(t, 1, diagnostic.Wire.History.Encrypted) - } - } - encoded, encodeErr := json.Marshal(diagnostic) - require.NoError(t, encodeErr) - // The administrator's diagnostic keeps the upstream message of - // the final rejection; request history stays summarized, and the - // returned error never carries upstream text. - require.Equal(t, gjson.Get(payload, "message").String(), diagnostic.UpstreamError.Message.Value) - for _, secret := range []string{"opaque-old", "visible summary", "source-token"} { - require.NotContains(t, string(encoded), secret) - } - for _, secret := range []string{"opaque-old", "visible summary", "private-signature-detail-8162", "private-validation-detail-7514", "source-token"} { - require.NotContains(t, err.Error(), secret) - } - }) - } - } - } -} - -func TestOpenAIReasoningRecoveryFrozenDiagnosticDoesNotReadGetBody(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - state.BindDiagnosticRequest([]byte(reasoningRecoveryFixture), req) - require.Nil(t, req.GetBody) - // An unrelated caller's prepared body cannot replace the frozen wire proof. - diagnostic := buildOpenAIContinuationDiagnostic(state.c, []byte(reasoningRecoveryFixture), req, []byte(`{"input":[]}`), []byte(`{"error":{"code":"thinking_signature_invalid"}}`), "thinking_signature_invalid") - require.Equal(t, "frozen_request", diagnostic.Wire.BodySource) - require.False(t, diagnostic.Wire.InspectionLimited) - require.Equal(t, len(reasoningRecoveryFixture), diagnostic.Wire.BodyBytes) - require.Equal(t, 1, diagnostic.Wire.History.Encrypted) - require.Equal(t, 1, diagnostic.Wire.History.Calls) - require.Equal(t, 1, diagnostic.Wire.History.Outputs) - require.Nil(t, req.GetBody, "diagnostics must not re-enable automatic POST replay") - stripped, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"thinking_signature_invalid"}}`), false) - require.True(t, retry) - changedSource := req.Clone(context.Background()) - changedSource.Header.Set("Authorization", "Bearer different-source") - _, _, err := state.PrepareRequest(changedSource, stripped, "") - var stopped *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, err, &stopped) - blocked := lastReasoningRecoveryDiagnostic(t, state.c).ContinuationDiagnostic - require.False(t, blocked.Recovery.RetryAttempted, "a prepared but blocked retry was never dispatched") - require.Equal(t, "source_changed", blocked.Recovery.NotAttemptedReason) - require.Equal(t, 1, blocked.Wire.History.Encrypted, "the last sent snapshot is still the original attempt") -} - -func TestOpenAIReasoningRecoveryNotAttemptedDiagnosticReasons(t *testing.T) { - for _, reason := range []string{"disabled", "request_cancelled", "semantic_output_committed", "server_held_context", "no_reasoning_ciphertext", "invalid_tool_history"} { - t.Run(reason, func(t *testing.T) { - body := reasoningRecoveryFixture - switch reason { - case "server_held_context": - body = strings.Replace(body, `"store":false`, `"store":false,"previous_response_id":"private-upstream-reference"`, 1) - case "no_reasoning_ciphertext": - body = strings.Replace(body, `"encrypted_content":"opaque-old",`, "", 1) - case "invalid_tool_history": - body = strings.Replace(body, `"type":"function_call_output","call_id":"call_one"`, `"type":"function_call_output","call_id":"missing"`, 1) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - state, req, _ := newReasoningRecoveryTestState(t, ctx, body) - state.BindDiagnosticRequest([]byte(body), req) - state.ObserveResponse(&http.Response{StatusCode: http.StatusBadRequest}) - if reason == "disabled" { - state.enabled = false - } else if reason == "request_cancelled" { - cancel() - } - payload := []byte(`{"error":{"code":"thinking_signature_invalid","message":"private-rejection-detail"}}`) - _, retry := state.TryRecover(http.StatusBadRequest, nil, payload, reason == "semantic_output_committed") - require.False(t, retry) - err := state.StopError(NewOpenAIContinuationStateUnavailableError(http.StatusBadRequest, nil, payload)) - require.Error(t, err) - diagnostic := lastReasoningRecoveryDiagnostic(t, state.c).ContinuationDiagnostic - require.Equal(t, "thinking_signature_invalid", diagnostic.Classification) - require.Equal(t, "not_attempted", diagnostic.Recovery.Disposition) - require.Equal(t, reason, diagnostic.Recovery.NotAttemptedReason) - require.False(t, diagnostic.Recovery.RetryAttempted) - require.Nil(t, req.GetBody) - }) - } -} - -func TestOpenAIContinuationDiagnosticRecoveryEnumSanitization(t *testing.T) { - diagnostic := &OpenAIContinuationDiagnostic{Recovery: &openAIContinuationRecoveryShape{ - CacheSkippedItems: -1, Disposition: "private-disposition-52913", NotAttemptedReason: "private-reason-27184", - }} - safe := sanitizeOpenAIContinuationDiagnostic(diagnostic) - require.NotNil(t, safe) - require.NotNil(t, safe.Recovery) - require.Zero(t, safe.Recovery.CacheSkippedItems) - require.Equal(t, "unknown", safe.Recovery.Disposition) - require.Empty(t, safe.Recovery.NotAttemptedReason) - encoded, err := json.Marshal(safe) - require.NoError(t, err) - require.NotContains(t, string(encoded), "private-") -} - -type reasoningRecoveryTerminalWriteFailure struct{ gin.ResponseWriter } - -func (w reasoningRecoveryTerminalWriteFailure) Write(body []byte) (int, error) { - if strings.Contains(string(body), "response.failed") { - return 0, io.ErrClosedPipe - } - return w.ResponseWriter.Write(body) -} - -func TestOpenAIReasoningRecoveryTerminalDeliveryDistinguishesInterruptedStream(t *testing.T) { - const rejection = `{"error":{"code":"invalid_encrypted_content","param":"input[1].encrypted_content"}}` - const delta = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"visible answer\"}\n\n" - const failed = "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}}}\n\n" - for _, passthrough := range []bool{false, true} { - for _, ending := range []string{"failed_terminal", "interrupted_after_delta", "terminal_write_failed"} { - t.Run(fmt.Sprintf("passthrough_%t/%s", passthrough, ending), func(t *testing.T) { - responseBody := delta - if ending != "interrupted_after_delta" { - responseBody += failed - } - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - newJSONResponse(http.StatusBadRequest, rejection), - {StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(responseBody))}, - }} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, recorder := newOpenAIImageGenerationControlTestContext(false, "test-client") - if ending == "terminal_write_failed" { - c.Writer = reasoningRecoveryTerminalWriteFailure{c.Writer} - } - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - input := strings.Replace(reasoningRecoveryFixture, `"store":false`, `"store":false,"stream":true`, 1) - _, err := svc.Forward(context.Background(), c, account, []byte(input)) - var terminal *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, err, &terminal) - require.Len(t, upstream.requests, 2) - require.Equal(t, ending == "failed_terminal", terminal.FailureTerminalForwarded) - require.Contains(t, recorder.Body.String(), "visible answer") - if ending == "interrupted_after_delta" { - require.NotContains(t, recorder.Body.String(), "response.failed", "the handler must still synthesize the missing terminal") - } else if ending == "failed_terminal" { - require.Equal(t, 1, strings.Count(recorder.Body.String(), `"type":"response.failed"`)) - } - var retryable *UpstreamFailoverError - require.False(t, errors.As(err, &retryable)) - }) - } - } -} diff --git a/backend/internal/service/openai_reasoning_recovery_test.go b/backend/internal/service/openai_reasoning_recovery_test.go deleted file mode 100644 index a01c3eb5a9c0..000000000000 --- a/backend/internal/service/openai_reasoning_recovery_test.go +++ /dev/null @@ -1,712 +0,0 @@ -//go:build unit - -package service - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" -) - -const reasoningRecoveryFixture = `{"model":"gpt-5.6-sol","reasoning":{"effort":"xhigh","mode":"standard","context":"keep","extension":{"x":1}},"store":false,"input":[{"role":"user","content":"constraints"},{"type":"reasoning","id":"rs_old","phase":"analysis","summary":[{"type":"summary_text","text":"visible summary"}],"encrypted_content":"opaque-old","unknown":{"keep":true}},{"type":"function_call","id":"fc_one","call_id":"call_one","name":"load_orders","arguments":"{}"},{"type":"function_call_output","call_id":"call_one","output":"[]"}]}` - -// A Codex multi-agent history: an inter-agent message carries its body as -// ciphertext in a content part, between two reasoning items. -const reasoningRecoveryAgentMessageFixture = `{"model":"gpt-5.6-sol","input":[{"type":"reasoning","id":"rs_a","encrypted_content":"a"},{"type":"agent_message","id":"amsg_b","author":"/root","recipient":"/root/worker","content":[{"type":"input_text","text":"Payload:"},{"type":"encrypted_content","encrypted_content":"b"}]},{"type":"reasoning","id":"rs_c","encrypted_content":"c"}]}` - -func newReasoningRecoveryTestState(t *testing.T, ctx context.Context, body string) (*openAIReasoningRecoveryState, *http.Request, *httptest.ResponseRecorder) { - t.Helper() - gin.SetMode(gin.TestMode) - recorder := httptest.NewRecorder() - c, _ := gin.CreateTestContext(recorder) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil).WithContext(ctx) - groupID := int64(7) - c.Set("api_key", &APIKey{ID: 19, UserID: 5, GroupID: &groupID}) - account := &Account{ID: 31, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} - svc := &OpenAIGatewayService{} - state := svc.newOpenAIReasoningRecoveryState(ctx, c, account, "source-token") - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "https://upstream.example/v1/responses", strings.NewReader(body)) - require.NoError(t, err) - req.Header.Set("Authorization", "Bearer source-token") - req, _, err = state.PrepareRequest(req, []byte(body), "") - require.NoError(t, err) - t.Cleanup(state.Close) - return state, req, recorder -} - -func TestOpenAIReasoningRecoveryStrictErrorsAndScope(t *testing.T) { - tests := []struct { - name, body, payload string - want bool - }{ - {"exact_unqualified_single", reasoningRecoveryFixture, `{"error":{"code":"invalid_encrypted_content"}}`, true}, - {"exact_signature_index", reasoningRecoveryFixture, `{"error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}`, true}, - {"failed_envelope", reasoningRecoveryFixture, `{"type":"response.failed","response":{"status":"failed","error":{"code":"invalid_encrypted_content","param":"input.1"}}}`, true}, - {"done_failed_envelope", reasoningRecoveryFixture, `{"type":"response.done","response":{"status":"failed","error":{"code":"thinking_signature_invalid","param":"input.1"}}}`, true}, - {"bare_error", reasoningRecoveryFixture, `{"type":"error","code":"invalid_encrypted_content"}`, true}, - {"message_substring_not_code", reasoningRecoveryFixture, `{"error":{"message":"invalid_encrypted_content","code":"bad_request"}}`, false}, - {"numeric_code", reasoningRecoveryFixture, `{"error":{"code":400,"message":"thinking_signature_invalid"}}`, false}, - {"duplicate_error_code", reasoningRecoveryFixture, `{"error":{"code":"invalid_encrypted_content","code":"server_error"}}`, false}, - {"completed_is_not_rejected", reasoningRecoveryFixture, `{"type":"response.completed","response":{"status":"completed","error":{"code":"invalid_encrypted_content"}}}`, false}, - {"incomplete_is_not_rejected", reasoningRecoveryFixture, `{"status":"incomplete","error":{"code":"invalid_encrypted_content"}}`, false}, - {"generic_status", reasoningRecoveryFixture, `{"error":{"code":"server_error"}}`, false}, - {"case_sensitive_code", reasoningRecoveryFixture, `{"error":{"code":"INVALID_ENCRYPTED_CONTENT"}}`, false}, - {"wrong_target", reasoningRecoveryFixture, `{"error":{"code":"invalid_encrypted_content","param":"input[2].encrypted_content"}}`, false}, - {"unknown_target", reasoningRecoveryFixture, `{"error":{"code":"invalid_encrypted_content","param":"input[1].summary"}}`, false}, - {"unknown_param_shape", reasoningRecoveryFixture, `{"error":{"code":"invalid_encrypted_content","param":["input",1]}}`, false}, - {"local_reasoning_candidate_set", `{"model":"gpt-5.6-sol","input":[{"type":"reasoning","encrypted_content":"a"},{"type":"reasoning","encrypted_content":"b"}]}`, `{"error":{"code":"invalid_encrypted_content"}}`, true}, - {"ambiguous_compaction", `{"model":"gpt-5.6-sol","input":[{"type":"reasoning","encrypted_content":"a"},{"type":"compaction","encrypted_content":"b"}]}`, `{"error":{"code":"invalid_encrypted_content"}}`, false}, - {"specific_despite_compaction", `{"model":"gpt-5.6-sol","input":[{"type":"reasoning","encrypted_content":"a"},{"type":"compaction","encrypted_content":"b"}]}`, `{"error":{"code":"invalid_encrypted_content","param":"input[0].encrypted_content"}}`, true}, - {"reference_not_local_proof", `{"model":"gpt-5.6-sol","previous_response_id":"resp_old","input":[{"type":"reasoning","encrypted_content":"a"}]}`, `{"error":{"code":"invalid_encrypted_content"}}`, false}, - {"no_reasoning_cipher", `{"model":"gpt-5.6-sol","input":[{"type":"compaction","encrypted_content":"a"}]}`, `{"error":{"code":"invalid_encrypted_content"}}`, false}, - {"named_reasoning_despite_agent_message", reasoningRecoveryAgentMessageFixture, `{"error":{"message":"The encrypted content for item rs_c could not be verified. Reason: Encrypted content could not be decrypted or parsed.","type":"invalid_request_error","param":null,"code":"invalid_encrypted_content"}}`, true}, - {"named_reasoning_failed_envelope", reasoningRecoveryAgentMessageFixture, `{"type":"response.failed","response":{"status":"failed","error":{"code":"invalid_encrypted_content","message":"The encrypted content for item rs_a could not be verified. Reason: Encrypted content could not be decrypted or parsed."}}}`, true}, - {"unnamed_with_agent_message", reasoningRecoveryAgentMessageFixture, `{"error":{"message":"invalid encrypted content","param":null,"code":"invalid_encrypted_content"}}`, false}, - {"named_agent_message", reasoningRecoveryAgentMessageFixture, `{"error":{"message":"The encrypted content for item amsg_b could not be verified. Reason: Encrypted content could not be decrypted or parsed.","param":null,"code":"invalid_encrypted_content"}}`, false}, - {"named_longer_identifier", reasoningRecoveryAgentMessageFixture, `{"error":{"message":"The encrypted content for item rs_ab could not be verified. Reason: Encrypted content could not be decrypted or parsed.","param":null,"code":"invalid_encrypted_content"}}`, false}, - {"orphan_tool_is_not_repaired", `{"model":"gpt-5.6-sol","input":[{"type":"reasoning","encrypted_content":"a"},{"type":"function_call_output","call_id":"missing","output":"result"}]}`, `{"error":{"code":"invalid_encrypted_content","param":"input[0].encrypted_content"}}`, false}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), test.body) - body, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(test.payload), false) - require.Equal(t, test.want, retry) - require.Equal(t, test.body, string(state.original)) - if retry { - require.NotEqual(t, test.body, string(body)) - require.True(t, state.RecoveryAttempt()) - _, retry = state.TryRecover(http.StatusBadRequest, nil, []byte(test.payload), false) - require.False(t, retry, "a request has only one extra POST") - } - }) - } -} - -func TestOpenAIReasoningRecoveryNamedReasoningLeavesOtherCarriers(t *testing.T) { - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryAgentMessageFixture) - payload := []byte(`{"error":{"message":"The encrypted content for item rs_c could not be verified. Reason: Encrypted content could not be decrypted or parsed.","param":null,"code":"invalid_encrypted_content"}}`) - after, retry := state.TryRecover(http.StatusBadRequest, nil, payload, false) - require.True(t, retry) - require.False(t, gjson.GetBytes(after, "input.0.encrypted_content").Exists()) - require.False(t, gjson.GetBytes(after, "input.2.encrypted_content").Exists()) - for _, path := range []string{"input.0.id", "input.1", "input.2.id"} { - require.Equal(t, gjson.Get(reasoningRecoveryAgentMessageFixture, path).Raw, gjson.GetBytes(after, path).Raw, path) - } -} - -func TestOpenAIReasoningRecoveryStopErrorMarksAccountMismatch(t *testing.T) { - const signature = `{"error":{"message":"The encrypted content for item amsg_b could not be verified.","param":null,"code":"invalid_encrypted_content"}}` - const undecryptable = `{"error":{"type":"invalid_request_error","message":"Encrypted function output content could not be decrypted or decoded."}}` - const validation = `{"error":{"type":"invalid_request_error","code":"missing_required_parameter","message":"input[3].call_id is required"}}` - serverHeld := strings.Replace(reasoningRecoveryAgentMessageFixture, `{"model"`, `{"previous_response_id":"resp_old","model"`, 1) - tests := []struct { - name, body, payload string - status int - arrange func(*openAIReasoningRecoveryState) - want bool - }{ - {"ciphertext_the_strip_cannot_reach", reasoningRecoveryAgentMessageFixture, signature, http.StatusBadRequest, nil, true}, - {"undecryptable_without_a_code", reasoningRecoveryAgentMessageFixture, undecryptable, http.StatusBadRequest, nil, true}, - {"ordinary_validation", reasoningRecoveryAgentMessageFixture, validation, http.StatusBadRequest, nil, false}, - {"history_held_by_the_account", serverHeld, signature, http.StatusBadRequest, nil, false}, - {"history_held_by_an_account_without_the_stripped_retry", serverHeld, signature, http.StatusBadRequest, func(r *openAIReasoningRecoveryState) { r.enabled, r.wire = false, nil }, false}, - {"status_about_the_account", reasoningRecoveryAgentMessageFixture, signature, http.StatusForbidden, nil, false}, - {"switch_off", reasoningRecoveryAgentMessageFixture, signature, http.StatusBadRequest, func(r *openAIReasoningRecoveryState) { r.enabled, r.switchAccounts = false, false }, false}, - {"output_already_sent", reasoningRecoveryAgentMessageFixture, signature, http.StatusBadRequest, func(r *openAIReasoningRecoveryState) { r.semanticCommitted = true }, false}, - {"failure_already_forwarded", reasoningRecoveryAgentMessageFixture, signature, http.StatusBadRequest, func(r *openAIReasoningRecoveryState) { r.failureTerminalForwarded = true }, false}, - {"request_cancelled", reasoningRecoveryAgentMessageFixture, signature, http.StatusBadRequest, func(r *openAIReasoningRecoveryState) { - cancelled, cancel := context.WithCancel(context.Background()) - cancel() - r.ctx = cancelled - }, false}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), test.body) - state.BindDiagnosticRequest([]byte(test.body), req) - _, retry := state.TryRecover(test.status, nil, []byte(test.payload), false) - require.False(t, retry) - if test.arrange != nil { - test.arrange(state) - } - var attempt error = NewOpenAIRequestRejectedError(test.status, nil) - if _, rejected := parseOpenAIReasoningRejection([]byte(test.payload)); rejected { - attempt = NewOpenAIContinuationStateUnavailableError(test.status, nil, []byte(test.payload)) - } - var failure *UpstreamFailoverError - stopped := state.StopError(attempt) - if !errors.As(stopped, &failure) { - var terminal *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, stopped, &terminal) - failure = terminal.Failure - } - require.Equal(t, test.want, failure.IsOpenAICiphertextAccountMismatch()) - require.Equal(t, test.want, failure.ShouldRetryNextAccount()) - require.True(t, failure.SuppressAccountHealthPenalty) - if test.want { - // The hand-off is on the Ops record whichever way it was recognised. - events, _ := state.c.Get(OpsUpstreamErrorsKey) - recorded := events.([]*OpsUpstreamErrorEvent) - require.True(t, recorded[len(recorded)-1].ContinuationDiagnostic.Recovery.AccountMismatch) - } - var original *UpstreamFailoverError - require.ErrorAs(t, attempt, &original) - require.Equal(t, original.Reason, failure.Reason, "the mark keeps the failure's own classification") - require.False(t, original.IsOpenAICiphertextAccountMismatch(), "the attempt's own error value is not modified") - }) - } -} - -func TestOpenAIReasoningRecoveryStopErrorHandsAnAccountFailureOfTheStrippedRetryOn(t *testing.T) { - provider := func() *UpstreamFailoverError { - return &UpstreamFailoverError{StatusCode: http.StatusBadGateway, ResponseBody: []byte(`{"error":{"code":"credit_balance_exhausted","message":"no credits"}}`), RetryableOnSameAccount: true, SameAccountRetryMax: 2} - } - tests := []struct { - name string - attempt func() error - arrange func(*openAIReasoningRecoveryState) - want bool - }{ - {"provider_failure", func() error { return provider() }, nil, true}, - {"request_rejected", func() error { return NewOpenAIRequestRejectedError(http.StatusBadRequest, nil) }, nil, false}, - {"refusal", func() error { return NewOpenAIRefusalRecoveryFailoverError(nil) }, nil, false}, - {"failure_that_stops", func() error { f := provider(); f.NextAccountAction = NextAccountStop; return f }, nil, false}, - {"unclassified_error", func() error { return errors.New("stream read error") }, nil, false}, - {"retry_not_sent", func() error { return provider() }, func(r *openAIReasoningRecoveryState) { r.retryDispatched = false }, false}, - {"output_already_sent", func() error { return provider() }, func(r *openAIReasoningRecoveryState) { r.semanticCommitted = true }, false}, - {"failure_already_forwarded", func() error { return provider() }, func(r *openAIReasoningRecoveryState) { - // A forwarded terminal always follows an observed failure payload. - r.ObserveFailure([]byte(`{"type":"response.failed","response":{"status":"failed","error":{"code":"credit_balance_exhausted"}}}`), false) - r.failureTerminalForwarded = true - }, false}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - state.BindDiagnosticRequest([]byte(reasoningRecoveryFixture), req) - _, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - // The stripped retry is sent; its failure is observed on a new attempt. - state.BindDiagnosticRequest([]byte(reasoningRecoveryFixture), req) - state.MarkAttemptDispatched() - state.redactPayload = func([]byte) []byte { return []byte(`{"error":{"message":"redacted"}}`) } - if test.arrange != nil { - test.arrange(state) - } - attempt := test.attempt() - stopped := state.StopError(attempt) - var failure *UpstreamFailoverError - var terminal *OpenAIReasoningRecoveryTerminalError - if errors.As(stopped, &terminal) { - require.False(t, test.want, "an account failure of the stripped retry is not a terminal") - require.False(t, terminal.Failure.ShouldRetryNextAccount()) - return - } - require.ErrorAs(t, stopped, &failure) - require.Equal(t, test.want, failure.IsOpenAIRecoveryRetrySpent()) - if !test.want { - return - } - require.True(t, failure.ShouldRetryNextAccount()) - require.Equal(t, http.StatusBadGateway, failure.StatusCode) - // Everything account health reads is what a first send carries. - require.True(t, failure.RetryableOnSameAccount) - require.Equal(t, 2, failure.SameAccountRetryMax) - require.JSONEq(t, `{"error":{"message":"redacted"}}`, string(failure.ResponseBody), "the body leaves the attempt redacted") - var original *UpstreamFailoverError - require.ErrorAs(t, attempt, &original) - require.False(t, original.IsOpenAIRecoveryRetrySpent(), "the attempt's own error value is not modified") - require.JSONEq(t, string(provider().ResponseBody), string(original.ResponseBody)) - events, _ := state.c.Get(OpsUpstreamErrorsKey) - recorded := events.([]*OpsUpstreamErrorEvent) - require.Equal(t, "recovery_failed", recorded[len(recorded)-1].Message) - require.True(t, recorded[len(recorded)-1].ContinuationDiagnostic.Recovery.AccountFailure) - }) - } -} - -func TestResetOpenAIReasoningRecoveryAttemptDetachesTheState(t *testing.T) { - state, _, rec := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - _, retry := state.TryRecover(http.StatusBadRequest, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - require.Same(t, state, openAIReasoningRecoveryStateFromContext(state.c)) - ResetOpenAIReasoningRecoveryAttempt(state.c) - require.Nil(t, openAIReasoningRecoveryStateFromContext(state.c), "the next account must not read this attempt") - require.Equal(t, "retry_without_encrypted_content", rec.Header().Get(openAIReasoningRecoveryHeader), "the header names the last recovery action of the request") -} - -func TestOpenAIReasoningRecoveryPreservesEveryNonCipherField(t *testing.T) { - state, _, rec := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - before := bytes.Clone(state.wire) - payload := []byte(`{"type":"response.failed","response":{"error":{"code":"invalid_encrypted_content","param":"input.1.encrypted_content"},"usage":{"input_tokens":3,"output_tokens":2}}}`) - signal := openAIReasoningRecoverySignal(state.c, payload, false) - require.Error(t, signal) - require.False(t, state.RecoveryAttempt(), "SSE signal must be pure before authoritative terminal") - after, retry := state.TryRecoverError(signal) - require.True(t, retry) - require.Equal(t, before, state.wire) - require.False(t, gjson.GetBytes(after, "input.1.encrypted_content").Exists()) - for _, path := range []string{"model", "reasoning", "store", "input.0", "input.1.id", "input.1.phase", "input.1.summary", "input.1.unknown", "input.2", "input.3"} { - require.Equal(t, gjson.GetBytes(before, path).Raw, gjson.GetBytes(after, path).Raw, path) - } - require.Equal(t, "retry_without_encrypted_content", rec.Header().Get(openAIReasoningRecoveryHeader)) - events, _ := state.c.Get(OpsUpstreamErrorsKey) - entry := events.([]*OpsUpstreamErrorEvent)[0] - require.Contains(t, entry.Detail, `"usage_status":"available"`) - require.Contains(t, entry.Detail, `"input_tokens":3`) - require.NotContains(t, entry.Detail, "opaque-old") - require.Empty(t, entry.UpstreamResponseBody) -} - -func TestOpenAIReasoningRecoveryCommittedCanceledDisabledAndFailover(t *testing.T) { - payload := []byte(`{"error":{"code":"invalid_encrypted_content"}}`) - state, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - require.Nil(t, openAIReasoningRecoverySignal(state.c, payload, true)) - _, retry := state.TryRecover(400, nil, payload, true) - require.False(t, retry) - state.enabled = false - _, retry = state.TryRecover(400, nil, payload, false) - require.False(t, retry) - state.enabled = true - _, retry = state.TryRecover(400, nil, payload, false) - require.True(t, retry) - var failover *UpstreamFailoverError - stopped := state.StopError(&UpstreamFailoverError{StatusCode: 503, RetryableOnSameAccount: true}) - require.False(t, errors.As(stopped, &failover)) - require.NotContains(t, stopped.Error(), "503") - ctx, cancel := context.WithCancel(context.Background()) - canceled, _, _ := newReasoningRecoveryTestState(t, ctx, reasoningRecoveryFixture) - cancel() - _, retry = canceled.TryRecover(400, nil, payload, false) - require.False(t, retry) - for _, status := range []int{401, 402, 403, 407, 429} { - protected, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - _, retry := protected.TryRecover(status, nil, payload, false) - require.False(t, retry, "authentication and quota statuses never gain an extra inference POST") - } -} - -func TestOpenAIReasoningRecoveryRetryPreservesContextAndIdentity(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - state, firstReq, _ := newReasoningRecoveryTestState(t, ctx, reasoningRecoveryFixture) - body, retry := state.TryRecover(400, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - req := firstReq.Clone(context.Background()) - req.Body = io.NopCloser(bytes.NewReader(body)) - req.GetBody = nil - req, actual, err := state.PrepareRequest(req, body, "") - require.NoError(t, err) - require.Equal(t, body, actual) - deadline, ok := req.Context().Deadline() - expected, _ := ctx.Deadline() - require.True(t, ok) - require.Equal(t, expected, deadline) - require.Nil(t, req.GetBody) - require.True(t, HTTPUpstreamRedirectsDisabled(req.Context())) - cancel() - select { - case <-req.Context().Done(): - case <-time.After(time.Second): - t.Fatal("retry detached caller cancellation") - } - for _, change := range []string{"auth", "endpoint", "non_responses_endpoint", "method", "nil_request", "proxy", "body"} { - t.Run(change, func(t *testing.T) { - s, original, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - stripped, _ := s.TryRecover(400, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - next := original.Clone(context.Background()) - next.GetBody = nil - proxy := "" - switch change { - case "auth": - next.Header.Set("Authorization", "Bearer different") - case "endpoint": - next.URL.Path = "/other/responses" - case "non_responses_endpoint": - next.URL.Path = "/v1/chat/completions" - case "method": - next.Method = http.MethodGet - case "nil_request": - next = nil - case "proxy": - proxy = "http://different-proxy.example" - case "body": - stripped = append(stripped, ' ') - } - _, _, err := s.PrepareRequest(next, stripped, proxy) - require.Error(t, err) - }) - } -} - -// reasoningRecoveryMemoryStore is the rejection memory of one test. Embedding a -// GatewayCache lets a forwarding test install it as the service cache. -type reasoningRecoveryMemoryStore struct { - GatewayCache - values map[string]OpenAIRejectedReasoning - gets int - puts int - fail bool -} - -var _ OpenAIReasoningStateStore = (*reasoningRecoveryMemoryStore)(nil) - -func (m *reasoningRecoveryMemoryStore) GetOpenAIRejectedReasoning(_ context.Context, scope OpenAIReasoningCacheScope, hashes []string) (map[string]OpenAIRejectedReasoning, error) { - m.gets++ - if m.fail { - return nil, errors.New("cache unavailable") - } - if len(hashes) > OpenAIReasoningStateMaxLookupEntries { - return nil, ErrOpenAIReasoningCacheInput - } - out := map[string]OpenAIRejectedReasoning{} - for _, hash := range hashes { - if value, ok := m.values[scope.ScopeHash+":"+hash]; ok { - out[hash] = value - } - } - return out, nil -} - -func (m *reasoningRecoveryMemoryStore) PutOpenAIRejectedReasoning(_ context.Context, scope OpenAIReasoningCacheScope, hashes []string) error { - if m.fail { - return errors.New("cache unavailable") - } - if len(hashes) > OpenAIReasoningStateMaxLookupEntries { - return ErrOpenAIReasoningCacheInput - } - m.puts++ - if m.values == nil { - m.values = map[string]OpenAIRejectedReasoning{} - } - for _, hash := range hashes { - m.values[scope.ScopeHash+":"+hash] = OpenAIRejectedReasoning{RejectedAt: time.Now(), ExpiresAt: time.Now().Add(24 * time.Hour)} - } - return nil -} - -func TestOpenAIReasoningRecoveryNegativeMemoryIsExactAndNonSliding(t *testing.T) { - store := &reasoningRecoveryMemoryStore{} - first, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - first.store = store - _, retry := first.TryRecover(400, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - require.Equal(t, 1, store.puts) - oldKey := first.scope.ScopeHash + ":" + openAIReasoningDigest([]byte("opaque-old")) - oldEntry := store.values[oldKey] - next, _, _ := newReasoningRecoveryTestState(t, context.Background(), reasoningRecoveryFixture) - next.store = store - body := strings.Replace(reasoningRecoveryFixture, `"output":"[]"}`, `"output":"[]"},{"type":"reasoning","encrypted_content":"opaque-new","summary":[]}`, 1) - stripped := next.skipRejectedHistory([]byte(body)) - require.False(t, gjson.GetBytes(stripped, "input.1.encrypted_content").Exists()) - require.Equal(t, "opaque-new", gjson.GetBytes(stripped, "input.4.encrypted_content").String()) - require.Equal(t, oldEntry, store.values[oldKey], "cache hit must not renew the rejection timestamp") - require.Equal(t, 1, store.puts) - next.scope.ScopeHash = strings.Repeat("b", 64) - require.Equal(t, body, string(next.skipRejectedHistory([]byte(body)))) - next.scope = first.scope - store.fail = true - require.Equal(t, body, string(next.skipRejectedHistory([]byte(body)))) - store.fail = false - store.values[oldKey] = OpenAIRejectedReasoning{RejectedAt: time.Now().Add(-25 * time.Hour), ExpiresAt: time.Now().Add(-time.Hour)} - require.Equal(t, body, string(next.skipRejectedHistory([]byte(body)))) -} - -func TestOpenAIReasoningRecoveryNegativeMemoryCoversLongHistory(t *testing.T) { - const items = 2*OpenAIReasoningStateMaxLookupEntries + 22 - var history strings.Builder - for index := range items { - if index > 0 { - history.WriteByte(',') - } - fmt.Fprintf(&history, `{"type":"reasoning","encrypted_content":"cipher-%d"}`, index) - } - body := `{"model":"gpt-5.6-sol","input":[` + history.String() + `]}` - store := &reasoningRecoveryMemoryStore{} - first, _, _ := newReasoningRecoveryTestState(t, context.Background(), body) - first.store = store - _, retry := first.TryRecover(400, nil, []byte(`{"error":{"code":"invalid_encrypted_content"}}`), false) - require.True(t, retry) - require.Equal(t, 3, store.puts, "the store takes at most one batch per call") - require.Len(t, store.values, items) - next, _, _ := newReasoningRecoveryTestState(t, context.Background(), body) - next.store = store - stripped := next.skipRejectedHistory([]byte(body)) - require.Equal(t, 3, store.gets) - require.Equal(t, items, next.cacheSkippedItems) - require.NotContains(t, string(stripped), "encrypted_content") -} - -func TestOpenAIReasoningRecoveryIdentityWithoutReasoningStillIsolatesSources(t *testing.T) { - state, req, _ := newReasoningRecoveryTestState(t, context.Background(), `{"model":"gpt-5.6-sol","input":"test"}`) - initial, err := buildOpenAIReasoningScope(state.c, state.account, req, []byte(`{"model":"gpt-5.6-sol"}`)) - require.NoError(t, err) - changed := req.Clone(context.Background()) - changed.Header.Set("Authorization", "Bearer different-source") - other, err := buildOpenAIReasoningScope(state.c, state.account, changed, []byte(`{"model":"gpt-5.6-sol"}`)) - require.NoError(t, err) - require.NotEqual(t, initial.ScopeHash, other.ScopeHash) - a := openAIReasoningRequestIdentity(state.account, req, "", "") - b := openAIReasoningRequestIdentity(state.account, req, "http://other-proxy.example", "") - require.NotEqual(t, a, b) - require.NotEqual(t, openAIReasoningDigest(nil), a) - one, err := buildOpenAIReasoningScope(state.c, state.account, req, []byte(`{"model":"gpt-5.6-sol","reasoning":{"effort":"xhigh","context":"keep"}}`)) - require.NoError(t, err) - two, err := buildOpenAIReasoningScope(state.c, state.account, req, []byte(`{"model":"gpt-5.6-sol","reasoning":{"context":"keep","effort":"xhigh"}}`)) - require.NoError(t, err) - require.Equal(t, one, two, "cache scope uses semantic JSON identity, not reasoning key order") -} - -// A status the pool retries on the same account is the pool's condition, not -// the account's: the API-key health breaker does not count it on a first send, -// and must not count it when it answers the stripped retry either. -func TestOpenAIReasoningRecoveryRetryAccountFailureKeepsThePoolExemption(t *testing.T) { - for _, passthrough := range []bool{false, true} { - t.Run(map[bool]string{true: "passthrough", false: "native"}[passthrough], func(t *testing.T) { - first := newJSONResponse(400, `{"error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}`) - first.Header.Set("Content-Type", "application/json") - second := newJSONResponse(http.StatusTooManyRequests, `{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded","message":"slow down"}}`) - second.Header.Set("Content-Type", "application/json") - upstream := &httpUpstreamRecorder{responses: []*http.Response{first, second}} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, _ := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Credentials["pool_mode"] = true - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - require.True(t, account.IsPoolMode() && account.IsPoolModeRetryableStatus(http.StatusTooManyRequests), "fixture: 429 is pool-retryable") - _, err := svc.Forward(context.Background(), c, account, []byte(reasoningRecoveryFixture)) - var failover *UpstreamFailoverError - require.ErrorAs(t, err, &failover) - require.True(t, failover.IsOpenAIRecoveryRetrySpent()) - require.True(t, failover.RetryableOnSameAccount) - _, _, counted := classifyOpenAIAPIKeyHealthFailure(err) - require.False(t, counted, "the breaker exemption of a first send is kept") - }) - } -} - -// An answer a first send classifies by more than its status keeps that -// classification when it answers the stripped retry, so account health sees -// the same failure either way. -func TestOpenAIReasoningRecoveryRetryAccountFailureKeepsTheFirstSendClassification(t *testing.T) { - tests := []struct { - name string - status int - body string - pool bool - check func(*testing.T, *UpstreamFailoverError, error) - }{ - { - name: "request_budget_rejected", - status: http.StatusForbidden, - body: `{"error":{"type":"balance_insufficient_error","code":"balance_insufficient","message":"private balance"}}`, - check: func(t *testing.T, failure *UpstreamFailoverError, _ error) { - require.Equal(t, GatewayFailureScopeRequest, failure.Scope) - require.True(t, failure.SuppressAccountHealthPenalty) - require.Equal(t, "upstream_request_budget_rejected", failure.ClientErrorCode) - }, - }, - { - name: "model_not_supported", - status: http.StatusBadRequest, - body: `{"error":{"type":"model_not_supported","code":"400","message":"The requested model is not supported"}}`, - check: func(t *testing.T, failure *UpstreamFailoverError, _ error) { - require.Equal(t, openAIModelNotSupportedReason, failure.Reason) - require.True(t, failure.SuppressAccountHealthPenalty) - }, - }, - { - name: "budget_exceeded_on_a_pool_account", - status: http.StatusTooManyRequests, - body: `{"error":{"type":"budget_exceeded","message":"budget spent"}}`, - pool: true, - check: func(t *testing.T, failure *UpstreamFailoverError, err error) { - require.Equal(t, GatewayFailureScopeAccount, failure.Scope) - require.False(t, failure.RetryableOnSameAccount, "a spent budget is the account's, whatever the pool retries") - _, _, counted := classifyOpenAIAPIKeyHealthFailure(err) - require.True(t, counted) - }, - }, - } - for _, test := range tests { - for _, passthrough := range []bool{false, true} { - t.Run(test.name+map[bool]string{true: "/passthrough", false: "/native"}[passthrough], func(t *testing.T) { - first := newJSONResponse(400, `{"error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}`) - first.Header.Set("Content-Type", "application/json") - second := newJSONResponse(test.status, test.body) - second.Header.Set("Content-Type", "application/json") - upstream := &httpUpstreamRecorder{responses: []*http.Response{first, second}} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, _ := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Credentials["pool_mode"] = test.pool - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - _, err := svc.Forward(context.Background(), c, account, []byte(reasoningRecoveryFixture)) - var failover *UpstreamFailoverError - require.ErrorAs(t, err, &failover) - require.Len(t, upstream.requests, 2) - require.True(t, failover.IsOpenAIRecoveryRetrySpent()) - require.True(t, failover.ShouldRetryNextAccount()) - require.Equal(t, test.status, failover.StatusCode) - test.check(t, failover, err) - }) - } - } -} - -// An answer to the stripped retry that a first send would not move to another -// account, and that is no ciphertext rejection either, is not the account's -// failure: the request ends on this account. -func TestOpenAIReasoningRecoveryRetryAnswerThatIsNoFailoverEndsTheRequest(t *testing.T) { - for _, passthrough := range []bool{false, true} { - t.Run(map[bool]string{true: "passthrough", false: "native"}[passthrough], func(t *testing.T) { - first := newJSONResponse(400, `{"error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}`) - first.Header.Set("Content-Type", "application/json") - second := newJSONResponse(http.StatusNotFound, `{"error":{"code":"not_found","message":"private upstream detail"}}`) - second.Header.Set("Content-Type", "application/json") - upstream := &httpUpstreamRecorder{responses: []*http.Response{first, second}} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, _ := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - _, err := svc.Forward(context.Background(), c, account, []byte(reasoningRecoveryFixture)) - require.Len(t, upstream.requests, 2) - var stopped *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, err, &stopped) - require.False(t, stopped.Failure.IsOpenAIRecoveryRetrySpent()) - require.False(t, stopped.Failure.ShouldRetryNextAccount()) - require.Equal(t, http.StatusNotFound, stopped.Failure.StatusCode) - }) - } -} - -func TestOpenAIReasoningRecoveryForwardHTTPAndPassthroughSingleRetry(t *testing.T) { - for _, passthrough := range []bool{false, true} { - for _, secondOK := range []bool{false, true} { - t.Run(strings.Join([]string{map[bool]string{true: "passthrough", false: "native"}[passthrough], map[bool]string{true: "completed", false: "rejected"}[secondOK]}, "/"), func(t *testing.T) { - first := newJSONResponse(400, `{"error":{"code":"thinking_signature_invalid","param":"input[1].encrypted_content"}}`) - first.Header.Set("Content-Type", "application/json") - second := newJSONResponse(503, `{"error":{"code":"server_error","message":"private upstream failure"}}`) - if secondOK { - second = newJSONResponse(200, `{"id":"resp_final","status":"completed","model":"gpt-5.6-sol","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":11,"output_tokens":7}}`) - } - second.Header.Set("Content-Type", "application/json") - upstream := &httpUpstreamRecorder{responses: []*http.Response{first, second}} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, rec := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true, "pool_mode_enabled": true, "pool_mode_retry_count": 0} - original := []byte(reasoningRecoveryFixture) - result, err := svc.Forward(context.Background(), c, account, original) - require.Len(t, upstream.requests, 2, "the extra retry is independent of pool retry count and never permits a third") - require.Equal(t, reasoningRecoveryFixture, string(original)) - require.True(t, gjson.GetBytes(upstream.bodies[0], "input.1.encrypted_content").Exists()) - require.False(t, gjson.GetBytes(upstream.bodies[1], "input.1.encrypted_content").Exists()) - require.Equal(t, gjson.GetBytes(upstream.bodies[0], "reasoning").Raw, gjson.GetBytes(upstream.bodies[1], "reasoning").Raw) - if secondOK { - require.NoError(t, err) - require.Equal(t, 11, result.Usage.InputTokens) - require.Equal(t, 7, result.Usage.OutputTokens) - require.Contains(t, rec.Body.String(), "resp_final") - } else { - require.Error(t, err) - // A provider failure of the stripped retry is the account's own: - // the request is handed to the next account, as on a first send, - // and never back to this one. - var stopped *OpenAIReasoningRecoveryTerminalError - require.False(t, errors.As(err, &stopped)) - var failover *UpstreamFailoverError - require.ErrorAs(t, err, &failover) - require.True(t, failover.IsOpenAIRecoveryRetrySpent()) - require.Equal(t, http.StatusServiceUnavailable, failover.StatusCode) - require.False(t, failover.IsOpenAIRequestRejected()) - require.False(t, failover.SuppressAccountHealthPenalty, "a genuine provider failure is not fabricated as a request validation error") - require.True(t, failover.ShouldReportAccountScheduleFailure()) - require.True(t, failover.ShouldRetryNextAccount()) - require.NotContains(t, err.Error(), "private upstream failure") - } - }) - } - } -} - -func TestOpenAIReasoningRecoveryForwardSSEAndBufferedPreserveFailedAttemptUsage(t *testing.T) { - failed := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_failed\",\"status\":\"in_progress\"}}\n\n" + - "data: {\"type\":\"error\",\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"}\n\n" + - "data: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_failed\",\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"},\"usage\":{\"input_tokens\":3,\"output_tokens\":2}}}\n\n" - completed := "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_success\",\"status\":\"completed\",\"model\":\"gpt-5.6-sol\",\"output\":[{\"id\":\"msg_success\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\"}]}],\"usage\":{\"input_tokens\":11,\"output_tokens\":7}}}\n\n" - for _, passthrough := range []bool{false, true} { - for _, stream := range []bool{false, true} { - name := map[bool]string{false: "native", true: "passthrough"}[passthrough] + "/" + map[bool]string{false: "buffered", true: "stream"}[stream] - t.Run(name, func(t *testing.T) { - upstream := &httpUpstreamRecorder{responses: []*http.Response{ - {StatusCode: 200, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(failed))}, - {StatusCode: 200, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(completed))}, - }} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, recorder := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - body := reasoningRecoveryFixture - if stream { - body = strings.Replace(body, `"store":false`, `"stream":true,"store":false`, 1) - } - result, err := svc.Forward(context.Background(), c, account, []byte(body)) - require.NoError(t, err) - require.Len(t, upstream.requests, 2) - require.Equal(t, 11, result.Usage.InputTokens) - require.Equal(t, 7, result.Usage.OutputTokens) - require.NotContains(t, recorder.Body.String(), "resp_failed") - require.Contains(t, recorder.Body.String(), "resp_success") - events, _ := c.Get(OpsUpstreamErrorsKey) - recorded := false - for _, entry := range events.([]*OpsUpstreamErrorEvent) { - if entry.Kind == "reasoning_recovery" && entry.Message == "retry_without_encrypted_content" { - recorded = true - require.Contains(t, entry.Detail, `"input_tokens":3`) - require.Contains(t, entry.Detail, `"output_tokens":2`) - // The rejected upstream payload itself, not only its hash. - require.Contains(t, gjson.Get(entry.Detail, "payload").String(), `"invalid_encrypted_content"`) - require.NotEmpty(t, gjson.Get(entry.Detail, "payload_sha256").String()) - } - } - require.True(t, recorded) - }) - } - } -} - -func TestOpenAIReasoningRecoveryNeverReplaysCommittedSSE(t *testing.T) { - for _, passthrough := range []bool{false, true} { - t.Run(map[bool]string{false: "native", true: "passthrough"}[passthrough], func(t *testing.T) { - body := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"visible answer\"}\n\n" + - "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"code\":\"invalid_encrypted_content\",\"param\":\"input[1].encrypted_content\"},\"usage\":{\"input_tokens\":3,\"output_tokens\":2}}}\n\n" - upstream := &httpUpstreamRecorder{resp: &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader(body))}} - svc := newOpenAIImageGenerationControlTestService(upstream) - c, recorder := newOpenAIImageGenerationControlTestContext(false, "test-client") - account := newOpenAIImageGenerationControlTestAccount() - account.Extra = map[string]any{"openai_passthrough": passthrough, "responses_api_supported": true} - input := strings.Replace(reasoningRecoveryFixture, `"store":false`, `"stream":true,"store":false`, 1) - _, err := svc.Forward(context.Background(), c, account, []byte(input)) - require.Error(t, err) - require.Len(t, upstream.requests, 1) - require.Contains(t, recorder.Body.String(), "visible answer") - var failover *UpstreamFailoverError - require.False(t, errors.As(err, &failover), "post-commit terminal errors cannot be externally replayed") - var terminal *OpenAIReasoningRecoveryTerminalError - require.ErrorAs(t, err, &terminal) - require.True(t, terminal.FailureTerminalForwarded, "the existing failed terminal must not be rendered a second time") - }) - } -} diff --git a/backend/internal/service/openai_reasoning_state_cache.go b/backend/internal/service/openai_reasoning_state_cache.go deleted file mode 100644 index f7a4454f00a0..000000000000 --- a/backend/internal/service/openai_reasoning_state_cache.go +++ /dev/null @@ -1,245 +0,0 @@ -package service - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "io" - "strings" - "sync" - "time" - - "github.com/gin-gonic/gin" -) - -const ( - OpenAIReasoningStateTTL = 24 * time.Hour - OpenAIReasoningStateIOBudget = 100 * time.Millisecond - OpenAIRejectedReasoningMaxBytes = 8 << 20 - OpenAIRejectedReasoningMaxEntries = 32768 - OpenAIReasoningStateMaxLookupEntries = 64 -) - -var ( - ErrOpenAIReasoningCacheInput = errors.New("invalid reasoning state cache input") - ErrOpenAIReasoningCacheBudget = errors.New("reasoning state cache I/O budget exhausted") -) - -// OpenAIReasoningCacheScope contains only digests, never authentication values, -// upstream URLs, prompts, or ciphertext. TenantHash bounds one downstream key's -// storage independently of its current model, group, and upstream account. -type OpenAIReasoningCacheScope struct { - ScopeHash string - TenantHash string -} - -// OpenAIReasoningScopeInput describes the actual selected source and wire -// configuration, not the original requested model or an account-pool label. -// SourceIdentityHash must cover authentication identity and the selected proxy. -type OpenAIReasoningScopeInput struct { - UserID int64 - APIKeyID int64 - GroupID int64 - AccountID int64 - SourceIdentityHash string - Endpoint string - Model string - Reasoning json.RawMessage -} - -// BuildOpenAIReasoningCacheScope refuses anonymous/unknown-source caches. Every -// reasoning subfield participates in identity; mode is never inferred as effort. -func BuildOpenAIReasoningCacheScope(input OpenAIReasoningScopeInput) (OpenAIReasoningCacheScope, error) { - if input.UserID <= 0 || input.APIKeyID <= 0 || input.GroupID < 0 || input.AccountID <= 0 || - !IsOpenAIReasoningCacheDigest(input.SourceIdentityHash) || - strings.TrimSpace(input.Endpoint) == "" || strings.TrimSpace(input.Model) == "" { - return OpenAIReasoningCacheScope{}, ErrOpenAIReasoningCacheInput - } - canonical, err := canonicalReasoningCacheJSON(input.Reasoning) - if err != nil { - return OpenAIReasoningCacheScope{}, ErrOpenAIReasoningCacheInput - } - input.Reasoning = canonical - identity, err := json.Marshal(input) - if err != nil { - return OpenAIReasoningCacheScope{}, ErrOpenAIReasoningCacheInput - } - tenant, _ := json.Marshal([2]int64{input.UserID, input.APIKeyID}) - return OpenAIReasoningCacheScope{ - ScopeHash: reasoningStateDigest(identity), - TenantHash: reasoningStateDigest(tenant), - }, nil -} - -func canonicalReasoningCacheJSON(raw json.RawMessage) ([]byte, error) { - if len(bytes.TrimSpace(raw)) == 0 { - return []byte("null"), nil - } - decoder := json.NewDecoder(bytes.NewReader(raw)) - decoder.UseNumber() - value, err := decodeReasoningCacheValue(decoder, 0) - if err != nil { - return nil, err - } - if err := decoder.Decode(new(any)); err != io.EOF { - return nil, ErrOpenAIReasoningCacheInput - } - return json.Marshal(value) -} - -// A standard map decode silently collapses duplicate keys. Cache identity must -// never merge two potentially different wire configurations, so ambiguous or -// excessive nesting disables only this optional cache, not normal forwarding. -func decodeReasoningCacheValue(decoder *json.Decoder, depth int) (any, error) { - if depth > 64 { - return nil, ErrOpenAIReasoningCacheInput - } - token, err := decoder.Token() - if err != nil { - return nil, err - } - delim, container := token.(json.Delim) - if !container { - return token, nil - } - switch delim { - case '{': - value := make(map[string]any) - for decoder.More() { - keyToken, err := decoder.Token() - if err != nil { - return nil, err - } - key, valid := keyToken.(string) - if !valid { - return nil, ErrOpenAIReasoningCacheInput - } - if _, duplicate := value[key]; duplicate { - return nil, ErrOpenAIReasoningCacheInput - } - child, err := decodeReasoningCacheValue(decoder, depth+1) - if err != nil { - return nil, err - } - value[key] = child - } - if end, err := decoder.Token(); err != nil || end != json.Delim('}') { - return nil, ErrOpenAIReasoningCacheInput - } - return value, nil - case '[': - value := make([]any, 0) - for decoder.More() { - child, err := decodeReasoningCacheValue(decoder, depth+1) - if err != nil { - return nil, err - } - value = append(value, child) - } - if end, err := decoder.Token(); err != nil || end != json.Delim(']') { - return nil, ErrOpenAIReasoningCacheInput - } - return value, nil - default: - return nil, ErrOpenAIReasoningCacheInput - } -} - -func reasoningStateDigest(value []byte) string { - sum := sha256.Sum256(value) - return hex.EncodeToString(sum[:]) -} - -func IsOpenAIReasoningCacheDigest(value string) bool { - if len(value) != sha256.Size*2 { - return false - } - for _, ch := range value { - if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { - return false - } - } - return true -} - -type OpenAIRejectedReasoning struct { - RejectedAt time.Time - ExpiresAt time.Time -} - -// OpenAIReasoningStateStore is optional: production GatewayCache implements it, -// but unrelated cache adapters and their mocks need not grow new methods. Cache -// errors mean unavailable, never evidence that any history should be erased. -type OpenAIReasoningStateStore interface { - GetOpenAIRejectedReasoning(ctx context.Context, scope OpenAIReasoningCacheScope, cipherHashes []string) (map[string]OpenAIRejectedReasoning, error) - // Put is only for a real structured rejection. Get never refreshes this TTL. - PutOpenAIRejectedReasoning(ctx context.Context, scope OpenAIReasoningCacheScope, cipherHashes []string) error -} - -func (s *OpenAIGatewayService) openAIReasoningStateStore() OpenAIReasoningStateStore { - if s == nil || s.cache == nil { - return nil - } - store, _ := s.cache.(OpenAIReasoningStateStore) - return store -} - -// OpenAIReasoningCacheBudget charges only I/O time, not model generation time. -// The rejection lookup and the rejection write share this one budget. -// Callbacks must respect the derived context and must not start detached work. -type OpenAIReasoningCacheBudget struct { - mu sync.Mutex - remaining time.Duration -} - -func NewOpenAIReasoningCacheBudget() *OpenAIReasoningCacheBudget { - return &OpenAIReasoningCacheBudget{remaining: OpenAIReasoningStateIOBudget} -} - -const openAIReasoningCacheBudgetContextKey = "openai_reasoning_state_cache_io_budget" - -// Gin's individual Get/Set methods are safe, but their get-or-create pair is -// not atomic. Keep concurrent consumers of one request on the same budget. -var openAIReasoningCacheBudgetInit sync.Mutex - -func openAIReasoningCacheBudgetForRequest(c *gin.Context) *OpenAIReasoningCacheBudget { - if c == nil { - return NewOpenAIReasoningCacheBudget() - } - openAIReasoningCacheBudgetInit.Lock() - defer openAIReasoningCacheBudgetInit.Unlock() - if existing, ok := c.Get(openAIReasoningCacheBudgetContextKey); ok { - if budget, valid := existing.(*OpenAIReasoningCacheBudget); valid && budget != nil { - return budget - } - } - budget := NewOpenAIReasoningCacheBudget() - c.Set(openAIReasoningCacheBudgetContextKey, budget) - return budget -} - -func (b *OpenAIReasoningCacheBudget) Do(ctx context.Context, fn func(context.Context) error) error { - if b == nil || fn == nil { - return ErrOpenAIReasoningCacheBudget - } - b.mu.Lock() - defer b.mu.Unlock() - if err := ctx.Err(); err != nil { - return err - } - if b.remaining <= 0 { - return ErrOpenAIReasoningCacheBudget - } - started := time.Now() - ioCtx, cancel := context.WithTimeout(ctx, b.remaining) - defer cancel() - err := fn(ioCtx) - b.remaining -= time.Since(started) - if err != nil { - return err - } - return ioCtx.Err() -} diff --git a/backend/internal/service/openai_reasoning_state_cache_test.go b/backend/internal/service/openai_reasoning_state_cache_test.go deleted file mode 100644 index 397b6a286c90..000000000000 --- a/backend/internal/service/openai_reasoning_state_cache_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "errors" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" -) - -func TestOpenAIReasoningCacheScope(t *testing.T) { - base := OpenAIReasoningScopeInput{ - UserID: 1, APIKeyID: 2, GroupID: 3, AccountID: 4, - SourceIdentityHash: strings.Repeat("a", 64), Endpoint: "https://provider.example/v1/responses", Model: "gpt-test", - Reasoning: json.RawMessage(`{"effort":"xhigh","mode":"standard","context":{"count":9007199254740993}}`), - } - original, err := BuildOpenAIReasoningCacheScope(base) - require.NoError(t, err) - require.True(t, IsOpenAIReasoningCacheDigest(original.ScopeHash)) - require.True(t, IsOpenAIReasoningCacheDigest(original.TenantHash)) - reordered := base - reordered.Reasoning = json.RawMessage(`{ "context": {"count": 9007199254740993}, "mode":"standard", "effort":"xhigh" }`) - identical, err := BuildOpenAIReasoningCacheScope(reordered) - require.NoError(t, err) - require.Equal(t, original, identical) - - changes := map[string]func(*OpenAIReasoningScopeInput){ - "user": func(v *OpenAIReasoningScopeInput) { v.UserID++ }, - "key": func(v *OpenAIReasoningScopeInput) { v.APIKeyID++ }, - "group": func(v *OpenAIReasoningScopeInput) { v.GroupID++ }, - "account": func(v *OpenAIReasoningScopeInput) { v.AccountID++ }, - "identity": func(v *OpenAIReasoningScopeInput) { v.SourceIdentityHash = strings.Repeat("b", 64) }, - "endpoint": func(v *OpenAIReasoningScopeInput) { v.Endpoint += "/compact" }, - "model": func(v *OpenAIReasoningScopeInput) { v.Model += "-other" }, - "mode": func(v *OpenAIReasoningScopeInput) { - v.Reasoning = json.RawMessage(`{"effort":"xhigh","mode":"pro","context":{"count":9007199254740993}}`) - }, - "integer beyond float precision": func(v *OpenAIReasoningScopeInput) { - v.Reasoning = json.RawMessage(`{"effort":"xhigh","mode":"standard","context":{"count":9007199254740992}}`) - }, - } - for name, change := range changes { - t.Run(name, func(t *testing.T) { - changed := base - change(&changed) - scope, err := BuildOpenAIReasoningCacheScope(changed) - require.NoError(t, err) - require.NotEqual(t, original.ScopeHash, scope.ScopeHash) - if name != "user" && name != "key" { - require.Equal(t, original.TenantHash, scope.TenantHash) - } else { - require.NotEqual(t, original.TenantHash, scope.TenantHash) - } - }) - } - - for _, raw := range []string{`{"mode":"pro","mode":"standard"}`, `{"context":{"x":1,"x":2}}`, `{} {}`, `{"effort":`} { - invalid := base - invalid.Reasoning = json.RawMessage(raw) - _, err := BuildOpenAIReasoningCacheScope(invalid) - require.ErrorIs(t, err, ErrOpenAIReasoningCacheInput) - } - base.UserID = 0 - _, err = BuildOpenAIReasoningCacheScope(base) - require.ErrorIs(t, err, ErrOpenAIReasoningCacheInput) -} - -func TestOpenAIReasoningCacheBudget(t *testing.T) { - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - budget := openAIReasoningCacheBudgetForRequest(c) - require.Same(t, budget, openAIReasoningCacheBudgetForRequest(c)) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - called := false - require.ErrorIs(t, budget.Do(ctx, func(context.Context) error { called = true; return nil }), context.Canceled) - require.False(t, called) - - budget.remaining = 10 * time.Millisecond - err := budget.Do(context.Background(), func(ioCtx context.Context) error { - <-ioCtx.Done() - return ioCtx.Err() - }) - require.ErrorIs(t, err, context.DeadlineExceeded) - require.ErrorIs(t, budget.Do(context.Background(), func(context.Context) error { called = true; return nil }), ErrOpenAIReasoningCacheBudget) - require.False(t, called) -} - -func TestOpenAIReasoningCacheBudgetChargesIOOnly(t *testing.T) { - budget := NewOpenAIReasoningCacheBudget() - errMarker := errors.New("cache unavailable") - require.ErrorIs(t, budget.Do(context.Background(), func(context.Context) error { return errMarker }), errMarker) - remainingAfterIO := budget.remaining - // Simulate time between operations (model generation). It must not consume - // the write-back budget as a single absolute request deadline would. - time.Sleep(15 * time.Millisecond) - require.NoError(t, budget.Do(context.Background(), func(ctx context.Context) error { - deadline, ok := ctx.Deadline() - require.True(t, ok) - require.Greater(t, time.Until(deadline), remainingAfterIO-10*time.Millisecond) - return nil - })) -} - -func TestOpenAIReasoningCacheBudgetConcurrentRequestConsumers(t *testing.T) { - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - results := make(chan *OpenAIReasoningCacheBudget, 8) - for range cap(results) { - go func() { results <- openAIReasoningCacheBudgetForRequest(c) }() - } - first := <-results - for range cap(results) - 1 { - require.Same(t, first, <-results) - } -} diff --git a/backend/internal/service/openai_responses_namespace.go b/backend/internal/service/openai_responses_namespace.go index 6d6418accd79..ec6fc62da186 100644 --- a/backend/internal/service/openai_responses_namespace.go +++ b/backend/internal/service/openai_responses_namespace.go @@ -242,38 +242,40 @@ func stripOpenAIResponsesInputNamespaces(body []byte, keepToolCallNamespaces boo var rebuilt bytes.Buffer keepStandaloneOutputNamespaces := len(standalone) > 0 && standalone[0] - rebuilt.Grow(len(input.Raw)) - _ = rebuilt.WriteByte('[') changed := false - first := true + // Offsets below are relative to input.Raw, not the full request body. + copyFrom := 0 var stripErr error input.ForEach(func(_, item gjson.Result) bool { - if !first { - _ = rebuilt.WriteByte(',') + if !item.IsObject() || !item.Get("namespace").Exists() || + (keepToolCallNamespaces && isOpenAIResponsesToolCallItemType(item.Get("type").String())) || + (keepStandaloneOutputNamespaces && isOpenAIResponsesNamedStandaloneOutput(item)) { + return true } - first = false - itemBody := []byte(item.Raw) - // 先判存在再判类型:长历史里绝大多数是 message/reasoning 等不带 namespace - // 的项,这样它们无需再扫一次 type。 - keepNamespace := keepToolCallNamespaces && isOpenAIResponsesToolCallItemType(item.Get("type").String()) - keepNamespace = keepNamespace || (keepStandaloneOutputNamespaces && isOpenAIResponsesNamedStandaloneOutput(item)) - if item.IsObject() && item.Get("namespace").Exists() && !keepNamespace { - itemBody, stripErr = sjson.DeleteBytes(itemBody, "namespace") - if stripErr != nil { - return false - } + itemBody, err := sjson.DeleteBytes([]byte(item.Raw), "namespace") + if err != nil { + stripErr = err + return false + } + if !changed { + // Allocate only on the first change; copy unchanged spans without + // traversing their items again or converting them to byte slices. + rebuilt.Grow(len(input.Raw)) changed = true } + itemStart := item.Index - input.Index + _, _ = rebuilt.WriteString(input.Raw[copyFrom:itemStart]) _, _ = rebuilt.Write(itemBody) + copyFrom = itemStart + len(item.Raw) return true }) - _ = rebuilt.WriteByte(']') if stripErr != nil { return body, fmt.Errorf("delete OpenAI input namespace: %w", stripErr) } if !changed { return body, nil } + _, _ = rebuilt.WriteString(input.Raw[copyFrom:]) stripped, err := sjson.SetRawBytes(body, "input", rebuilt.Bytes()) if err != nil { return body, fmt.Errorf("replace OpenAI input after namespace deletion: %w", err) diff --git a/backend/internal/service/openai_responses_namespace_benchmark_test.go b/backend/internal/service/openai_responses_namespace_benchmark_test.go new file mode 100644 index 000000000000..9297354b0219 --- /dev/null +++ b/backend/internal/service/openai_responses_namespace_benchmark_test.go @@ -0,0 +1,55 @@ +package service + +import ( + "bytes" + "fmt" + "runtime" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +func BenchmarkStripOpenAIResponsesInputNamespaces(b *testing.B) { + for _, size := range []int{1 << 20, 8 << 20} { + message := `{"type":"message","content":"` + strings.Repeat("x", size) + `"}` + call := `{"type":"function_call","namespace":"files","name":"search"}` + for _, tc := range []struct { + name string + input string + keep bool + stripIndex int + }{ + {"tools_only", message, false, -1}, + {"nested_only", `{"content":{"namespace":"nested"}},` + message, false, -1}, + {"keep_calls", call + "," + message, true, -1}, + {"strip_first", call + "," + message, false, 0}, + {"strip_last", message + "," + call, false, 1}, + } { + b.Run(fmt.Sprintf("%s/%dMiB", tc.name, size>>20), func(b *testing.B) { + body := []byte(`{"tools":[{"type":"namespace","name":"files","tools":[]}],"input":[` + tc.input + `]}`) + out, err := stripOpenAIResponsesInputNamespaces(body, tc.keep) + if err != nil { + b.Fatal(err) + } + if tc.stripIndex < 0 { + if !bytes.Equal(body, out) { + b.Fatal("no-op changed body") + } + } else if !gjson.ValidBytes(out) || gjson.GetBytes(out, fmt.Sprintf("input.%d.namespace", tc.stripIndex)).Exists() { + b.Fatal("namespace was not removed") + } + b.ReportAllocs() + b.SetBytes(int64(len(body))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + out, err = stripOpenAIResponsesInputNamespaces(body, tc.keep) + if err != nil { + b.Fatal(err) + } + runtime.KeepAlive(out) + } + }) + } + } +} diff --git a/backend/internal/service/openai_responses_namespace_test.go b/backend/internal/service/openai_responses_namespace_test.go index 5d847981b730..22364c6265d2 100644 --- a/backend/internal/service/openai_responses_namespace_test.go +++ b/backend/internal/service/openai_responses_namespace_test.go @@ -2,6 +2,7 @@ package service import ( "strconv" + "strings" "testing" "github.com/stretchr/testify/require" @@ -179,6 +180,8 @@ func TestStripOpenAIResponsesInputNamespaces(t *testing.T) { func TestStripOpenAIResponsesInputNamespacesLeavesOtherShapesByteExact(t *testing.T) { tests := [][]byte{ + []byte(`{"input":[],"tools":[{"type":"namespace"}]}`), + []byte(`{"input":[null,42,"namespace",{"content":{"namespace":"nested"}}]}`), []byte(`{"input":"text","namespace":"top-level"}`), []byte(`{"input":{"namespace":"single-object"}}`), []byte(`{"input":[{"content":{"namespace":"nested-only"}}],"tools":[{"namespace":"keep"}]}`), @@ -192,6 +195,42 @@ func TestStripOpenAIResponsesInputNamespacesLeavesOtherShapesByteExact(t *testin } } +func TestStripOpenAIResponsesInputNamespacesPreservesUnchangedSpans(t *testing.T) { + for _, input := range []string{ + `[ {"namespace":"drop","type":"message"} ]`, + `[ null, 9007199254740993, {"namespace":"drop","type":"message"} ]`, + `[ {"namespace":"drop","type":"message"}, "tail", {"content":{"namespace":"nested"}} ]`, + `[ {"namespace":"drop","type":"message"}, {"type":"function_call","namespace":"keep"}, {"namespace":"drop","type":"message"} ]`, + } { + t.Run(input, func(t *testing.T) { + body := []byte(" \n{\"prefix\":9007199254740993,\"input\":" + input + ",\"tail\":true} \n") + original := append([]byte(nil), body...) + out, err := stripOpenAIResponsesInputNamespaces(body, true) + require.NoError(t, err) + require.True(t, gjson.ValidBytes(out)) + require.Equal(t, strings.ReplaceAll(string(body), `"namespace":"drop",`, ""), string(out)) + require.Equal(t, original, body, "must not mutate the input body") + }) + } +} + +func TestStripOpenAIResponsesInputNamespacesNullAndKeptPrefix(t *testing.T) { + for _, keep := range []bool{false, true} { + body := []byte(`{"input":[null,42,"keep",{"type":"function_call","namespace":"files"},{"type":"message","namespace":null,"content":"keep"}]}`) + out, err := stripOpenAIResponsesInputNamespaces(body, keep) + require.NoError(t, err) + require.True(t, gjson.ValidBytes(out)) + require.Equal(t, 5, len(gjson.GetBytes(out, "input").Array())) + for index := 0; index < 3; index++ { + path := "input." + strconv.Itoa(index) + require.Equal(t, gjson.GetBytes(body, path).Raw, gjson.GetBytes(out, path).Raw) + } + require.Equal(t, keep, gjson.GetBytes(out, "input.3.namespace").Exists()) + require.False(t, gjson.GetBytes(out, "input.4.namespace").Exists()) + require.Equal(t, "keep", gjson.GetBytes(out, "input.4.content").String()) + } +} + // 保留模式下只有工具调用项留住 namespace:上游按 namespace 解析历史调用, // 而 message / reasoning / 输出项带该字段会被 schema 拒绝。 func TestStripOpenAIResponsesInputNamespacesKeepsToolCallNamespaces(t *testing.T) { diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index bb291c727469..f88971222c7e 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -662,6 +662,22 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( } } bridgePayloadRaw := currentBridgePayload.payloadRaw + if c != nil && sessionHash != "" { + c.Set(openAIWSIngressSessionHashContextKey, sessionHash) + } + if invalidDigests := s.sessionInvalidEncryptedContentDigests(groupID, sessionHash); len(invalidDigests) > 0 { + strippedPayload, strippedCount := s.stripSessionInvalidEncryptedContentLogged( + currentBridgePayload.payloadRaw, invalidDigests, "ingress_ws_http_bridge_invalid_encrypted_lineage_strip", account.ID, turn, + ) + if strippedCount > 0 { + currentBridgePayload.payloadRaw = strippedPayload + currentBridgePayload.payloadBytes = len(strippedPayload) + } + if bridgeReplayInputExists { + bridgeReplayInput, _ = stripOpenAIInvalidEncryptedContentFromReplayItems(bridgeReplayInput, invalidDigests) + } + + } bridgePayloadBytes := currentBridgePayload.payloadBytes toolOutputCoverage := AnalyzeToolCallOutputContextCoverageBytes(currentBridgePayload.payloadRaw) needsBridgeReplay := currentBridgePayload.previousResponseID != "" || @@ -1677,6 +1693,20 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( } } skipBeforeTurn = false + if c != nil && sessionHash != "" { + c.Set(openAIWSIngressSessionHashContextKey, sessionHash) + } + if invalidDigests := s.sessionInvalidEncryptedContentDigests(groupID, sessionHash); len(invalidDigests) > 0 { + strippedPayload, strippedCount := s.stripSessionInvalidEncryptedContentLogged( + currentPayload, invalidDigests, "ingress_ws_invalid_encrypted_lineage_strip", account.ID, turn, + ) + if strippedCount > 0 { + currentPayload = strippedPayload + } + if lastTurnReplayInputExists { + lastTurnReplayInput, _ = stripOpenAIInvalidEncryptedContentFromReplayItems(lastTurnReplayInput, invalidDigests) + } + } boundaryPayload, contextWindowBoundary, boundaryErr := normalizeOpenAIWSContextWindowBoundary( currentPayload, lastTurnWindowID, diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 857d05b5693c..b13aa10d5b66 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -655,6 +655,12 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } continue } + if resp.StatusCode == http.StatusBadRequest && + extractUpstreamErrorCode(respBody) == openAIWSFallbackReasonInvalidEncryptedContent { + s.markOpenAIWSInvalidEncryptedContentLineageFromPayload( + c, body, "ingress_ws_http_bridge_invalid_encrypted_lineage_mark", account.ID, turn, + ) + } upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) if upstreamMsg == "" { upstreamMsg = http.StatusText(resp.StatusCode) @@ -908,6 +914,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( shouldFailover := openAIStreamFailedEventShouldFailoverForAccount(account, upstreamMessage, errMessage) if eventType == "error" { errCodeRaw, errTypeRaw, _ := parseOpenAIWSErrorEventFields(upstreamMessage) + if reason, _ := classifyOpenAIWSErrorEventFromRaw(errCodeRaw, errTypeRaw, errMessage); reason == openAIWSFallbackReasonInvalidEncryptedContent { + s.markOpenAIWSInvalidEncryptedContentLineageFromPayload(c, body, "ingress_ws_http_bridge_invalid_encrypted_lineage_mark", account.ID, turn) + } statusCode = openAIWSErrorHTTPStatusFromRaw(errCodeRaw, errTypeRaw) shouldFailover = s.shouldFailoverOpenAIUpstreamResponse(account, statusCode, errMessage, upstreamMessage) // A model_not_supported event is an account/model capability diff --git a/backend/internal/service/openai_ws_protocol_forward_test.go b/backend/internal/service/openai_ws_protocol_forward_test.go index 0f325bacec7b..85d6131fc602 100644 --- a/backend/internal/service/openai_ws_protocol_forward_test.go +++ b/backend/internal/service/openai_ws_protocol_forward_test.go @@ -1,7 +1,6 @@ package service import ( - "bytes" "context" "encoding/json" "io" @@ -14,54 +13,12 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" - "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" ) -type httpUpstreamSequenceRecorder struct { - mu sync.Mutex - bodies [][]byte - reqs []*http.Request - - responses []*http.Response - errs []error - callCount int -} - -func (u *httpUpstreamSequenceRecorder) Do(req *http.Request, proxyURL string, accountID int64, accountConcurrency int) (*http.Response, error) { - u.mu.Lock() - defer u.mu.Unlock() - - idx := u.callCount - u.callCount++ - u.reqs = append(u.reqs, req) - if req != nil && req.Body != nil { - b, _ := io.ReadAll(req.Body) - u.bodies = append(u.bodies, b) - _ = req.Body.Close() - req.Body = io.NopCloser(bytes.NewReader(b)) - } else { - u.bodies = append(u.bodies, nil) - } - if idx < len(u.errs) && u.errs[idx] != nil { - return nil, u.errs[idx] - } - if idx < len(u.responses) { - return u.responses[idx], nil - } - if len(u.responses) == 0 { - return nil, nil - } - return u.responses[len(u.responses)-1], nil -} - -func (u *httpUpstreamSequenceRecorder) DoWithTLS(req *http.Request, proxyURL string, accountID int64, accountConcurrency int, profile *tlsfingerprint.Profile) (*http.Response, error) { - return u.Do(req, proxyURL, accountID, accountConcurrency) -} - func TestOpenAIGatewayService_Forward_PreservePreviousResponseIDWhenWSEnabled(t *testing.T) { gin.SetMode(gin.TestMode) wsFallbackServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -188,178 +145,6 @@ func TestOpenAIGatewayService_Forward_HTTPIngressPreservesNativeContinuationWhen require.Equal(t, "client_protocol_http", reason) } -func TestOpenAIGatewayService_Forward_HTTPIngressInvalidEncryptedContentIsTerminalWhenRecoveryDisabled(t *testing.T) { - gin.SetMode(gin.TestMode) - wsFallbackServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.NotFound(w, r) - })) - defer wsFallbackServer.Close() - - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) - c.Request.Header.Set("User-Agent", "custom-client/1.0") - SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - - upstream := &httpUpstreamSequenceRecorder{ - responses: []*http.Response{ - { - StatusCode: http.StatusBadRequest, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader( - `{"error":{"code":"invalid_encrypted_content","type":"invalid_request_error","message":"The encrypted content could not be verified."}}`, - )), - }, - { - StatusCode: http.StatusOK, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader( - `{"id":"resp_http_retry_ok","usage":{"input_tokens":1,"output_tokens":2,"input_tokens_details":{"cached_tokens":0}}}`, - )), - }, - }, - } - - cfg := &config.Config{} - cfg.Security.URLAllowlist.Enabled = false - cfg.Security.URLAllowlist.AllowInsecureHTTP = true - cfg.Gateway.OpenAIWS.Enabled = true - cfg.Gateway.OpenAIWS.OAuthEnabled = true - cfg.Gateway.OpenAIWS.APIKeyEnabled = true - cfg.Gateway.OpenAIWS.ResponsesWebsocketsV2 = true - - svc := &OpenAIGatewayService{ - cfg: cfg, - httpUpstream: upstream, - openaiWSResolver: NewOpenAIWSProtocolResolver(cfg), - } - - account := &Account{ - ID: 102, - Name: "openai-apikey", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{ - "api_key": "sk-test", - "base_url": wsFallbackServer.URL, - }, - Extra: map[string]any{ - "responses_websockets_v2_enabled": true, - }, - } - svc.SetReasoningRecoveryService(newReasoningRecoverySwitchForTest(false)) - - body := []byte(`{"model":"gpt-5.1","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA","summary":[{"type":"summary_text","text":"keep me"}]},{"type":"input_text","text":"hello"}]}`) - result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Equal(t, 1, upstream.callCount, "不能删除已拒绝的推理状态后重试") - require.Len(t, upstream.bodies, 1) - require.False(t, c.Writer.Written(), "continuation 应交由 handler 输出标准终态,不能伪造成功") - - firstBody := upstream.bodies[0] - require.Equal(t, "gAAA", gjson.GetBytes(firstBody, "input.0.encrypted_content").String(), "首次请求不应做发送前预清理") - require.Equal(t, "keep me", gjson.GetBytes(firstBody, "input.0.summary.0.text").String()) - require.JSONEq(t, gjson.GetBytes(body, "input").Raw, gjson.GetBytes(firstBody, "input").Raw) - - decision, _ := c.Get("openai_ws_transport_decision") - reason, _ := c.Get("openai_ws_transport_reason") - require.Equal(t, string(OpenAIUpstreamTransportHTTPSSE), decision) - require.Equal(t, "client_protocol_http", reason) -} - -func TestOpenAIGatewayService_Forward_HTTPIngressWrappedInvalidEncryptedContentIsNotRepairedOnTheAccount(t *testing.T) { - gin.SetMode(gin.TestMode) - wsFallbackServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.NotFound(w, r) - })) - defer wsFallbackServer.Close() - - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) - c.Request.Header.Set("User-Agent", "custom-client/1.0") - SetOpenAIClientTransport(c, OpenAIClientTransportHTTP) - - upstream := &httpUpstreamSequenceRecorder{ - responses: []*http.Response{ - { - StatusCode: http.StatusBadRequest, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader( - `{"error":{"code":null,"message":"{\"error\":{\"message\":\"The encrypted content could not be verified.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_encrypted_content\"}}(traceid: fb7ad1dbc7699c18f8a02f258f1af5ab)","param":null,"type":"invalid_request_error"}}`, - )), - }, - { - StatusCode: http.StatusOK, - Header: http.Header{ - "Content-Type": []string{"application/json"}, - "x-request-id": []string{"req_http_retry_wrapped_ok"}, - }, - Body: io.NopCloser(strings.NewReader( - `{"id":"resp_http_retry_wrapped_ok","usage":{"input_tokens":1,"output_tokens":2,"input_tokens_details":{"cached_tokens":0}}}`, - )), - }, - }, - } - - cfg := &config.Config{} - cfg.Security.URLAllowlist.Enabled = false - cfg.Security.URLAllowlist.AllowInsecureHTTP = true - cfg.Gateway.OpenAIWS.Enabled = true - cfg.Gateway.OpenAIWS.OAuthEnabled = true - cfg.Gateway.OpenAIWS.APIKeyEnabled = true - cfg.Gateway.OpenAIWS.ResponsesWebsocketsV2 = true - - svc := &OpenAIGatewayService{ - cfg: cfg, - httpUpstream: upstream, - openaiWSResolver: NewOpenAIWSProtocolResolver(cfg), - } - - account := &Account{ - ID: 103, - Name: "openai-apikey-wrapped", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{ - "api_key": "sk-test", - "base_url": wsFallbackServer.URL, - }, - Extra: map[string]any{ - "responses_websockets_v2_enabled": true, - }, - } - - body := []byte(`{"model":"gpt-5.1","stream":false,"input":[{"type":"reasoning","encrypted_content":"gAAA","summary":[{"type":"summary_text","text":"keep me too"}]},{"type":"input_text","text":"hello"}]}`) - result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.True(t, failoverErr.IsOpenAICiphertextAccountMismatch(), "原样请求可以交给其他账号") - require.True(t, failoverErr.SuppressAccountHealthPenalty) - require.Equal(t, 1, upstream.callCount, "wrapped invalid_encrypted_content 在本账号上不能删状态重试") - require.Len(t, upstream.bodies, 1) - require.False(t, c.Writer.Written(), "continuation 应交由 handler 输出标准终态,不能伪造成功") - - firstBody := upstream.bodies[0] - require.Equal(t, "gAAA", gjson.GetBytes(firstBody, "input.0.encrypted_content").String(), "首次请求不应做发送前预清理") - require.Equal(t, "keep me too", gjson.GetBytes(firstBody, "input.0.summary.0.text").String()) - require.JSONEq(t, gjson.GetBytes(body, "input").Raw, gjson.GetBytes(firstBody, "input").Raw) - - decision, _ := c.Get("openai_ws_transport_decision") - reason, _ := c.Get("openai_ws_transport_reason") - require.Equal(t, string(OpenAIUpstreamTransportHTTPSSE), decision) - require.Equal(t, "client_protocol_http", reason) -} - func TestOpenAIGatewayService_Forward_PreservePreviousResponseIDWhenWSDisabled(t *testing.T) { gin.SetMode(gin.TestMode) wsFallbackServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1455,7 +1240,7 @@ func TestOpenAIGatewayService_Forward_WSv2PreviousResponseNotFoundNeverRetriesMi require.JSONEq(t, gjson.GetBytes(body, "input").Raw, gjson.GetBytes(requests[0], "input").Raw) } -func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentPreservesStateAndTerminates(t *testing.T) { +func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentRecoversOnce(t *testing.T) { gin.SetMode(gin.TestMode) var wsAttempts atomic.Int32 @@ -1556,24 +1341,24 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentPreservesStateA body := []byte(`{"model":"gpt-5.3-codex","stream":false,"previous_response_id":"resp_prev_encrypted","input":[{"type":"reasoning","encrypted_content":"gAAA"},{"type":"compaction","encrypted_content":"cAAA"},{"type":"input_text","text":"hello"}]}`) result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "resp_ws_invalid_encrypted_content_recover_ok", result.RequestID) require.Nil(t, upstream.lastReq, "invalid_encrypted_content 不应回退 HTTP") - require.Equal(t, int32(1), wsAttempts.Load(), "invalid_encrypted_content 不得触发状态清洗或重试") - require.False(t, c.Writer.Written(), "continuation 应交由 handler 输出标准终态,不能伪造成功") + require.Equal(t, int32(2), wsAttempts.Load(), "invalid_encrypted_content 应触发一次清洗后重试") + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "resp_ws_invalid_encrypted_content_recover_ok", gjson.Get(rec.Body.String(), "id").String()) wsRequestMu.Lock() requests := append([][]byte(nil), wsRequestPayloads...) wsRequestMu.Unlock() - require.Len(t, requests, 1) - require.Equal(t, "resp_prev_encrypted", gjson.GetBytes(requests[0], "previous_response_id").String()) - require.Equal(t, "gAAA", gjson.GetBytes(requests[0], `input.0.encrypted_content`).String()) - require.Equal(t, "cAAA", gjson.GetBytes(requests[0], `input.1.encrypted_content`).String()) - require.JSONEq(t, gjson.GetBytes(body, "input").Raw, gjson.GetBytes(requests[0], "input").Raw) + require.Len(t, requests, 2) + require.True(t, gjson.GetBytes(requests[0], "previous_response_id").Exists(), "首轮请求应保留 previous_response_id") + require.True(t, gjson.GetBytes(requests[0], `input.0.encrypted_content`).Exists(), "首轮请求应保留 encrypted reasoning") + require.True(t, gjson.GetBytes(requests[0], `input.1.encrypted_content`).Exists(), "首轮请求应保留 encrypted compaction") + require.False(t, gjson.GetBytes(requests[1], "previous_response_id").Exists(), "恢复重试应移除 previous_response_id") + require.False(t, gjson.GetBytes(requests[1], `input.0.encrypted_content`).Exists(), "恢复重试应移除 encrypted reasoning item") + require.Equal(t, "input_text", gjson.GetBytes(requests[1], `input.0.type`).String()) } func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentSkipsRecoveryWithoutReasoningItem(t *testing.T) { @@ -1660,14 +1445,12 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentSkipsRecoveryWi body := []byte(`{"model":"gpt-5.3-codex","stream":false,"previous_response_id":"resp_prev_encrypted","input":[{"type":"input_text","text":"hello"}]}`) result, err := svc.Forward(context.Background(), c, account, body) + require.Error(t, err) require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) require.Nil(t, upstream.lastReq, "invalid_encrypted_content 不应回退 HTTP") require.Equal(t, int32(1), wsAttempts.Load(), "缺少 reasoning encrypted item 时应跳过自动恢复重试") - require.False(t, c.Writer.Written(), "continuation 应交由 handler 输出标准终态") + require.Equal(t, http.StatusBadRequest, rec.Code) + require.Contains(t, strings.ToLower(rec.Body.String()), "encrypted content") wsRequestMu.Lock() requests := append([][]byte(nil), wsRequestPayloads...) @@ -1677,7 +1460,7 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentSkipsRecoveryWi require.False(t, gjson.GetBytes(requests[0], `input.0.encrypted_content`).Exists()) } -func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentPreservesSingleObjectAndTerminates(t *testing.T) { +func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentRecoversSingleObjectInputAndKeepsSummary(t *testing.T) { gin.SetMode(gin.TestMode) var wsAttempts atomic.Int32 @@ -1778,27 +1561,24 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentPreservesSingle body := []byte(`{"model":"gpt-5.3-codex","stream":false,"previous_response_id":"resp_prev_encrypted","input":{"type":"reasoning","encrypted_content":"gAAA","summary":[{"type":"summary_text","text":"keep me"}]}}`) result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.False(t, failoverErr.ShouldRetryNextAccount()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "resp_ws_invalid_encrypted_content_object_ok", result.RequestID) require.Nil(t, upstream.lastReq, "invalid_encrypted_content 单对象 input 不应回退 HTTP") - require.Equal(t, int32(1), wsAttempts.Load(), "单对象 reasoning input 也不得删密文或重试") - require.False(t, c.Writer.Written(), "continuation 应交由 handler 输出标准终态,不能伪造成功") + require.Equal(t, int32(2), wsAttempts.Load(), "单对象 reasoning input 也应触发一次清洗后重试") wsRequestMu.Lock() requests := append([][]byte(nil), wsRequestPayloads...) wsRequestMu.Unlock() - require.Len(t, requests, 1) - require.Equal(t, "resp_prev_encrypted", gjson.GetBytes(requests[0], "previous_response_id").String()) - require.Equal(t, "gAAA", gjson.GetBytes(requests[0], `input.encrypted_content`).String()) - require.Equal(t, "keep me", gjson.GetBytes(requests[0], `input.summary.0.text`).String()) - require.JSONEq(t, gjson.GetBytes(body, "input").Raw, gjson.GetBytes(requests[0], "input").Raw) + require.Len(t, requests, 2) + require.True(t, gjson.GetBytes(requests[0], `input.encrypted_content`).Exists(), "首轮单对象应保留 encrypted_content") + require.True(t, gjson.GetBytes(requests[1], `input.summary.0.text`).Exists(), "恢复重试应保留 reasoning summary") + require.False(t, gjson.GetBytes(requests[1], `input.encrypted_content`).Exists(), "恢复重试只应移除 encrypted_content") + require.Equal(t, "reasoning", gjson.GetBytes(requests[1], `input.type`).String()) + require.False(t, gjson.GetBytes(requests[1], `previous_response_id`).Exists(), "恢复重试应移除 previous_response_id") } -func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentStopsForFunctionCallOutput(t *testing.T) { +func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentKeepsPreviousResponseIDForFunctionCallOutput(t *testing.T) { gin.SetMode(gin.TestMode) var wsAttempts atomic.Int32 @@ -1806,7 +1586,7 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentStopsForFunctio var wsRequestMu sync.Mutex upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }} wsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - wsAttempts.Add(1) + attempt := wsAttempts.Add(1) conn, err := upgrader.Upgrade(w, r, nil) if err != nil { t.Errorf("upgrade websocket failed: %v", err) @@ -1825,12 +1605,29 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentStopsForFunctio wsRequestMu.Lock() wsRequestPayloads = append(wsRequestPayloads, reqRaw) wsRequestMu.Unlock() + if attempt == 1 { + _ = conn.WriteJSON(map[string]any{ + "type": "error", + "error": map[string]any{ + "code": "invalid_encrypted_content", + "type": "invalid_request_error", + "message": "The encrypted content could not be verified.", + }, + }) + return + } _ = conn.WriteJSON(map[string]any{ - "type": "error", - "error": map[string]any{ - "code": "invalid_encrypted_content", - "type": "invalid_request_error", - "message": "The encrypted content could not be verified.", + "type": "response.completed", + "response": map[string]any{ + "id": "resp_ws_invalid_encrypted_content_function_call_output_ok", + "model": "gpt-5.3-codex", + "usage": map[string]any{ + "input_tokens": 1, + "output_tokens": 1, + "input_tokens_details": map[string]any{ + "cached_tokens": 0, + }, + }, }, }) })) @@ -1882,20 +1679,21 @@ func TestOpenAIGatewayService_Forward_WSv2InvalidEncryptedContentStopsForFunctio body := []byte(`{"model":"gpt-5.3-codex","stream":false,"previous_response_id":"resp_prev_function_call","input":[{"type":"reasoning","encrypted_content":"gAAA"},{"type":"function_call_output","call_id":"call_123","output":"ok"}]}`) result, err := svc.Forward(context.Background(), c, account, body) - require.Nil(t, result) - var failoverErr *UpstreamFailoverError - require.ErrorAs(t, err, &failoverErr) - require.True(t, failoverErr.IsOpenAIContinuationStateUnavailable()) - require.True(t, failoverErr.SuppressAccountHealthPenalty) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "resp_ws_invalid_encrypted_content_function_call_output_ok", result.RequestID) require.Nil(t, upstream.lastReq, "function_call_output + invalid_encrypted_content 不应回退 HTTP") - require.Equal(t, int32(1), wsAttempts.Load(), "function_call_output 场景不应清洗或重试") + require.Equal(t, int32(2), wsAttempts.Load(), "应只做一次保锚点的清洗后重试") wsRequestMu.Lock() requests := append([][]byte(nil), wsRequestPayloads...) wsRequestMu.Unlock() - require.Len(t, requests, 1) + require.Len(t, requests, 2) require.True(t, gjson.GetBytes(requests[0], "previous_response_id").Exists(), "首轮请求应保留 previous_response_id") - require.Equal(t, "gAAA", gjson.GetBytes(requests[0], `input.0.encrypted_content`).String()) - require.Equal(t, "function_call_output", gjson.GetBytes(requests[0], `input.1.type`).String()) - require.Equal(t, "resp_prev_function_call", gjson.GetBytes(requests[0], "previous_response_id").String()) + require.True(t, gjson.GetBytes(requests[1], "previous_response_id").Exists(), "function_call_output 恢复重试不应移除 previous_response_id") + require.False(t, gjson.GetBytes(requests[1], `input.0.encrypted_content`).Exists(), "恢复重试应移除 reasoning encrypted_content") + require.Equal(t, "function_call_output", gjson.GetBytes(requests[1], `input.0.type`).String(), "清洗后应保留 function_call_output 作为首个输入项") + require.Equal(t, "call_123", gjson.GetBytes(requests[1], `input.0.call_id`).String()) + require.Equal(t, "ok", gjson.GetBytes(requests[1], `input.0.output`).String()) + require.Equal(t, "resp_prev_function_call", gjson.GetBytes(requests[1], "previous_response_id").String()) } diff --git a/backend/internal/service/openai_ws_v2/passthrough_relay.go b/backend/internal/service/openai_ws_v2/passthrough_relay.go index 3c6bd076dcef..0fdbb9085232 100644 --- a/backend/internal/service/openai_ws_v2/passthrough_relay.go +++ b/backend/internal/service/openai_ws_v2/passthrough_relay.go @@ -26,6 +26,7 @@ type FrameConn interface { type Usage struct { InputTokens int + ImageInputTokens int OutputTokens int CacheCreationInputTokens int CacheReadInputTokens int @@ -1117,6 +1118,21 @@ func parseUsageAndAccumulate( if imageTokens == 0 { imageTokens = usageResult.Get("completion_tokens_details.image_tokens").Int() } + imageInputTokens := usageResult.Get("input_tokens_details.image_tokens").Int() + if imageInputTokens <= 0 { + imageInputTokens = usageResult.Get("prompt_tokens_details.image_tokens").Int() + } + // 与 HTTP Responses 同口径:usage 缺图片计数时由托管工具 tool_usage.image_gen 回填。 + imageGen := gjson.GetBytes(message, "response.tool_usage.image_gen") + if !imageGen.Exists() { + imageGen = gjson.GetBytes(message, "tool_usage.image_gen") + } + if imageTokens <= 0 { + imageTokens = max(imageGen.Get("output_tokens_details.image_tokens").Int(), 0) + } + if imageInputTokens <= 0 { + imageInputTokens = max(imageGen.Get("input_tokens_details.image_tokens").Int(), 0) + } requireTotals := isTerminalEvent(strings.TrimSpace(eventType)) inputTokens, inputOK := parseUsageIntField(inputResult, requireTotals) @@ -1141,6 +1157,7 @@ func parseUsageAndAccumulate( } parsedUsage := Usage{ InputTokens: inputTokens, + ImageInputTokens: int(imageInputTokens), OutputTokens: outputTokens, CacheCreationInputTokens: openAICacheCreationTokensFromUsage(usageResult), CacheReadInputTokens: cachedTokens, @@ -1161,7 +1178,7 @@ func parseUsageAndAccumulate( func relayUsageHasTokens(usage Usage) bool { return usage.InputTokens > 0 || usage.OutputTokens > 0 || usage.CacheCreationInputTokens > 0 || usage.CacheReadInputTokens > 0 || - usage.ImageOutputTokens > 0 + usage.ImageInputTokens > 0 || usage.ImageOutputTokens > 0 } func mergeRelayUsageNonZero(dst *Usage, src Usage) { @@ -1180,6 +1197,9 @@ func mergeRelayUsageNonZero(dst *Usage, src Usage) { if src.CacheReadInputTokens > 0 { dst.CacheReadInputTokens = src.CacheReadInputTokens } + if src.ImageInputTokens > 0 { + dst.ImageInputTokens = src.ImageInputTokens + } if src.ImageOutputTokens > 0 { dst.ImageOutputTokens = src.ImageOutputTokens } @@ -1191,6 +1211,7 @@ func finalizeRelayTurnUsage(state *relayState) Usage { } turnUsage := state.turnUsage state.usage.InputTokens += turnUsage.InputTokens + state.usage.ImageInputTokens += turnUsage.ImageInputTokens state.usage.OutputTokens += turnUsage.OutputTokens state.usage.CacheCreationInputTokens += turnUsage.CacheCreationInputTokens state.usage.CacheReadInputTokens += turnUsage.CacheReadInputTokens diff --git a/backend/internal/service/openai_ws_v2/passthrough_relay_image_usage_test.go b/backend/internal/service/openai_ws_v2/passthrough_relay_image_usage_test.go new file mode 100644 index 000000000000..74beacd6c96c --- /dev/null +++ b/backend/internal/service/openai_ws_v2/passthrough_relay_image_usage_test.go @@ -0,0 +1,47 @@ +package openai_ws_v2 + +import ( + "context" + "testing" + "time" + + coderws "github.com/coder/websocket" + "github.com/stretchr/testify/require" +) + +// WS passthrough must account image tokens the way the HTTP Responses path +// does: image input from input/prompt_tokens_details, and the hosted +// tool_usage.image_gen breakdown filling counters the usage object omits. +func TestRelay_ImageUsageMatchesHTTPResponsesAccounting(t *testing.T) { + t.Parallel() + + clientConn := newPassthroughTestFrameConn(nil, false) + upstreamConn := newPassthroughTestFrameConn([]passthroughTestFrame{ + {msgType: coderws.MessageText, payload: []byte(`{"type":"response.completed","response":{"id":"resp_tool_image","usage":{"input_tokens":100,"output_tokens":60},"tool_usage":{"image_gen":{"input_tokens_details":{"image_tokens":80},"output_tokens_details":{"image_tokens":50}}}}}`)}, + {msgType: coderws.MessageText, payload: []byte(`{"type":"response.completed","response":{"id":"resp_explicit_image","usage":{"input_tokens":30,"output_tokens":15,"input_tokens_details":{"image_tokens":20},"output_tokens_details":{"image_tokens":10}},"tool_usage":{"image_gen":{"input_tokens_details":{"image_tokens":800},"output_tokens_details":{"image_tokens":500}}}}}`)}, + {msgType: coderws.MessageText, payload: []byte(`{"type":"response.completed","response":{"id":"resp_prompt_details","usage":{"prompt_tokens":12,"completion_tokens":2,"prompt_tokens_details":{"image_tokens":7}}}}`)}, + {msgType: coderws.MessageText, payload: []byte(`{"type":"response.completed","response":{"id":"resp_text","usage":{"input_tokens":3,"output_tokens":4}}}`)}, + }, true) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + var turns []RelayTurnResult + result, relayExit := Relay(ctx, clientConn, upstreamConn, + []byte(`{"type":"response.create","model":"gpt-5.4","tools":[{"type":"image_generation"}],"input":[]}`), + RelayOptions{OnTurnComplete: func(turn RelayTurnResult) { turns = append(turns, turn) }}) + require.Nil(t, relayExit) + require.Len(t, turns, 4) + + require.Equal(t, 80, turns[0].Usage.ImageInputTokens, "tool_usage.image_gen fills a missing image input counter") + require.Equal(t, 50, turns[0].Usage.ImageOutputTokens, "tool_usage.image_gen fills a missing image output counter") + require.Equal(t, 20, turns[1].Usage.ImageInputTokens, "response usage takes precedence over the hosted tool breakdown") + require.Equal(t, 10, turns[1].Usage.ImageOutputTokens, "response usage takes precedence over the hosted tool breakdown") + require.Equal(t, 7, turns[2].Usage.ImageInputTokens) + require.Zero(t, turns[3].Usage.ImageInputTokens, "a text turn must not reuse earlier image usage") + require.Zero(t, turns[3].Usage.ImageOutputTokens, "a text turn must not reuse earlier image usage") + + require.Equal(t, 107, result.Usage.ImageInputTokens) + require.Equal(t, 60, result.Usage.ImageOutputTokens) + require.Equal(t, 145, result.Usage.InputTokens) + require.Equal(t, 81, result.Usage.OutputTokens) +} diff --git a/backend/internal/service/openai_ws_v2_passthrough_adapter.go b/backend/internal/service/openai_ws_v2_passthrough_adapter.go index 97f9c404f71e..2b102bdec262 100644 --- a/backend/internal/service/openai_ws_v2_passthrough_adapter.go +++ b/backend/internal/service/openai_ws_v2_passthrough_adapter.go @@ -1359,6 +1359,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( RequestID: turn.RequestID, Usage: OpenAIUsage{ InputTokens: turn.Usage.InputTokens, + ImageInputTokens: turn.Usage.ImageInputTokens, OutputTokens: turn.Usage.OutputTokens, CacheCreationInputTokens: turn.Usage.CacheCreationInputTokens, CacheReadInputTokens: turn.Usage.CacheReadInputTokens, @@ -1560,6 +1561,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( RequestID: relayResult.RequestID, Usage: OpenAIUsage{ InputTokens: relayResult.Usage.InputTokens, + ImageInputTokens: relayResult.Usage.ImageInputTokens, OutputTokens: relayResult.Usage.OutputTokens, CacheCreationInputTokens: relayResult.Usage.CacheCreationInputTokens, CacheReadInputTokens: relayResult.Usage.CacheReadInputTokens, diff --git a/backend/internal/service/openai_ws_v2_passthrough_image_usage_test.go b/backend/internal/service/openai_ws_v2_passthrough_image_usage_test.go new file mode 100644 index 000000000000..06d66df62d7d --- /dev/null +++ b/backend/internal/service/openai_ws_v2_passthrough_image_usage_test.go @@ -0,0 +1,103 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + coderws "github.com/coder/websocket" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// The billed turn result of a WS passthrough image generation must carry the +// image input tokens and the tool_usage.image_gen backfill, as HTTP does. +func TestOpenAIWSPassthroughTurnResultCarriesImageUsage(t *testing.T) { + gin.SetMode(gin.TestMode) + + cfg := &config.Config{} + cfg.Security.URLAllowlist.Enabled = false + cfg.Security.URLAllowlist.AllowInsecureHTTP = true + cfg.Gateway.OpenAIWS.Enabled = true + cfg.Gateway.OpenAIWS.APIKeyEnabled = true + cfg.Gateway.OpenAIWS.ResponsesWebsocketsV2 = true + cfg.Gateway.OpenAIWS.ModeRouterV2Enabled = true + cfg.Gateway.OpenAIWS.IngressModeDefault = OpenAIWSIngressModeCtxPool + cfg.Gateway.OpenAIWS.DialTimeoutSeconds = 3 + cfg.Gateway.OpenAIWS.ReadTimeoutSeconds = 3 + cfg.Gateway.OpenAIWS.WriteTimeoutSeconds = 3 + + upstreamConn := &openAIWSCaptureConn{ + events: [][]byte{ + []byte(`{"type":"response.completed","response":{"id":"resp_ws_image","model":"gpt-5.4","output":[{"id":"ig_ws_1","type":"image_generation_call","status":"completed","result":"final-image"}],"usage":{"input_tokens":120,"output_tokens":70,"input_tokens_details":{"image_tokens":40}},"tool_usage":{"image_gen":{"input_tokens_details":{"image_tokens":900},"output_tokens_details":{"image_tokens":55}}}}}`), + }, + } + svc := &OpenAIGatewayService{ + cfg: cfg, + httpUpstream: &httpUpstreamRecorder{}, + cache: &stubGatewayCache{}, + openaiWSResolver: NewOpenAIWSProtocolResolver(cfg), + toolCorrector: NewCodexToolCorrector(), + openaiWSPassthroughDialer: &openAIWSCaptureDialer{conn: upstreamConn}, + } + account := &Account{ + ID: 453, Name: "openai-ws-image-usage", Platform: PlatformOpenAI, Type: AccountTypeAPIKey, + Status: StatusActive, Schedulable: true, Concurrency: 1, + Credentials: map[string]any{"api_key": "sk-test"}, + Extra: map[string]any{"openai_apikey_responses_websockets_v2_mode": OpenAIWSIngressModePassthrough}, + } + + resultCh := make(chan *OpenAIForwardResult, 1) + hooks := &OpenAIWSIngressHooks{ + AfterTurn: func(_ int, result *OpenAIForwardResult, turnErr error) { + if turnErr == nil && result != nil { + resultCh <- result + } + }, + } + wsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := coderws.Accept(w, r, nil) + if err != nil { + return + } + defer func() { _ = conn.CloseNow() }() + ginCtx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ginCtx.Request = r.Clone(r.Context()) + readCtx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + _, firstMessage, readErr := conn.Read(readCtx) + cancel() + if readErr != nil { + return + } + _ = svc.ProxyResponsesWebSocketFromClient(r.Context(), ginCtx, conn, account, "sk-test", firstMessage, hooks) + })) + defer wsServer.Close() + + dialCtx, cancelDial := context.WithTimeout(context.Background(), 3*time.Second) + clientConn, _, err := coderws.Dial(dialCtx, "ws"+strings.TrimPrefix(wsServer.URL, "http"), nil) + cancelDial() + require.NoError(t, err) + defer func() { _ = clientConn.CloseNow() }() + + writeCtx, cancelWrite := context.WithTimeout(context.Background(), 3*time.Second) + require.NoError(t, clientConn.Write(writeCtx, coderws.MessageText, []byte(`{"type":"response.create","model":"gpt-5.4","tools":[{"type":"image_generation"}],"input":"draw"}`))) + cancelWrite() + readCtx, cancelRead := context.WithTimeout(context.Background(), 3*time.Second) + _, _, readErr := clientConn.Read(readCtx) + cancelRead() + require.NoError(t, readErr) + _ = clientConn.Close(coderws.StatusNormalClosure, "done") + + select { + case result := <-resultCh: + require.Equal(t, 120, result.Usage.InputTokens) + require.Equal(t, 40, result.Usage.ImageInputTokens, "usage image input wins over the hosted tool breakdown") + require.Equal(t, 55, result.Usage.ImageOutputTokens, "tool_usage.image_gen fills the missing image output counter") + case <-time.After(5 * time.Second): + t.Fatal("passthrough turn result was not reported") + } +} diff --git a/backend/internal/service/opencode_go.go b/backend/internal/service/opencode_go.go index 1364179b0166..248ced9589c1 100644 --- a/backend/internal/service/opencode_go.go +++ b/backend/internal/service/opencode_go.go @@ -3,9 +3,11 @@ package service import ( "fmt" "net/http" + "slices" "strings" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" + "github.com/gin-gonic/gin" ) // OpenCode Go 是 OpenCode Zen 的订阅网关:同一 API Key 下按模型分流到 @@ -18,9 +20,9 @@ const ( // the UI does not pick a model. glm-5.3 is a Chat Completions catalog ID. DefaultOpenCodeGoTestModel = "glm-5.3" - openCodeGoProtocolRulesKey = "protocol_rules" - maxOpenCodeGoProtocolRules = 64 - maxOpenCodeGoProtocolPatternLength = 128 + protocolRulesCredentialKey = "protocol_rules" + maxProtocolRules = 64 + maxProtocolRulePatternLength = 128 ) // DefaultOpenCodeGoModelIDs 是官方文档当前公开的模型 ID 目录, @@ -67,16 +69,64 @@ func normalizeOpenCodeGoModelID(model string) string { return model } -// OpenCodeGoProtocolRule is one model-pattern → native protocol mapping. +// IsOpenCodeUnsupportedModel 判断请求的模型是否属于 OpenCode 专有协议端点模型。 +// 官方 Zen 上 gemini-* 仅走 Google SDK 专属端点 /zen/v1/models/, +// jev-* 仅走 SystemOne 专属端点 /zen/v1/systemone, +// 在通用 Chat Completions / Anthropic Messages / Responses 网关端点上不可用。 +func IsOpenCodeUnsupportedModel(model string) bool { + normalized := normalizeOpenCodeGoModelID(model) + return strings.HasPrefix(normalized, "gemini-") || strings.HasPrefix(normalized, "jev-") +} + +func writeOpenCodeUnsupportedModelError(c *gin.Context, isAnthropic bool, model string) error { + msg := fmt.Sprintf("Model '%s' is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)", model) + if c != nil { + if isAnthropic { + c.JSON(http.StatusBadRequest, gin.H{ + "type": "error", + "error": gin.H{ + "type": "invalid_request_error", + "message": msg, + }, + }) + } else { + c.JSON(http.StatusBadRequest, gin.H{ + "error": gin.H{ + "type": "invalid_request_error", + "code": "model_not_supported", + "message": msg, + }, + }) + } + } + return fmt.Errorf("opencode unsupported model: %s", model) +} + +// ProtocolRule is one model-pattern → native protocol mapping. // Pattern is an exact ID or a suffix glob (foo* / *). First match wins. -type OpenCodeGoProtocolRule struct { +type ProtocolRule struct { Pattern string `json:"pattern"` Protocol string `json:"protocol"` + // Protocols 是命中模型支持的全部原生协议,首项即 Protocol(首选);为空表示只支持 + // Protocol。入站协议在其中时同协议直通,否则走首选协议(见 resolveModelRoutedProtocolFor)。 + Protocols []string `json:"protocols,omitempty"` +} + +// normalizeProtocolSet 返回以 preferred 开头、去重且只含原生协议的协议集合。 +func normalizeProtocolSet(preferred string, protocols []string) []string { + set := []string{preferred} + for _, protocol := range protocols { + if !isNativeUpstreamProtocol(protocol) || slices.Contains(set, protocol) { + continue + } + set = append(set, protocol) + } + return set } // DefaultOpenCodeGoProtocolRules is the built-in adaptive routing table for Go. -func DefaultOpenCodeGoProtocolRules() []OpenCodeGoProtocolRule { - return []OpenCodeGoProtocolRule{ +func DefaultOpenCodeGoProtocolRules() []ProtocolRule { + return []ProtocolRule{ {Pattern: "grok-*", Protocol: APIProtocolResponses}, {Pattern: "gpt-*", Protocol: APIProtocolResponses}, {Pattern: "muse-spark-*", Protocol: APIProtocolResponses}, @@ -86,25 +136,20 @@ func DefaultOpenCodeGoProtocolRules() []OpenCodeGoProtocolRule { } // DefaultOpenCodeZenProtocolRules 对齐 https://opencode.ai/docs/zen/ 端点表: -// GPT/Grok/Muse Spark → Responses,Claude/Qwen → Anthropic,其余 Chat Completions。 -func DefaultOpenCodeZenProtocolRules() []OpenCodeGoProtocolRule { - return []OpenCodeGoProtocolRule{ +// GPT/Grok/Muse Spark → Responses,Claude/Qwen(除 qwen3.8-max) → Anthropic, +// qwen3.8-max 及其余模型 → Chat Completions。 +func DefaultOpenCodeZenProtocolRules() []ProtocolRule { + return []ProtocolRule{ {Pattern: "grok-*", Protocol: APIProtocolResponses}, {Pattern: "gpt-*", Protocol: APIProtocolResponses}, {Pattern: "muse-spark-*", Protocol: APIProtocolResponses}, {Pattern: "claude-*", Protocol: APIProtocolAnthropic}, + {Pattern: "qwen3.8-max", Protocol: APIProtocolChatCompletions}, {Pattern: "qwen*", Protocol: APIProtocolAnthropic}, } } -func defaultOpenCodeProtocolRules(mode string) []OpenCodeGoProtocolRule { - if mode == AccountModeZen { - return DefaultOpenCodeZenProtocolRules() - } - return DefaultOpenCodeGoProtocolRules() -} - -func isNativeOpenCodeGoProtocol(protocol string) bool { +func isNativeUpstreamProtocol(protocol string) bool { switch protocol { case APIProtocolChatCompletions, APIProtocolAnthropic, APIProtocolResponses: return true @@ -113,7 +158,7 @@ func isNativeOpenCodeGoProtocol(protocol string) bool { } } -func openCodeGoPatternMatches(pattern, model string) bool { +func protocolRulePatternMatches(pattern, model string) bool { pattern = strings.ToLower(strings.TrimSpace(pattern)) model = normalizeOpenCodeGoModelID(model) if pattern == "" || model == "" { @@ -128,49 +173,51 @@ func openCodeGoPatternMatches(pattern, model string) bool { return model == pattern } -func matchOpenCodeGoProtocolRules(model string, rules []OpenCodeGoProtocolRule) string { +// matchProtocolRules 返回首条命中规则的首选协议;未命中时为 Chat Completions。 +func matchProtocolRules(model string, rules []ProtocolRule) string { + if set, ok := matchProtocolRuleSet(model, rules); ok { + return set[0] + } + return APIProtocolChatCompletions +} + +// matchProtocolRuleSet 返回首条命中规则支持的协议集合(首项为首选);未命中时 ok=false。 +func matchProtocolRuleSet(model string, rules []ProtocolRule) ([]string, bool) { for _, rule := range rules { - if !isNativeOpenCodeGoProtocol(rule.Protocol) { + if !isNativeUpstreamProtocol(rule.Protocol) { continue } - if openCodeGoPatternMatches(rule.Pattern, model) { - return rule.Protocol + if protocolRulePatternMatches(rule.Pattern, model) { + return normalizeProtocolSet(rule.Protocol, rule.Protocols), true } } - return APIProtocolChatCompletions + return nil, false } -// OpenCodeGoModelProtocol 返回内置默认表下模型对应的原生上游协议。 -// 未命中时为 Chat Completions。账号自定义 protocol_rules 见 -// ResolveOpenCodeGoUpstreamProtocol。 -func OpenCodeGoModelProtocol(model string) string { - return matchOpenCodeGoProtocolRules(model, DefaultOpenCodeGoProtocolRules()) -} - -func (a *Account) openCodeGoProtocolRules() ([]OpenCodeGoProtocolRule, bool) { +func (a *Account) configuredProtocolRules() ([]ProtocolRule, bool) { if a == nil || a.Credentials == nil { return nil, false } - raw, ok := a.Credentials[openCodeGoProtocolRulesKey] + raw, ok := a.Credentials[protocolRulesCredentialKey] if !ok || raw == nil { return nil, false } - rules, err := parseOpenCodeGoProtocolRules(raw) + rules, err := parseProtocolRules(raw) if err != nil { return nil, false } return rules, true } -func parseOpenCodeGoProtocolRules(raw any) ([]OpenCodeGoProtocolRule, error) { - items, err := openCodeGoProtocolRuleItems(raw) +func parseProtocolRules(raw any) ([]ProtocolRule, error) { + items, err := protocolRuleItems(raw) if err != nil { return nil, err } - if len(items) > maxOpenCodeGoProtocolRules { - return nil, fmt.Errorf("protocol_rules supports at most %d entries", maxOpenCodeGoProtocolRules) + if len(items) > maxProtocolRules { + return nil, fmt.Errorf("protocol_rules supports at most %d entries", maxProtocolRules) } - rules := make([]OpenCodeGoProtocolRule, 0, len(items)) + rules := make([]ProtocolRule, 0, len(items)) for i, item := range items { entry, ok := item.(map[string]any) if !ok { @@ -178,20 +225,59 @@ func parseOpenCodeGoProtocolRules(raw any) ([]OpenCodeGoProtocolRule, error) { } pattern, _ := entry["pattern"].(string) protocol, _ := entry["protocol"].(string) - pattern, err := normalizeOpenCodeGoProtocolPattern(pattern) + pattern, err := normalizeProtocolRulePattern(pattern) + if err != nil { + return nil, fmt.Errorf("protocol_rules[%d]: %w", i, err) + } + protocols, err := protocolRuleProtocols(entry["protocols"]) if err != nil { return nil, fmt.Errorf("protocol_rules[%d]: %w", i, err) } protocol = strings.TrimSpace(protocol) - if !isNativeOpenCodeGoProtocol(protocol) { + if protocol == "" && len(protocols) > 0 { + protocol = protocols[0] + } + if !isNativeUpstreamProtocol(protocol) { return nil, fmt.Errorf("protocol_rules[%d]: protocol must be chat_completions, anthropic, or responses", i) } - rules = append(rules, OpenCodeGoProtocolRule{Pattern: pattern, Protocol: protocol}) + rule := ProtocolRule{Pattern: pattern, Protocol: protocol} + if set := normalizeProtocolSet(protocol, protocols); len(set) > 1 { + rule.Protocols = set + } + rules = append(rules, rule) } return rules, nil } -func openCodeGoProtocolRuleItems(raw any) ([]any, error) { +// protocolRuleProtocols 解析规则的 protocols 列表(可缺省);其中每一项都须为原生协议。 +func protocolRuleProtocols(raw any) ([]string, error) { + if raw == nil { + return nil, nil + } + var items []any + switch values := raw.(type) { + case []any: + items = values + case []string: + for _, value := range values { + items = append(items, value) + } + default: + return nil, fmt.Errorf("protocols must be an array") + } + out := make([]string, 0, len(items)) + for _, item := range items { + protocol, _ := item.(string) + protocol = strings.TrimSpace(protocol) + if !isNativeUpstreamProtocol(protocol) { + return nil, fmt.Errorf("protocols must only contain chat_completions, anthropic, or responses") + } + out = append(out, protocol) + } + return out, nil +} + +func protocolRuleItems(raw any) ([]any, error) { switch items := raw.(type) { case []any: return items, nil @@ -206,12 +292,12 @@ func openCodeGoProtocolRuleItems(raw any) ([]any, error) { } } -func normalizeOpenCodeGoProtocolPattern(pattern string) (string, error) { +func normalizeProtocolRulePattern(pattern string) (string, error) { pattern = strings.ToLower(strings.TrimSpace(pattern)) if pattern == "" { return "", fmt.Errorf("pattern is required") } - if len(pattern) > maxOpenCodeGoProtocolPatternLength { + if len(pattern) > maxProtocolRulePatternLength { return "", fmt.Errorf("pattern is too long") } if strings.ContainsAny(pattern, " \t") { @@ -224,28 +310,36 @@ func normalizeOpenCodeGoProtocolPattern(pattern string) (string, error) { return pattern, nil } -// NormalizeOpenCodeGoProtocolRulesCredentials 校验并原地规范化 credentials.protocol_rules。 +// NormalizeProtocolRulesCredentials 校验并原地规范化 credentials.protocol_rules。 // 未携带该字段时为 no-op,使旧账号继续使用内置默认表。 -func NormalizeOpenCodeGoProtocolRulesCredentials(credentials map[string]any) error { +func NormalizeProtocolRulesCredentials(credentials map[string]any) error { if credentials == nil { return nil } - raw, ok := credentials[openCodeGoProtocolRulesKey] + raw, ok := credentials[protocolRulesCredentialKey] if !ok || raw == nil { return nil } - rules, err := parseOpenCodeGoProtocolRules(raw) + rules, err := parseProtocolRules(raw) if err != nil { return infraerrors.New(http.StatusBadRequest, "INVALID_OPENCODE_GO_PROTOCOL_RULES", err.Error()) } encoded := make([]any, 0, len(rules)) for _, rule := range rules { - encoded = append(encoded, map[string]any{ + item := map[string]any{ "pattern": rule.Pattern, "protocol": rule.Protocol, - }) + } + if len(rule.Protocols) > 1 { + protocols := make([]any, 0, len(rule.Protocols)) + for _, protocol := range rule.Protocols { + protocols = append(protocols, protocol) + } + item["protocols"] = protocols + } + encoded = append(encoded, item) } - credentials[openCodeGoProtocolRulesKey] = encoded + credentials[protocolRulesCredentialKey] = encoded return nil } @@ -272,56 +366,10 @@ func (a *Account) IsOpenCodeGoPlan() bool { return a.GetOpenCodeAccountMode() == AccountModeGo } -func (a *Account) openCodeDefaultChatBaseURL() string { - if a.IsOpenCodeZen() { - return DefaultOpenCodeZenBaseURL - } - return DefaultOpenCodeGoBaseURL -} - -func (a *Account) openCodeDefaultAnthropicBaseURL() string { - if a.IsOpenCodeZen() { - return DefaultOpenCodeZenAnthropicBaseURL - } - return DefaultOpenCodeGoAnthropicBaseURL -} - func (a *Account) IsMultiProtocolAPIKey() bool { return a != nil && IsMultiProtocolAPIKeyProvider(a.Platform) } -// openCodeGoNativeProtocol 返回 OpenCode Go 实际上游协议。 -// 规则未命中、空值或未知协议一律兜底 Chat Completions,避免落入 Responses 转换链。 -func openCodeGoNativeProtocol(account *Account, model string) string { - if account == nil { - return APIProtocolChatCompletions - } - switch proto := account.ResolveOpenCodeGoUpstreamProtocol(model); proto { - case APIProtocolAnthropic, APIProtocolResponses: - return proto - default: - return APIProtocolChatCompletions - } -} - -// ResolveOpenCodeGoUpstreamProtocol 按账号协议配置与模型规则决定上游协议。 -// 显式 pinned 协议优先;adaptive(默认)先走 credentials.protocol_rules, -// 未配置时回落内置默认表;已配置但未命中则走 Chat Completions。 -func (a *Account) ResolveOpenCodeGoUpstreamProtocol(model string) string { - if a == nil || !a.IsOpenCodeGo() { - return "" - } - switch a.GetAPIProtocol() { - case APIProtocolChatCompletions, APIProtocolAnthropic, APIProtocolResponses: - return a.GetAPIProtocol() - default: - if rules, present := a.openCodeGoProtocolRules(); present { - return matchOpenCodeGoProtocolRules(model, rules) - } - return matchOpenCodeGoProtocolRules(model, defaultOpenCodeProtocolRules(a.GetOpenCodeAccountMode())) - } -} - // openCodeGoQuotaURL 根据 base_url 解析 OpenCode Go 额度端点。 // /zen/go/v1(Chat 协议默认,DefaultOpenCodeGoBaseURL)与 /zen/go // (Anthropic 协议默认,DefaultOpenCodeGoAnthropicBaseURL)两种 base 统一 diff --git a/backend/internal/service/opencode_go_test.go b/backend/internal/service/opencode_go_test.go index 796487c95253..c3bbc2558a22 100644 --- a/backend/internal/service/opencode_go_test.go +++ b/backend/internal/service/opencode_go_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestOpenCodeGoModelProtocol(t *testing.T) { +func TestOpenCodeGoDefaultProtocolRules(t *testing.T) { t.Parallel() cases := []struct { model string @@ -28,114 +28,119 @@ func TestOpenCodeGoModelProtocol(t *testing.T) { {"unknown-model", APIProtocolChatCompletions}, } for _, tc := range cases { - require.Equal(t, tc.want, OpenCodeGoModelProtocol(tc.model), "model=%s", tc.model) + require.Equal(t, tc.want, matchProtocolRules(tc.model, DefaultOpenCodeGoProtocolRules()), "model=%s", tc.model) } } -func TestResolveOpenCodeGoUpstreamProtocol(t *testing.T) { +func TestOpenCodeGoModelRoutedProtocol(t *testing.T) { t.Parallel() adaptive := &Account{Platform: PlatformOpenCodeGo, Credentials: map[string]any{"api_protocol": APIProtocolAdaptive}} - require.Equal(t, APIProtocolAnthropic, adaptive.ResolveOpenCodeGoUpstreamProtocol("minimax-m3")) - require.Equal(t, APIProtocolResponses, adaptive.ResolveOpenCodeGoUpstreamProtocol("grok-4.6")) - require.Equal(t, APIProtocolChatCompletions, adaptive.ResolveOpenCodeGoUpstreamProtocol("glm-5.3")) + require.Equal(t, APIProtocolAnthropic, adaptive.resolveModelRoutedProtocol("minimax-m3")) + require.Equal(t, APIProtocolResponses, adaptive.resolveModelRoutedProtocol("grok-4.6")) + require.Equal(t, APIProtocolChatCompletions, adaptive.resolveModelRoutedProtocol("glm-5.3")) pinned := &Account{Platform: PlatformOpenCodeGo, Credentials: map[string]any{"api_protocol": APIProtocolChatCompletions}} - require.Equal(t, APIProtocolChatCompletions, pinned.ResolveOpenCodeGoUpstreamProtocol("minimax-m3")) + require.Equal(t, APIProtocolChatCompletions, pinned.resolveModelRoutedProtocol("minimax-m3")) empty := &Account{Platform: PlatformOpenCodeGo} - require.Equal(t, APIProtocolResponses, empty.ResolveOpenCodeGoUpstreamProtocol("gpt-5.6-luna")) + require.Equal(t, APIProtocolResponses, empty.resolveModelRoutedProtocol("gpt-5.6-luna")) require.Equal(t, AccountModeGo, empty.GetOpenCodeAccountMode()) require.True(t, empty.IsOpenCodeGoPlan()) zen := &Account{Platform: PlatformOpenCodeGo, Credentials: map[string]any{"account_mode": AccountModeZen, "api_protocol": APIProtocolAdaptive}} - require.Equal(t, APIProtocolChatCompletions, zen.ResolveOpenCodeGoUpstreamProtocol("minimax-m3")) - require.Equal(t, APIProtocolAnthropic, zen.ResolveOpenCodeGoUpstreamProtocol("claude-opus-4-6")) + require.Equal(t, APIProtocolChatCompletions, zen.resolveModelRoutedProtocol("minimax-m3")) + require.Equal(t, APIProtocolAnthropic, zen.resolveModelRoutedProtocol("claude-opus-4-6")) + require.Equal(t, APIProtocolChatCompletions, zen.resolveModelRoutedProtocol("qwen3.8-max")) + require.Equal(t, APIProtocolAnthropic, zen.resolveModelRoutedProtocol("qwen3.8-flash")) + require.Equal(t, APIProtocolAnthropic, zen.resolveModelRoutedProtocol("qwen3.7-max")) require.Equal(t, DefaultOpenCodeZenBaseURL, zen.GetOpenAIBaseURL()) - require.Equal(t, "", (&Account{Platform: PlatformKimi}).ResolveOpenCodeGoUpstreamProtocol("glm-5.3")) + goAccount := &Account{Platform: PlatformOpenCodeGo, Credentials: map[string]any{"account_mode": AccountModeGo, "api_protocol": APIProtocolAdaptive}} + require.Equal(t, APIProtocolAnthropic, goAccount.resolveModelRoutedProtocol("qwen3.8-max")) + + require.False(t, (&Account{Platform: PlatformKimi}).routesByModel()) } -func TestResolveOpenCodeGoUpstreamProtocolUsesAccountRules(t *testing.T) { +func TestOpenCodeGoModelRoutedProtocolUsesAccountRules(t *testing.T) { t.Parallel() account := &Account{ Platform: PlatformOpenCodeGo, Credentials: map[string]any{ "api_protocol": APIProtocolAdaptive, - openCodeGoProtocolRulesKey: []any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": "grok-*", "protocol": APIProtocolChatCompletions}, map[string]any{"pattern": "deepseek-v4-flash", "protocol": APIProtocolResponses}, map[string]any{"pattern": "qwen*", "protocol": APIProtocolAnthropic}, }, }, } - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("grok-4.6")) - require.Equal(t, APIProtocolResponses, account.ResolveOpenCodeGoUpstreamProtocol("deepseek-v4-flash")) - require.Equal(t, APIProtocolAnthropic, account.ResolveOpenCodeGoUpstreamProtocol("qwen3.8-max")) - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("glm-5.3")) - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("minimax-m3")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("grok-4.6")) + require.Equal(t, APIProtocolResponses, account.resolveModelRoutedProtocol("deepseek-v4-flash")) + require.Equal(t, APIProtocolAnthropic, account.resolveModelRoutedProtocol("qwen3.8-max")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("glm-5.3")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("minimax-m3")) } -func TestResolveOpenCodeGoUpstreamProtocolEmptyRulesAreAllChatCompletions(t *testing.T) { +func TestOpenCodeGoModelRoutedProtocolEmptyRulesAreAllChatCompletions(t *testing.T) { t.Parallel() account := &Account{ Platform: PlatformOpenCodeGo, Credentials: map[string]any{ - openCodeGoProtocolRulesKey: []any{}, + protocolRulesCredentialKey: []any{}, }, } - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("grok-4.6")) - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("minimax-m3")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("grok-4.6")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("minimax-m3")) } -func TestResolveOpenCodeGoUpstreamProtocolFirstMatchWins(t *testing.T) { +func TestOpenCodeGoModelRoutedProtocolFirstMatchWins(t *testing.T) { t.Parallel() account := &Account{ Platform: PlatformOpenCodeGo, Credentials: map[string]any{ - openCodeGoProtocolRulesKey: []any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": "gpt-5.6-luna", "protocol": APIProtocolChatCompletions}, map[string]any{"pattern": "gpt-*", "protocol": APIProtocolResponses}, }, }, } - require.Equal(t, APIProtocolChatCompletions, account.ResolveOpenCodeGoUpstreamProtocol("gpt-5.6-luna")) - require.Equal(t, APIProtocolResponses, account.ResolveOpenCodeGoUpstreamProtocol("gpt-5.4")) + require.Equal(t, APIProtocolChatCompletions, account.resolveModelRoutedProtocol("gpt-5.6-luna")) + require.Equal(t, APIProtocolResponses, account.resolveModelRoutedProtocol("gpt-5.4")) } -func TestOpenCodeGoNativeProtocolUnmatchedFallsBackToChatCompletions(t *testing.T) { +func TestOpenCodeGoUnmatchedFallsBackToChatCompletions(t *testing.T) { t.Parallel() withRules := &Account{ Platform: PlatformOpenCodeGo, Credentials: map[string]any{ "api_protocol": APIProtocolAdaptive, - openCodeGoProtocolRulesKey: []any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": "grok-*", "protocol": APIProtocolResponses}, map[string]any{"pattern": "gpt-*", "protocol": APIProtocolResponses}, map[string]any{"pattern": "qwen*", "protocol": APIProtocolAnthropic}, }, }, } - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(withRules, "deepseek-v4-flash")) - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(withRules, "glm-5.3")) - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(withRules, "omen-alpha")) - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(withRules, "kimi-k3")) - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(withRules, "unknown-model")) - require.Equal(t, APIProtocolResponses, openCodeGoNativeProtocol(withRules, "grok-4.6")) - require.Equal(t, APIProtocolAnthropic, openCodeGoNativeProtocol(withRules, "qwen3.8-flash")) + require.Equal(t, APIProtocolChatCompletions, withRules.resolveModelRoutedProtocol("deepseek-v4-flash")) + require.Equal(t, APIProtocolChatCompletions, withRules.resolveModelRoutedProtocol("glm-5.3")) + require.Equal(t, APIProtocolChatCompletions, withRules.resolveModelRoutedProtocol("omen-alpha")) + require.Equal(t, APIProtocolChatCompletions, withRules.resolveModelRoutedProtocol("kimi-k3")) + require.Equal(t, APIProtocolChatCompletions, withRules.resolveModelRoutedProtocol("unknown-model")) + require.Equal(t, APIProtocolResponses, withRules.resolveModelRoutedProtocol("grok-4.6")) + require.Equal(t, APIProtocolAnthropic, withRules.resolveModelRoutedProtocol("qwen3.8-flash")) defaults := &Account{Platform: PlatformOpenCodeGo} - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(defaults, "deepseek-v4-flash")) - require.Equal(t, APIProtocolChatCompletions, openCodeGoNativeProtocol(nil, "grok-4.6")) + require.Equal(t, APIProtocolChatCompletions, defaults.resolveModelRoutedProtocol("deepseek-v4-flash")) } func TestParseOpenCodeGoProtocolRulesAcceptsTypedMaps(t *testing.T) { t.Parallel() - rules, err := parseOpenCodeGoProtocolRules([]map[string]any{ + rules, err := parseProtocolRules([]map[string]any{ {"pattern": "grok-*", "protocol": APIProtocolResponses}, {"pattern": "qwen*", "protocol": APIProtocolAnthropic}, }) require.NoError(t, err) - require.Equal(t, []OpenCodeGoProtocolRule{ + require.Equal(t, []ProtocolRule{ {Pattern: "grok-*", Protocol: APIProtocolResponses}, {Pattern: "qwen*", Protocol: APIProtocolAnthropic}, }, rules) @@ -154,9 +159,9 @@ func TestShouldForwardOpenAIResponsesViaRawChatCompletions_OpenCodeGoIgnoresProb }, } require.False(t, shouldForwardOpenAIResponsesViaRawChatCompletions(account)) - require.Equal(t, APIProtocolResponses, account.ResolveOpenCodeGoUpstreamProtocol("grok-4.6")) - require.Equal(t, APIProtocolResponses, account.ResolveOpenCodeGoUpstreamProtocol("gpt-5.6-luna")) - require.Equal(t, APIProtocolAnthropic, account.ResolveOpenCodeGoUpstreamProtocol("qwen3.8-flash")) + require.Equal(t, APIProtocolResponses, account.resolveModelRoutedProtocol("grok-4.6")) + require.Equal(t, APIProtocolResponses, account.resolveModelRoutedProtocol("gpt-5.6-luna")) + require.Equal(t, APIProtocolAnthropic, account.resolveModelRoutedProtocol("qwen3.8-flash")) } func TestStampOpenAIResponsesUpstreamEndpoint(t *testing.T) { @@ -172,31 +177,31 @@ func TestStampOpenAIResponsesUpstreamEndpoint(t *testing.T) { func TestNormalizeOpenCodeGoProtocolRulesCredentials(t *testing.T) { t.Parallel() - require.NoError(t, NormalizeOpenCodeGoProtocolRulesCredentials(nil)) - require.NoError(t, NormalizeOpenCodeGoProtocolRulesCredentials(map[string]any{})) + require.NoError(t, NormalizeProtocolRulesCredentials(nil)) + require.NoError(t, NormalizeProtocolRulesCredentials(map[string]any{})) creds := map[string]any{ - openCodeGoProtocolRulesKey: []any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": " Grok-* ", "protocol": APIProtocolResponses}, }, } - require.NoError(t, NormalizeOpenCodeGoProtocolRulesCredentials(creds)) - rules, ok := creds[openCodeGoProtocolRulesKey].([]any) + require.NoError(t, NormalizeProtocolRulesCredentials(creds)) + rules, ok := creds[protocolRulesCredentialKey].([]any) require.True(t, ok) require.NotEmpty(t, rules) entry, ok := rules[0].(map[string]any) require.True(t, ok) require.Equal(t, "grok-*", entry["pattern"]) - err := NormalizeOpenCodeGoProtocolRulesCredentials(map[string]any{ - openCodeGoProtocolRulesKey: []any{ + err := NormalizeProtocolRulesCredentials(map[string]any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": "*grok", "protocol": APIProtocolResponses}, }, }) require.Error(t, err) - err = NormalizeOpenCodeGoProtocolRulesCredentials(map[string]any{ - openCodeGoProtocolRulesKey: []any{ + err = NormalizeProtocolRulesCredentials(map[string]any{ + protocolRulesCredentialKey: []any{ map[string]any{"pattern": "grok-*", "protocol": "adaptive"}, }, }) @@ -235,3 +240,29 @@ func TestDefaultOpenCodeGoModelIDsCoverDocumentedCatalog(t *testing.T) { require.Contains(t, ids, "glm-5.3") require.NotEmpty(t, ids) } + +func TestIsOpenCodeUnsupportedModel(t *testing.T) { + t.Parallel() + cases := []struct { + model string + want bool + }{ + {"gemini-3.8-flash", true}, + {"gemini-3.7-flash", true}, + {"opencode/gemini-3.8-flash", true}, + {"opencode-go/gemini-3.1-pro", true}, + {"jev-1.13", true}, + {"jev-1.13-free", true}, + {"opencode/jev-1.13", true}, + {"qwen3.8-max", false}, + {"qwen3.8-flash", false}, + {"gpt-5.6-luna", false}, + {"grok-4.6", false}, + {"minimax-m3", false}, + {"glm-5.3", false}, + {"", false}, + } + for _, tc := range cases { + require.Equal(t, tc.want, IsOpenCodeUnsupportedModel(tc.model), "model=%s", tc.model) + } +} diff --git a/backend/internal/service/opencode_go_usage.go b/backend/internal/service/opencode_go_usage.go index f0408a0f30ac..2e96d3384b1c 100644 --- a/backend/internal/service/opencode_go_usage.go +++ b/backend/internal/service/opencode_go_usage.go @@ -779,7 +779,7 @@ func (s *OpenCodeGoUsageService) refreshLoadedAccount(ctx context.Context, accou return s.persistFailure(ctx, account, intervalMinutes, now, resp.StatusCode, "unauthorized", retryAfter(resp.Header, now), true) } if resp.StatusCode == http.StatusForbidden { - return s.persistFailure(ctx, account, intervalMinutes, now, resp.StatusCode, "OpenCode Go subscription required (403)", retryAfter(resp.Header, now), false) + return s.persistFailure(ctx, account, intervalMinutes, now, resp.StatusCode, "forbidden", retryAfter(resp.Header, now), false) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return s.persistFailure(ctx, account, intervalMinutes, now, resp.StatusCode, "http_error", retryAfter(resp.Header, now), false) @@ -1000,8 +1000,17 @@ func decodeOpenCodeGoUsageSnapshot(extra map[string]any) *OpenCodeGoUsageSnapsho // nextOpenCodeGoUsageDelay computes the not-before delay for the next refresh: // interval * 2^min(failureCount-1, 6) capped at 24h, ±10% jitter capped at 5min, -// never below a Retry-After hint or one minute. +// never below a Retry-After hint or one minute. The hint is honored over the +// local backoff but clamped to the same 24h ceiling: it arrives over a network +// path we do not control (upstream CDN, account proxy), and an unbounded value +// would freeze a group's auto refresh indefinitely. func nextOpenCodeGoUsageDelay(intervalMinutes, failureCount int, retryAfterDuration time.Duration) time.Duration { + if retryAfterDuration < 0 { + retryAfterDuration = 0 + } + if retryAfterDuration > opencodeGoUsageMaxDelay { + retryAfterDuration = opencodeGoUsageMaxDelay + } minimumDelay := retryAfterDuration base := time.Duration(intervalMinutes) * time.Minute if base < opencodeGoUsageMinIntervalMinutes*time.Minute { diff --git a/backend/internal/service/opencode_go_usage_test.go b/backend/internal/service/opencode_go_usage_test.go index 91b904704246..a775584260c3 100644 --- a/backend/internal/service/opencode_go_usage_test.go +++ b/backend/internal/service/opencode_go_usage_test.go @@ -320,7 +320,7 @@ func TestOpenCodeGoUsageRefresh401Unauthorized(t *testing.T) { require.True(t, openCodeGoUsageAutoRefreshEnabled(account)) } -func TestOpenCodeGoUsageRefresh403SubscriptionRequired(t *testing.T) { +func TestOpenCodeGoUsageRefresh403Forbidden(t *testing.T) { account := openCodeGoUsageAccount(7) repo := &openCodeGoUsageTestRepo{accounts: map[int64]*Account{7: account}} stub := &openCodeGoUsageHTTPStub{status: http.StatusForbidden} @@ -330,7 +330,7 @@ func TestOpenCodeGoUsageRefresh403SubscriptionRequired(t *testing.T) { require.NoError(t, err) require.Equal(t, OpenCodeGoUsageStatusFailed, state.Snapshot.Status) require.Equal(t, http.StatusForbidden, state.Snapshot.HTTPStatus) - require.Equal(t, "OpenCode Go subscription required (403)", state.Snapshot.LastError) + require.Equal(t, "forbidden", state.Snapshot.LastError) require.Equal(t, 1, state.Snapshot.FailureCount) } @@ -403,11 +403,39 @@ func TestNextOpenCodeGoUsageDelayBackoff(t *testing.T) { delay = nextOpenCodeGoUsageDelay(15, 0, 2*time.Hour) require.GreaterOrEqual(t, delay, 2*time.Hour) + // a hint beyond the 24h ceiling is clamped, never adopted verbatim + delay = nextOpenCodeGoUsageDelay(15, 0, 48*time.Hour) + require.GreaterOrEqual(t, delay, 23*time.Hour+55*time.Minute) + require.LessOrEqual(t, delay, 24*time.Hour+5*time.Minute) + // floor of one minute delay = nextOpenCodeGoUsageDelay(5, 0, 0) require.GreaterOrEqual(t, delay, time.Minute) } +func TestOpenCodeGoUsageRefreshClampsHugeRetryAfter(t *testing.T) { + now := time.Now().UTC() + account := openCodeGoUsageAccount(7) + repo := &openCodeGoUsageTestRepo{accounts: map[int64]*Account{7: account}} + stub := &openCodeGoUsageHTTPStub{ + status: http.StatusTooManyRequests, + header: http.Header{"Retry-After": []string{"99999999"}}, + } + svc := newOpenCodeGoUsageTestService(t, repo, stub, &upstreamBillingProbeSettingRepo{}) + svc.now = func() time.Time { return now } + + state, err := svc.Refresh(context.Background(), 7) + require.NoError(t, err) + require.Equal(t, OpenCodeGoUsageStatusFailed, state.Snapshot.Status) + require.Equal(t, http.StatusTooManyRequests, state.Snapshot.HTTPStatus) + require.Equal(t, "http_error", state.Snapshot.LastError) + + maxExpected := now.Add(24*time.Hour + 5*time.Minute) + minExpected := now.Add(23*time.Hour + 55*time.Minute) + require.True(t, state.Snapshot.NextRefreshAt.Before(maxExpected.Add(time.Second))) + require.True(t, state.Snapshot.NextRefreshAt.After(minExpected.Add(-time.Second))) +} + func TestOpenCodeGoUsageManualRefreshThrottle(t *testing.T) { now := time.Now().UTC() account := openCodeGoUsageAccount(7) diff --git a/backend/internal/service/opencode_unsupported_models_routing_test.go b/backend/internal/service/opencode_unsupported_models_routing_test.go new file mode 100644 index 000000000000..3e1b514e2faa --- /dev/null +++ b/backend/internal/service/opencode_unsupported_models_routing_test.go @@ -0,0 +1,86 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestOpenCodeUnsupportedModelsRejectMappedAliasesBeforeProtocolSelection(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, ingress := range routingMatrixIngresses() { + for _, protocol := range []string{APIProtocolAdaptive, APIProtocolResponses, APIProtocolChatCompletions, APIProtocolAnthropic} { + for _, mode := range []string{AccountModeGo, AccountModeZen} { + t.Run(ingress.name+"/"+protocol+"/"+mode, func(t *testing.T) { + account := &Account{ID: 105, Platform: PlatformOpenCodeGo, Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "test-key", "api_protocol": protocol, "account_mode": mode, + "model_mapping": map[string]any{"public-model": "gemini-3.8-flash"}, + "protocol_rules": []any{map[string]any{"pattern": "*", "protocol": APIProtocolChatCompletions}}, + }} + body := (routingMatrixCase{ingress: ingress, model: "public-model"}).body() + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, ingress.path, bytes.NewReader(body)) + upstream := &httpUpstreamRecorder{err: errors.New("unexpected upstream request")} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + + err := ingress.forward(svc, c, account, body) + + require.ErrorContains(t, err, "opencode unsupported model: gemini-3.8-flash") + require.Equal(t, http.StatusBadRequest, recorder.Code) + require.Empty(t, upstream.requests) + }) + } + } + } +} + +func TestCommandCodeGeminiModelReachesUpstream(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, ingress := range routingMatrixIngresses() { + t.Run(ingress.name, func(t *testing.T) { + account := commandCodeTestAccount(106) + account.Credentials["api_protocol"] = APIProtocolChatCompletions + body := (routingMatrixCase{ingress: ingress, model: "gemini-3.8-flash"}).body() + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, ingress.path, bytes.NewReader(body)) + upstream := &httpUpstreamRecorder{err: errors.New("stop after capture")} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + + err := ingress.forward(svc, c, account, body) + + require.Error(t, err) + require.NotContains(t, err.Error(), "opencode unsupported model") + require.Len(t, upstream.requests, 1) + require.Equal(t, "https://api.commandcode.ai/provider/v1/chat/completions", upstream.requests[0].URL.String()) + }) + } + account := commandCodeTestAccount(107) + svc, upstream := adaptiveCNAccountTestService(account, adaptiveCNChatTestResponse()) + c, _ := newTestContext() + require.NoError(t, svc.testModelRoutedAccountConnection(c, account, "gemini-3.8-flash", "hi")) + require.Len(t, upstream.requests, 1) +} + +func TestOpenCodeConnectionTestRejectsMappedUnsupportedModel(t *testing.T) { + account := &Account{Platform: PlatformOpenCodeGo, Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_protocol": APIProtocolChatCompletions, + "model_mapping": map[string]any{"public-model": "jev-1.13"}, + }} + c, _ := newTestContext() + c.Request = httptest.NewRequest(http.MethodPost, "/test", nil).WithContext(context.Background()) + err := (&AccountTestService{}).testModelRoutedAccountConnection(c, account, "public-model", "hi") + require.ErrorContains(t, err, "not supported on OpenCode standard gateway") + require.ErrorContains(t, err, "jev-1.13") +} diff --git a/backend/internal/service/opencode_unsupported_models_test.go b/backend/internal/service/opencode_unsupported_models_test.go new file mode 100644 index 000000000000..d6d6518cb464 --- /dev/null +++ b/backend/internal/service/opencode_unsupported_models_test.go @@ -0,0 +1,138 @@ +package service + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestForwardOpenCodeUnsupportedModels_ChatCompletionsReturns400(t *testing.T) { + gin.SetMode(gin.TestMode) + + account := &Account{ + ID: 101, + Platform: PlatformOpenCodeGo, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "test-key", + "account_mode": AccountModeZen, + }, + } + + svc := &OpenAIGatewayService{ + cfg: &config.Config{}, + } + + for _, model := range []string{"gemini-3.8-flash", "jev-1.13", "opencode/gemini-3.7-flash", "opencode-go/jev-1.13-free"} { + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"` + model + `","messages":[{"role":"user","content":"hello"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + require.Nil(t, result) + require.Error(t, err) + require.Contains(t, err.Error(), "opencode unsupported model") + + require.Equal(t, http.StatusBadRequest, rec.Code) + respBody := rec.Body.String() + require.Equal(t, "invalid_request_error", gjson.Get(respBody, "error.type").String()) + require.Equal(t, "model_not_supported", gjson.Get(respBody, "error.code").String()) + require.Contains(t, gjson.Get(respBody, "error.message").String(), "not supported on OpenCode standard gateway") + } +} + +func TestForwardOpenCodeUnsupportedModels_ResponsesReturns400(t *testing.T) { + gin.SetMode(gin.TestMode) + + account := &Account{ + ID: 102, + Platform: PlatformOpenCodeGo, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "test-key", + "account_mode": AccountModeZen, + }, + } + + svc := &OpenAIGatewayService{ + cfg: &config.Config{}, + } + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"gemini-3.8-flash","input":[{"role":"user","content":"hello"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) + + result, err := svc.Forward(context.Background(), c, account, body) + require.Nil(t, result) + require.Error(t, err) + require.Contains(t, err.Error(), "opencode unsupported model") + + require.Equal(t, http.StatusBadRequest, rec.Code) + respBody := rec.Body.String() + require.Equal(t, "invalid_request_error", gjson.Get(respBody, "error.type").String()) + require.Equal(t, "model_not_supported", gjson.Get(respBody, "error.code").String()) +} + +func TestForwardOpenCodeUnsupportedModels_MessagesReturns400(t *testing.T) { + gin.SetMode(gin.TestMode) + + account := &Account{ + ID: 103, + Platform: PlatformOpenCodeGo, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "test-key", + "account_mode": AccountModeZen, + }, + } + + svc := &OpenAIGatewayService{ + cfg: &config.Config{}, + } + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"gemini-3.8-flash","messages":[{"role":"user","content":"hello"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(body)) + + result, err := svc.ForwardAsAnthropic(context.Background(), c, account, body, "", "") + require.Nil(t, result) + require.Error(t, err) + require.Contains(t, err.Error(), "opencode unsupported model") + + require.Equal(t, http.StatusBadRequest, rec.Code) + respBody := rec.Body.String() + require.Equal(t, "invalid_request_error", gjson.Get(respBody, "error.type").String()) + require.Contains(t, gjson.Get(respBody, "error.message").String(), "not supported on OpenCode standard gateway") +} + +func TestAccountTestService_OpenCodeUnsupportedModelReturnsError(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/test", nil) + + account := &Account{ + ID: 104, + Platform: PlatformOpenCodeGo, + Type: AccountTypeAPIKey, + } + + svc := &AccountTestService{} + err := svc.testModelRoutedAccountConnection(c, account, "gemini-3.8-flash", "hi") + require.Error(t, err) + require.Contains(t, err.Error(), "not supported on OpenCode standard gateway") + + err = svc.testModelRoutedAccountConnection(c, account, "jev-1.13", "hi") + require.Error(t, err) + require.Contains(t, err.Error(), "not supported on OpenCode standard gateway") +} diff --git a/backend/internal/service/ops_dashboard_models.go b/backend/internal/service/ops_dashboard_models.go index f189031bd198..7ab9e69bca39 100644 --- a/backend/internal/service/ops_dashboard_models.go +++ b/backend/internal/service/ops_dashboard_models.go @@ -29,6 +29,15 @@ type OpsPercentiles struct { Max *int `json:"max_ms"` } +// OpsOutputTPS describes per-request output rates, not aggregate token throughput. +type OpsOutputTPS struct { + P5 *float64 `json:"p5"` + P10 *float64 `json:"p10"` + P50 *float64 `json:"p50"` + Avg *float64 `json:"avg"` + SampleCount int64 `json:"sample_count"` +} + type OpsDashboardOverview struct { StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` @@ -62,8 +71,9 @@ type OpsDashboardOverview struct { Upstream429Count int64 `json:"upstream_429_count"` Upstream529Count int64 `json:"upstream_529_count"` - QPS OpsRateSummary `json:"qps"` - TPS OpsRateSummary `json:"tps"` + QPS OpsRateSummary `json:"qps"` + TPS OpsRateSummary `json:"tps"` + OutputTPS *OpsOutputTPS `json:"output_tps"` Duration OpsPercentiles `json:"duration"` TTFT OpsPercentiles `json:"ttft"` diff --git a/backend/internal/service/pelican_test_catalog.go b/backend/internal/service/pelican_test_catalog.go index 704c3f2fbab0..ce3dc7ae4c20 100644 --- a/backend/internal/service/pelican_test_catalog.go +++ b/backend/internal/service/pelican_test_catalog.go @@ -325,7 +325,7 @@ func pelicanTestUsesClaudeEffort(account *Account, model string) bool { return true } if account.IsOpenCodeGo() { - return openCodeGoNativeProtocol(account, model) == APIProtocolAnthropic + return account.resolveModelRoutedProtocol(model) == APIProtocolAnthropic } return account.IsCNProvider() && account.GetAPIProtocol() == APIProtocolAnthropic } @@ -338,7 +338,7 @@ func pelicanTestSupportsReasoningWire(account *Account, model string) bool { case account.IsOpenAI(): return true case account.IsOpenCodeGo(): - protocol := openCodeGoNativeProtocol(account, model) + protocol := account.resolveModelRoutedProtocol(model) return protocol == APIProtocolChatCompletions || protocol == APIProtocolResponses case account.IsCNProvider(): // A Pelican task sends one native request. Adaptive accounts do not have diff --git a/backend/internal/service/platform_catalog.go b/backend/internal/service/platform_catalog.go new file mode 100644 index 000000000000..7ef15368c55f --- /dev/null +++ b/backend/internal/service/platform_catalog.go @@ -0,0 +1,110 @@ +package service + +import ( + "sort" + + "github.com/Wei-Shaw/sub2api/internal/domain" +) + +// PlatformCatalog 是前端使用的平台清单:domain 平台清单加上多协议供应商的 +// profile(接入模式、各协议默认基址、内置分流规则)。它被写成前端内置清单 +// frontend/src/constants/platformCatalog.builtin.json(见 +// TestFrontendBuiltinPlatformCatalogInSync),前端据此生成平台选项、展示名与 +// 多协议账号表单,新登记的平台只需重新生成该文件,无需修改前端代码。 +type PlatformCatalog struct { + Platforms []PlatformCatalogEntry `json:"platforms"` + // CompositePrecedence 是组合分组逐平台回退的顺序(先命中者生效)。 + CompositePrecedence []string `json:"composite_precedence"` +} + +// PlatformCatalogEntry 描述一个具体平台;MultiProtocol 仅多协议 API Key 供应商有值。 +type PlatformCatalogEntry struct { + domain.PlatformSpec + MultiProtocol *ProviderProfileInfo `json:"multi_protocol,omitempty"` +} + +// ProviderProfileInfo 是 ProviderProfile 的对外形状。 +type ProviderProfileInfo struct { + DefaultMode string `json:"default_mode"` + Routing ProviderRouting `json:"routing"` + ResponsesPath string `json:"responses_path,omitempty"` + // ModelCatalog 表示模型支持的协议由上游 /models 自动获取(账号规则仍优先)。 + ModelCatalog bool `json:"model_catalog,omitempty"` + // Modes 以默认模式开头,其余按模式名排序。 + Modes []ProviderModeInfo `json:"modes"` +} + +// ProviderModeInfo 是某接入模式的默认端点;BaseURLs 缺少某协议即该模式不提供该原生端点。 +type ProviderModeInfo struct { + Mode string `json:"mode"` + BaseURLs map[string]string `json:"base_urls"` + ProtocolRules []ProtocolRule `json:"protocol_rules,omitempty"` +} + +// BuildPlatformCatalog 按展示顺序返回全部具体平台(不含 composite)。 +func BuildPlatformCatalog() PlatformCatalog { + specs := domain.Platforms() + catalog := PlatformCatalog{ + Platforms: make([]PlatformCatalogEntry, 0, len(specs)), + CompositePrecedence: domain.CompositePrecedencePlatformIDs(), + } + for _, spec := range specs { + catalog.Platforms = append(catalog.Platforms, PlatformCatalogEntry{ + PlatformSpec: spec, + MultiProtocol: providerProfileInfo(LookupProviderProfile(spec.ID)), + }) + } + return catalog +} + +func providerProfileInfo(profile *ProviderProfile) *ProviderProfileInfo { + if profile == nil { + return nil + } + routing := profile.Routing + if routing == "" { + routing = ProviderRoutingByInbound + } + modes := make([]string, 0, len(profile.Modes)) + for mode := range profile.Modes { + if mode != profile.DefaultMode { + modes = append(modes, mode) + } + } + sort.Strings(modes) + if _, ok := profile.Modes[profile.DefaultMode]; ok { + modes = append([]string{profile.DefaultMode}, modes...) + } + info := &ProviderProfileInfo{ + DefaultMode: profile.DefaultMode, + Routing: routing, + ResponsesPath: profile.ResponsesPath, + ModelCatalog: profile.ModelCatalog, + Modes: make([]ProviderModeInfo, 0, len(modes)), + } + for _, mode := range modes { + endpoints := profile.Modes[mode] + baseURLs := make(map[string]string, len(endpoints.BaseURLs)) + for protocol, baseURL := range endpoints.BaseURLs { + baseURLs[protocol] = baseURL + } + info.Modes = append(info.Modes, ProviderModeInfo{ + Mode: mode, + BaseURLs: baseURLs, + ProtocolRules: cloneProtocolRules(endpoints.ProtocolRules), + }) + } + return info +} + +func cloneProtocolRules(rules []ProtocolRule) []ProtocolRule { + if rules == nil { + return nil + } + out := make([]ProtocolRule, len(rules)) + for i, rule := range rules { + out[i] = rule + out[i].Protocols = append([]string(nil), rule.Protocols...) + } + return out +} diff --git a/backend/internal/service/platform_catalog_test.go b/backend/internal/service/platform_catalog_test.go new file mode 100644 index 000000000000..b6da5ae5d6b5 --- /dev/null +++ b/backend/internal/service/platform_catalog_test.go @@ -0,0 +1,140 @@ +//go:build unit + +package service + +import ( + "encoding/json" + "errors" + "os" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestBuildPlatformCatalogMirrorsPlatformListAndProfiles(t *testing.T) { + catalog := BuildPlatformCatalog() + require.Equal(t, domain.CompositePrecedencePlatformIDs(), catalog.CompositePrecedence) + require.Len(t, catalog.Platforms, len(domain.ConcretePlatformIDs())) + + for i, entry := range catalog.Platforms { + require.Equal(t, domain.ConcretePlatformIDs()[i], entry.ID) + profile := LookupProviderProfile(entry.ID) + if profile == nil { + require.Nil(t, entry.MultiProtocol, entry.ID) + continue + } + require.NotNil(t, entry.MultiProtocol, entry.ID) + require.Equal(t, profile.DefaultMode, entry.MultiProtocol.DefaultMode, entry.ID) + require.Equal(t, profile.DefaultMode, entry.MultiProtocol.Modes[0].Mode, entry.ID) + require.Equal(t, profile.ResponsesPath, entry.MultiProtocol.ResponsesPath, entry.ID) + require.Len(t, entry.MultiProtocol.Modes, len(profile.Modes), entry.ID) + for _, mode := range entry.MultiProtocol.Modes { + endpoints := profile.Modes[mode.Mode] + require.Equal(t, endpoints.BaseURLs, mode.BaseURLs, "%s/%s", entry.ID, mode.Mode) + require.Equal(t, len(endpoints.ProtocolRules), len(mode.ProtocolRules), "%s/%s", entry.ID, mode.Mode) + for j, rule := range endpoints.ProtocolRules { + require.Equal(t, rule, mode.ProtocolRules[j]) + } + } + } +} + +func TestBuildPlatformCatalogJSONShape(t *testing.T) { + raw, err := json.Marshal(BuildPlatformCatalog()) + require.NoError(t, err) + + type modeJSON struct { + Mode string `json:"mode"` + BaseURLs map[string]string `json:"base_urls"` + ProtocolRules []ProtocolRule `json:"protocol_rules"` + } + type platformJSON struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` + Gateway string `json:"gateway"` + CNProvider *bool `json:"cn_provider"` + MultiProtocol *struct { + DefaultMode string `json:"default_mode"` + Routing string `json:"routing"` + ResponsesPath string `json:"responses_path"` + Modes []modeJSON `json:"modes"` + } `json:"multi_protocol"` + } + var decoded struct { + Platforms []platformJSON `json:"platforms"` + CompositePrecedence []string `json:"composite_precedence"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, domain.CompositePrecedencePlatformIDs(), decoded.CompositePrecedence) + byID := map[string]platformJSON{} + for _, p := range decoded.Platforms { + byID[p.ID] = p + } + + anthropic := byID[PlatformAnthropic] + require.Equal(t, "Anthropic", anthropic.DisplayName) + require.Equal(t, "anthropic", anthropic.Gateway) + require.NotNil(t, anthropic.CNProvider) + require.False(t, *anthropic.CNProvider) + require.Nil(t, anthropic.MultiProtocol) + require.NotContains(t, string(raw), `"multi_protocol":null`) + + deepseek := byID[PlatformDeepseek].MultiProtocol + require.NotNil(t, deepseek) + require.Equal(t, "by_inbound", deepseek.Routing) + require.Equal(t, "/responses", deepseek.ResponsesPath) + + openCode := byID[PlatformOpenCodeGo].MultiProtocol + require.NotNil(t, openCode) + require.Equal(t, "by_model", openCode.Routing) + require.Equal(t, AccountModeGo, openCode.DefaultMode) + require.Len(t, openCode.Modes, 2) + require.Equal(t, AccountModeGo, openCode.Modes[0].Mode) + require.Equal(t, DefaultOpenCodeGoBaseURL, openCode.Modes[0].BaseURLs[APIProtocolChatCompletions]) + require.Equal(t, DefaultOpenCodeGoProtocolRules(), openCode.Modes[0].ProtocolRules) + require.Equal(t, AccountModeZen, openCode.Modes[1].Mode) +} + +// 下发的是副本:调用方修改结果不影响内置 profile。 +func TestBuildPlatformCatalogReturnsCopies(t *testing.T) { + catalog := BuildPlatformCatalog() + for _, entry := range catalog.Platforms { + if entry.MultiProtocol == nil { + continue + } + for _, mode := range entry.MultiProtocol.Modes { + for protocol := range mode.BaseURLs { + mode.BaseURLs[protocol] = "mutated" + } + for i := range mode.ProtocolRules { + mode.ProtocolRules[i].Protocol = "mutated" + } + } + } + require.Equal(t, DefaultOpenCodeGoBaseURL, LookupProviderProfile(PlatformOpenCodeGo).DefaultBaseURL(AccountModeGo, APIProtocolChatCompletions)) + require.NotEqual(t, "mutated", LookupProviderProfile(PlatformOpenCodeGo).Modes[AccountModeGo].ProtocolRules[0].Protocol) +} + +// 前端内置的平台清单(接口加载前与加载失败时使用)必须与后端一致。 +// 更新平台清单或 profile 后运行: +// +// UPDATE_PLATFORM_CATALOG=1 go test -tags unit ./internal/service/ -run TestFrontendBuiltinPlatformCatalogInSync +func TestFrontendBuiltinPlatformCatalogInSync(t *testing.T) { + const path = "../../../frontend/src/constants/platformCatalog.builtin.json" + want, err := json.MarshalIndent(BuildPlatformCatalog(), "", " ") + require.NoError(t, err) + want = append(want, '\n') + + if os.Getenv("UPDATE_PLATFORM_CATALOG") == "1" { + require.NoError(t, os.WriteFile(path, want, 0o644)) + return + } + got, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + t.Skip("frontend sources not available") + } + require.NoError(t, err) + require.JSONEq(t, string(want), string(got), + "frontend/src/constants/platformCatalog.builtin.json 与后端平台清单不一致,按测试注释重新生成") +} diff --git a/backend/internal/service/platform_list_test.go b/backend/internal/service/platform_list_test.go new file mode 100644 index 000000000000..e6a0d83af846 --- /dev/null +++ b/backend/internal/service/platform_list_test.go @@ -0,0 +1,180 @@ +//go:build unit + +package service + +import ( + "testing" + + "github.com/Wei-Shaw/sub2api/internal/domain" + "github.com/stretchr/testify/require" +) + +// 以下为重构前各处手写的平台列表 / switch,作为平台清单派生结果的等价基准; +// 重构后新登记的平台(Command Code、Cline)按同类平台(OpenCode)的位置补入。 +var ( + legacyAllPlatforms = []string{ + PlatformAnthropic, PlatformOpenAI, PlatformGemini, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, + PlatformCommandCode, + PlatformCline, + } + legacySchedulerSnapshotPlatforms = []string{ + PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe, + PlatformCommandCode, + PlatformCline, + } + legacyCompositeMatchingPlatforms = legacySchedulerSnapshotPlatforms + legacyMultiProtocolProviders = []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformCommandCode, PlatformCline} + platformProbeValues = append(append([]string{}, legacyAllPlatforms...), PlatformComposite, "", "moonshot", "Kimi", "openai ", "glm", "bogus") + platformProbeAccountTypes = []string{AccountTypeAPIKey, AccountTypeOAuth, AccountTypeSetupToken, AccountTypeUpstream, ""} +) + +func legacyIsCNProvider(platform string) bool { + switch platform { + case PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax: + return true + } + return false +} + +func legacyIsOpenAICompatible(platform string) bool { + return platform == PlatformOpenAI || platform == PlatformGrok || legacyIsCNProvider(platform) || platform == PlatformOpenCodeGo || platform == PlatformCommandCode || platform == PlatformCline +} + +func legacyNormalizeOpenAICompatiblePlatform(platform string) string { + switch platform { + case PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformCommandCode, PlatformCline: + return platform + } + return PlatformOpenAI +} + +func legacyIsUpstreamBillingProbeIdentity(platform, accountType string) bool { + if accountType != AccountTypeAPIKey { + return false + } + switch platform { + case PlatformOpenAI, PlatformAnthropic, PlatformGemini, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, + PlatformTypeSafe, PlatformCommandCode, PlatformCline: + return true + } + return false +} + +func legacyIsHeaderOverrideEligible(platform, accountType string) bool { + switch platform { + case PlatformAnthropic, PlatformOpenAI, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformCommandCode, PlatformCline: + return accountType == AccountTypeAPIKey + case PlatformGrok: + return accountType == AccountTypeAPIKey || accountType == AccountTypeOAuth + } + return false +} + +func legacyIsConcreteRequestPlatform(platform string) bool { + for _, p := range legacyAllPlatforms { + if p == platform { + return true + } + } + return false +} + +func TestPlatformListDerivedListsMatchLegacy(t *testing.T) { + require.Equal(t, legacyAllPlatforms, AllowedQuotaPlatforms) + require.Equal(t, legacySchedulerSnapshotPlatforms, schedulerSnapshotPlatforms()) + require.Equal(t, legacyCompositeMatchingPlatforms, matchingPlatforms(PlatformComposite)) + require.Equal(t, []string{PlatformKimi}, matchingPlatforms(PlatformKimi)) + + var multiProtocol []string + for _, platform := range legacyAllPlatforms { + if IsMultiProtocolAPIKeyProvider(platform) { + multiProtocol = append(multiProtocol, platform) + } + } + require.Equal(t, legacyMultiProtocolProviders, multiProtocol) +} + +func TestPlatformListDerivedPredicatesMatchLegacy(t *testing.T) { + for _, platform := range platformProbeValues { + require.Equal(t, legacyIsCNProvider(platform), IsCNProvider(platform), platform) + require.Equal(t, legacyIsConcreteRequestPlatform(platform), isConcreteRequestPlatform(platform), platform) + require.Equal(t, legacyNormalizeOpenAICompatiblePlatform(platform), NormalizeOpenAICompatiblePlatform(platform), platform) + require.Equal(t, legacyIsOpenAICompatible(platform), (&Account{Platform: platform}).IsOpenAICompatible(), platform) + for _, accountType := range platformProbeAccountTypes { + require.Equal(t, legacyIsUpstreamBillingProbeIdentity(platform, accountType), + IsUpstreamBillingProbeIdentity(platform, accountType), "%s/%s", platform, accountType) + require.Equal(t, legacyIsHeaderOverrideEligible(platform, accountType), + (&Account{Platform: platform, Type: accountType}).IsHeaderOverrideEligible(), "%s/%s", platform, accountType) + } + } +} + +// 多协议 API Key 供应商经 OpenAI 网关转发(NormalizeOpenAICompatiblePlatform、 +// countTokens 路由等依赖此约束),ProviderProfile 登记的平台必须在平台清单中声明 openai 网关。 +func TestProviderProfilesUseOpenAIGateway(t *testing.T) { + for platform := range providerProfiles { + spec, ok := domain.LookupPlatform(platform) + require.True(t, ok, platform) + require.Equal(t, domain.PlatformGatewayOpenAI, spec.Gateway, platform) + } +} + +// 转发、探测与计费中"多协议供应商"类判断改为查 profile:现有平台上与改动前的 +// 国产厂商 / 国产厂商 ∪ OpenCode 判断逐一相同。 +func TestProviderProfileAccountPredicatesMatchLegacy(t *testing.T) { + for _, platform := range platformProbeValues { + account := &Account{Platform: platform, Type: AccountTypeAPIKey} + // Cline 只有 Chat Completions 一个端点,按入站协议分流(其余入站转换)。 + require.Equal(t, legacyIsCNProvider(platform) || platform == PlatformCline, account.RoutesProtocolByInbound(), platform) + require.Equal(t, legacyIsCNProvider(platform) || platform == PlatformOpenCodeGo || platform == PlatformCommandCode || platform == PlatformCline, account.IsMultiProtocolAPIKey(), platform) + } + var nilAccount *Account + require.False(t, nilAccount.RoutesProtocolByInbound()) +} + +// 仅在 providerProfiles 中登记的新供应商按分流方式自动获得对应行为:按入站协议 +// 分流的与国产厂商一致,按模型分流的与 OpenCode 一致。 +func TestNewlyRegisteredProviderInheritsRoutingBehaviour(t *testing.T) { + const byInbound, byModel = "test_inbound_provider", "test_model_provider" + providerProfiles[byInbound] = &ProviderProfile{ + Platform: byInbound, DefaultMode: AccountModePayG, Routing: ProviderRoutingByInbound, + Modes: map[string]ProviderEndpoints{AccountModePayG: {BaseURLs: map[string]string{ + APIProtocolChatCompletions: "https://inbound.example.com/v1", + APIProtocolAnthropic: "https://inbound.example.com/anthropic", + }}}, + } + providerProfiles[byModel] = &ProviderProfile{ + Platform: byModel, DefaultMode: AccountModePayG, Routing: ProviderRoutingByModel, + Modes: map[string]ProviderEndpoints{AccountModePayG: {BaseURLs: map[string]string{ + APIProtocolChatCompletions: "https://model.example.com/v1", + }}}, + } + t.Cleanup(func() { + delete(providerProfiles, byInbound) + delete(providerProfiles, byModel) + }) + + inbound := &Account{Platform: byInbound, Type: AccountTypeAPIKey, Credentials: map[string]any{ + "api_key": "sk-test", "api_protocol": APIProtocolAdaptive, + "base_url": "https://inbound.example.com/v1", + }} + require.True(t, inbound.IsMultiProtocolAPIKey()) + require.True(t, inbound.RoutesProtocolByInbound()) + // 无原生 Responses 端点的 adaptive 账号回退 Chat Completions(与 GLM 相同)。 + require.True(t, shouldForwardOpenAIResponsesViaRawChatCompletions(inbound)) + require.True(t, shouldEstimateOpenAIInputTokensLocally(inbound)) + require.Equal(t, "https://inbound.example.com/v1", upstreamModelRegistryBaseURL(inbound)) + + model := &Account{Platform: byModel, Type: AccountTypeAPIKey, Credentials: map[string]any{ + "api_key": "sk-test", "api_protocol": APIProtocolAdaptive, + "base_url": "https://model.example.com/v1", + }} + require.True(t, model.IsMultiProtocolAPIKey()) + require.False(t, model.RoutesProtocolByInbound()) + // 按模型分流:protocol_rules 决定协议,不按 Responses 能力标记回退(与 OpenCode 相同)。 + require.False(t, shouldForwardOpenAIResponsesViaRawChatCompletions(model)) + require.Equal(t, "https://model.example.com/v1", upstreamModelRegistryBaseURL(model)) +} diff --git a/backend/internal/service/provider_profile.go b/backend/internal/service/provider_profile.go new file mode 100644 index 000000000000..37841403a7dc --- /dev/null +++ b/backend/internal/service/provider_profile.go @@ -0,0 +1,267 @@ +package service + +import "strings" + +// ProviderEndpoints 是多协议 API Key 供应商在某个接入模式下的官方默认端点与 +// 内置分流规则。账号凭证里的 base_url / api_base_urls / protocol_rules 仍优先于这里。 +type ProviderEndpoints struct { + // BaseURLs 以上游协议(chat_completions / responses / anthropic)为键, + // 值为该协议的官方默认基址。缺少某协议的条目即表示该模式不提供该原生端点。 + // Anthropic 基址的上游路径为 {base}/v1/messages。 + BaseURLs map[string]string + // ProtocolRules 是按模型分流的内置默认表(首条命中生效);为空表示该模式 + // 不按模型分流,由账号 api_protocol 决定上游协议。 + ProtocolRules []ProtocolRule +} + +// ProviderRouting 决定多协议供应商账号如何选择上游协议。 +type ProviderRouting string + +const ( + // ProviderRoutingByInbound 适用于单一厂商:模型在各原生端点都可用,adaptive 账号 + // 按入站协议选同协议端点,供应商不提供该协议时回落 Chat Completions。 + ProviderRoutingByInbound ProviderRouting = "by_inbound" + // ProviderRoutingByModel 适用于多模型聚合平台:每个模型只在某一协议端点提供, + // 按 protocol_rules 以模型名选协议,与入站协议无关。 + ProviderRoutingByModel ProviderRouting = "by_model" +) + +// ProviderProfile 描述一个多协议 API Key 供应商(国产厂商或多模型聚合平台)的 +// 静态数据:各接入模式的默认端点、内置分流规则与端点路径约定。 +// 额度解析、会话头等行为差异不在这里,仍由各自的代码路径处理。 +type ProviderProfile struct { + Platform string + // DefaultMode 在 credentials.account_mode 缺失或不在 Modes 中时使用。 + DefaultMode string + Modes map[string]ProviderEndpoints + // Routing 为空时按 ProviderRoutingByInbound 处理。 + Routing ProviderRouting + // ResponsesPath 覆盖 Responses 端点路径(按 buildOpenAIEndpointURL 的版本感知 + // 规则拼接),空值为 /v1/responses。对自定义 base_url 的账号同样生效。 + ResponsesPath string + // DefaultTestModel 是管理端连接测试未指定模型时使用的模型;空值时按各协议的 + // 通用默认模型。 + DefaultTestModel string + // ModelCatalog 表示上游 {Chat Completions 基址}/models 为每个模型给出 + // supported_endpoints;按模型分流时据此判断模型支持哪些协议(见 model_protocol_catalog.go)。 + ModelCatalog bool +} + +// providerDefaultTestModel 返回账号连接测试未指定模型时的默认模型;非多协议供应商或 +// 未设置时为空。 +func (a *Account) providerDefaultTestModel() string { + profile := a.providerProfile() + if profile == nil { + return "" + } + // Cline 订阅了 ClinePass 的账号用订阅模型测试,不消耗积分;ClinePass 钱包冷却中 + // (窗口用满等)时改用按量模型,否则测试必然失败,而积分模型照常可用。 + if a.IsCline() && a.clinePassSubscribed() && !a.isRateLimitActiveForKey(clinePassRateLimitKey) { + return DefaultClinePassTestModel + } + return profile.DefaultTestModel +} + +// Endpoints 返回指定接入模式的端点;未知模式回落 DefaultMode。 +func (p *ProviderProfile) Endpoints(mode string) ProviderEndpoints { + if p == nil { + return ProviderEndpoints{} + } + if endpoints, ok := p.Modes[strings.TrimSpace(mode)]; ok { + return endpoints + } + return p.Modes[p.DefaultMode] +} + +// DefaultBaseURL 返回指定接入模式与上游协议的官方默认基址;该模式不提供此协议时为空串。 +func (p *ProviderProfile) DefaultBaseURL(mode, protocol string) string { + return p.Endpoints(mode).BaseURLs[protocol] +} + +// SupportsProtocol 报告指定接入模式是否提供该协议的原生端点。 +func (p *ProviderProfile) SupportsProtocol(mode, protocol string) bool { + return p.DefaultBaseURL(mode, protocol) != "" +} + +// providerProfiles 是多协议 API Key 供应商的单一数据来源。新增同类供应商时在此 +// 登记一条即可获得默认端点、原生 Responses 能力判定(有 responses 条目即支持)、 +// 上游协议分流方式、Responses 路径约定与 IsMultiProtocolAPIKeyProvider 归属。 +// 默认端点需与前端 credentialsBuilder.ts 的预设保持一致。 +var providerProfiles = map[string]*ProviderProfile{ + PlatformKimi: { + Platform: PlatformKimi, + DefaultMode: AccountModePayG, + Routing: ProviderRoutingByInbound, + Modes: map[string]ProviderEndpoints{ + AccountModePayG: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultKimiPayGBaseURL, + APIProtocolResponses: DefaultKimiPayGBaseURL, + APIProtocolAnthropic: DefaultKimiPayGAnthropicBaseURL, + }, + }, + AccountModeCoding: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultKimiCodingBaseURL, + APIProtocolResponses: DefaultKimiCodingBaseURL, + APIProtocolAnthropic: DefaultKimiCodingAnthropicBaseURL, + }, + }, + }, + }, + PlatformZhipu: { + Platform: PlatformZhipu, + DefaultMode: AccountModePayG, + Routing: ProviderRoutingByInbound, + // 智谱没有原生 Responses 端点,故不登记 responses 条目。 + Modes: map[string]ProviderEndpoints{ + AccountModePayG: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultZhipuPayGBaseURL, + APIProtocolAnthropic: DefaultZhipuAnthropicBaseURL, + }, + }, + AccountModeCoding: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultZhipuCodingBaseURL, + APIProtocolAnthropic: DefaultZhipuAnthropicBaseURL, + }, + }, + }, + }, + PlatformDeepseek: { + Platform: PlatformDeepseek, + DefaultMode: AccountModePayG, + Routing: ProviderRoutingByInbound, + // DeepSeek 官方 Responses 端点为 /responses(无 /v1 前缀,适配 Codex)。 + ResponsesPath: "/responses", + Modes: map[string]ProviderEndpoints{ + AccountModePayG: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultDeepseekBaseURL, + APIProtocolResponses: DefaultDeepseekBaseURL, + APIProtocolAnthropic: DefaultDeepseekAnthropicBaseURL, + }, + }, + }, + }, + PlatformMiniMax: { + Platform: PlatformMiniMax, + DefaultMode: AccountModePayG, + Routing: ProviderRoutingByInbound, + Modes: map[string]ProviderEndpoints{ + // 按量付费与 Coding/Token Plan 共用推理域名,靠 API Key 区分套餐。 + AccountModePayG: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultMiniMaxBaseURL, + APIProtocolResponses: DefaultMiniMaxBaseURL, + APIProtocolAnthropic: DefaultMiniMaxAnthropicBaseURL, + }, + }, + AccountModeCoding: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultMiniMaxBaseURL, + APIProtocolResponses: DefaultMiniMaxBaseURL, + APIProtocolAnthropic: DefaultMiniMaxAnthropicBaseURL, + }, + }, + }, + }, + PlatformOpenCodeGo: { + Platform: PlatformOpenCodeGo, + DefaultMode: AccountModeGo, + Routing: ProviderRoutingByModel, + DefaultTestModel: DefaultOpenCodeGoTestModel, + Modes: map[string]ProviderEndpoints{ + AccountModeGo: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultOpenCodeGoBaseURL, + APIProtocolResponses: DefaultOpenCodeGoBaseURL, + APIProtocolAnthropic: DefaultOpenCodeGoAnthropicBaseURL, + }, + ProtocolRules: DefaultOpenCodeGoProtocolRules(), + }, + AccountModeZen: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultOpenCodeZenBaseURL, + APIProtocolResponses: DefaultOpenCodeZenBaseURL, + APIProtocolAnthropic: DefaultOpenCodeZenAnthropicBaseURL, + }, + ProtocolRules: DefaultOpenCodeZenProtocolRules(), + }, + }, + }, + PlatformCommandCode: { + Platform: PlatformCommandCode, + DefaultMode: AccountModePayG, + Routing: ProviderRoutingByModel, + DefaultTestModel: DefaultCommandCodeTestModel, + ModelCatalog: true, + // 订阅套餐与 Provider 按量计费共用同一个 API Key 与端点,只有一个接入模式。 + Modes: map[string]ProviderEndpoints{ + AccountModePayG: { + BaseURLs: map[string]string{ + APIProtocolChatCompletions: DefaultCommandCodeBaseURL, + APIProtocolResponses: DefaultCommandCodeBaseURL, + APIProtocolAnthropic: DefaultCommandCodeAnthropicBaseURL, + }, + ProtocolRules: DefaultCommandCodeProtocolRules(), + }, + }, + }, + PlatformCline: { + Platform: PlatformCline, + DefaultMode: AccountModePayG, + DefaultTestModel: DefaultClineTestModel, + // 按量积分与 ClinePass 订阅共用同一个 API Key 与端点,由模型名决定扣哪一边 + // (cline-pass/* 走订阅),只有一个接入模式;两边的超限分别冷却(见 cline.go)。 + Modes: map[string]ProviderEndpoints{ + AccountModePayG: { + BaseURLs: map[string]string{APIProtocolChatCompletions: DefaultClineBaseURL}, + }, + }, + }, +} + +// LookupProviderProfile 返回 platform 对应的多协议供应商 profile;非此类平台返回 nil。 +func LookupProviderProfile(platform string) *ProviderProfile { + return providerProfiles[platform] +} + +func (a *Account) providerProfile() *ProviderProfile { + if a == nil { + return nil + } + return LookupProviderProfile(a.Platform) +} + +// defaultProviderBaseURL 按账号 account_mode 返回多协议供应商指定协议的官方默认基址; +// 非多协议供应商或协议未知时为空串。 +func (a *Account) defaultProviderBaseURL(protocol string) string { + profile := a.providerProfile() + if profile == nil { + return "" + } + return profile.DefaultBaseURL(a.GetCredential("account_mode"), protocol) +} + +// providerSupportsProtocol 报告账号所在供应商在其 account_mode 下是否提供该协议的原生端点。 +func (a *Account) providerSupportsProtocol(protocol string) bool { + profile := a.providerProfile() + return profile != nil && profile.SupportsProtocol(a.GetCredential("account_mode"), protocol) +} + +// routesByModel 报告账号所在供应商是否按模型名分流上游协议(多模型聚合平台)。 +func (a *Account) routesByModel() bool { + profile := a.providerProfile() + return profile != nil && profile.Routing == ProviderRoutingByModel +} + +// RoutesProtocolByInbound 报告账号是否为按入站协议分流的多协议 API Key 供应商 +// (单一厂商型,目前即国产厂商)。这类账号共享"显式协议优先、adaptive 选同协议 +// 原生端点"的转发、Responses 能力标记与本地 token 估算逻辑;按模型分流的聚合 +// 平台由 protocol_rules 决定协议,走 routesByModel 分支。 +func (a *Account) RoutesProtocolByInbound() bool { + profile := a.providerProfile() + return profile != nil && profile.Routing != ProviderRoutingByModel +} diff --git a/backend/internal/service/provider_profile_test.go b/backend/internal/service/provider_profile_test.go new file mode 100644 index 000000000000..e9fc2994bd4e --- /dev/null +++ b/backend/internal/service/provider_profile_test.go @@ -0,0 +1,188 @@ +//go:build unit + +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// legacyProviderDefaultBaseURL 复述 provider profile 引入前 account.go 中逐平台 +// switch 的默认端点逻辑,作为等价性基准。 +func legacyProviderDefaultBaseURL(platform, mode, protocol string) string { + coding := mode == AccountModeCoding + zen := mode == AccountModeZen + switch protocol { + case APIProtocolAnthropic: + switch platform { + case PlatformKimi: + if coding { + return DefaultKimiCodingAnthropicBaseURL + } + return DefaultKimiPayGAnthropicBaseURL + case PlatformZhipu: + return DefaultZhipuAnthropicBaseURL + case PlatformDeepseek: + return DefaultDeepseekAnthropicBaseURL + case PlatformMiniMax: + return DefaultMiniMaxAnthropicBaseURL + case PlatformOpenCodeGo: + if zen { + return DefaultOpenCodeZenAnthropicBaseURL + } + return DefaultOpenCodeGoAnthropicBaseURL + } + case APIProtocolChatCompletions, APIProtocolResponses: + if platform == PlatformZhipu && protocol == APIProtocolResponses { + // 唯一的函数级差异:旧 switch 对智谱 Responses 也返回 CC 基址,profile + // 以"缺条目即不支持"建模后返回空串。所有运行时调用方都先经 + // UsesNativeCNResponses / SupportsNativeCNResponses 守卫,智谱不会走到这里。 + return "" + } + switch platform { + case PlatformKimi: + if coding { + return DefaultKimiCodingBaseURL + } + return DefaultKimiPayGBaseURL + case PlatformZhipu: + if coding { + return DefaultZhipuCodingBaseURL + } + return DefaultZhipuPayGBaseURL + case PlatformDeepseek: + return DefaultDeepseekBaseURL + case PlatformMiniMax: + return DefaultMiniMaxBaseURL + case PlatformOpenCodeGo: + if zen { + return DefaultOpenCodeZenBaseURL + } + return DefaultOpenCodeGoBaseURL + } + } + return "" +} + +func TestProviderProfile_DefaultBaseURLsMatchLegacySwitches(t *testing.T) { + t.Parallel() + + platforms := []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo} + modes := []string{"", AccountModePayG, AccountModeCoding, AccountModeZen, AccountModeGo, " coding ", "unknown"} + + for _, platform := range platforms { + for _, mode := range modes { + legacyMode := mode + if legacyMode == " coding " { + legacyMode = AccountModeCoding + } + newAccount := func(protocol string) *Account { + return &Account{ + Platform: platform, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "sk-test", + "account_mode": mode, + "api_protocol": protocol, + }, + } + } + name := platform + "/" + mode + + chat := newAccount(APIProtocolChatCompletions) + require.Equal(t, legacyProviderDefaultBaseURL(platform, legacyMode, APIProtocolChatCompletions), chat.GetOpenAIBaseURL(), name) + + anthropic := newAccount(APIProtocolAnthropic) + require.Equal(t, legacyProviderDefaultBaseURL(platform, legacyMode, APIProtocolAnthropic), anthropic.GetAnthropicProtocolBaseURL(), name) + require.Equal(t, legacyProviderDefaultBaseURL(platform, legacyMode, APIProtocolChatCompletions), anthropic.GetOpenAIFormatBaseURL(), name) + + adaptive := newAccount(APIProtocolAdaptive) + for _, protocol := range []string{APIProtocolChatCompletions, APIProtocolResponses, APIProtocolAnthropic, "unknown"} { + require.Equal(t, legacyProviderDefaultBaseURL(platform, legacyMode, protocol), adaptive.GetCNProtocolBaseURL(protocol), name+"/"+protocol) + } + } + } +} + +func TestProviderProfile_ProfilesMatchMultiProtocolPlatforms(t *testing.T) { + t.Parallel() + + for _, platform := range []string{ + PlatformAnthropic, PlatformOpenAI, PlatformGemini, PlatformAntigravity, PlatformGrok, + PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformComposite, "", + } { + want := IsCNProvider(platform) || platform == PlatformOpenCodeGo + require.Equal(t, want, IsMultiProtocolAPIKeyProvider(platform), platform) + require.Equal(t, want, LookupProviderProfile(platform) != nil, platform) + } + + for platform, profile := range providerProfiles { + require.Equal(t, platform, profile.Platform) + _, ok := profile.Modes[profile.DefaultMode] + require.True(t, ok, "%s default mode %q must be registered", platform, profile.DefaultMode) + } +} + +func TestProviderProfile_NativeResponsesMatchesLegacy(t *testing.T) { + t.Parallel() + + legacy := map[string]bool{ + PlatformDeepseek: true, + PlatformKimi: true, + PlatformMiniMax: true, + PlatformOpenCodeGo: true, + } + modes := []string{"", AccountModePayG, AccountModeCoding, AccountModeZen, AccountModeGo, "unknown"} + for _, platform := range []string{PlatformOpenAI, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo} { + for _, mode := range modes { + account := &Account{Platform: platform, Type: AccountTypeAPIKey, Credentials: map[string]any{"account_mode": mode}} + require.Equal(t, legacy[platform], account.SupportsNativeCNResponses(), platform+"/"+mode) + } + } + require.False(t, (*Account)(nil).SupportsNativeCNResponses()) +} + +func TestProviderProfile_BaseURLKeysAreNativeProtocols(t *testing.T) { + t.Parallel() + + for platform, profile := range providerProfiles { + for mode, endpoints := range profile.Modes { + require.NotEmpty(t, endpoints.BaseURLs[APIProtocolChatCompletions], "%s/%s must have a chat_completions base", platform, mode) + for protocol, baseURL := range endpoints.BaseURLs { + require.True(t, isNativeUpstreamProtocol(protocol), "%s/%s has unknown protocol key %q", platform, mode, protocol) + require.NotEmpty(t, baseURL, "%s/%s/%s", platform, mode, protocol) + } + } + } +} + +func TestProviderProfile_ResponsesPathMatchesLegacy(t *testing.T) { + t.Parallel() + + legacy := func(platform, base string) string { + if platform == PlatformDeepseek { + return buildOpenAIEndpointURL(base, "/responses") + } + return buildOpenAIResponsesURL(base) + } + bases := []string{ + "https://api.deepseek.com", "https://relay.example.com", "https://relay.example.com/v1", + "https://opencode.ai/zen/go/v1", "https://open.bigmodel.cn/api/paas/v4", "https://api.moonshot.cn/v1", + } + for _, platform := range []string{PlatformOpenAI, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo} { + for _, base := range bases { + require.Equal(t, legacy(platform, base), buildOpenAIResponsesURLForPlatform(platform, base), platform+" "+base) + } + } +} + +func TestProviderProfile_OpenCodeProtocolRulesByMode(t *testing.T) { + t.Parallel() + + openCode := LookupProviderProfile(PlatformOpenCodeGo) + require.Equal(t, DefaultOpenCodeGoProtocolRules(), openCode.Endpoints(AccountModeGo).ProtocolRules) + require.Equal(t, DefaultOpenCodeGoProtocolRules(), openCode.Endpoints("").ProtocolRules) + require.Equal(t, DefaultOpenCodeZenProtocolRules(), openCode.Endpoints(AccountModeZen).ProtocolRules) + require.Empty(t, LookupProviderProfile(PlatformKimi).Endpoints(AccountModeCoding).ProtocolRules) +} diff --git a/backend/internal/service/ratelimit_cline.go b/backend/internal/service/ratelimit_cline.go new file mode 100644 index 000000000000..a91f630fecca --- /dev/null +++ b/backend/internal/service/ratelimit_cline.go @@ -0,0 +1,270 @@ +package service + +import ( + "context" + "log/slog" + "net/http" + "regexp" + "strings" + "time" + + "github.com/tidwall/gjson" +) + +// Cline 上游错误的响应式处理。依据官方错误说明(https://docs.cline.bot/api/errors)与 +// Cline 客户端的错误分类(cline/cline:sdk/packages/llms/src/providers/errors.ts、 +// apps/vscode/src/services/error/ClineError.ts): +// +// - 402 insufficient_credits:积分不足; +// - 429 SPEND_LIMIT_EXCEEDED:组织为成员设的花费上限; +// - INFERENCE_CAP_ERROR:推理额度上限(客户端归为 QuotaExceeded,文案由服务端给出); +// - "You have reached your … ClinePass limit … Please try again later.":ClinePass +// 5 小时 / 7 天 / 30 天滚动用量上限,文案不带重置时间; +// - "the user is not subscribed to required model plan" 与 "organization accounts +// cannot use individual model inference subscriptions":没有可用的 ClinePass 订阅; +// - "Free limit reached on model … try again in …":免费模型的每日额度,只限该模型。 +// +// 只读取结构化 error.code 与错误消息;ClinePass / 免费模型限额兼容官方固定文案。 +// metadata 等字段可能回显请求内容,不能据此冷却钱包;未识别的错误交回默认逻辑。 +// +// 作用范围:积分与 ClinePass 是同一个 Key 下互不影响的两个钱包,一边超限只冷却该边的 +// 全部模型(模型级冷却键,见 clineWalletRateLimitKey),不停整个账号;钱包按请求模型 +// 判定——cline-pass/* 请求只会被订阅额度拒绝,其余付费模型只会被积分拒绝。 + +type clineErrorKind int + +const ( + clineErrorNone clineErrorKind = iota + // clineCreditsExhausted 积分不足:冷却积分钱包,用量探测发现余额恢复后解除。 + clineCreditsExhausted + // clineSpendLimit 组织花费上限:冷却后由下一次请求重新确认。 + clineSpendLimit + // clineInferenceCap 推理额度上限:归属由请求模型所在的钱包决定。 + clineInferenceCap + // clinePassLimit ClinePass 用量上限:冷却到对应窗口的重置时间。 + clinePassLimit + // clinePassUnavailable 没有可用的 ClinePass 订阅(未订阅或组织账户)。 + clinePassUnavailable + // clineFreeModelLimit 免费模型的每日额度:只限当前模型。 + clineFreeModelLimit +) + +const ( + clinePassLimitReason = "cline_pass_limit" + clinePassUnavailableReason = "cline_pass_unavailable" + clineSpendLimitReason = "cline_spend_limit" + clineInferenceCapReason = "cline_inference_cap" + clineFreeModelLimitReason = "cline_free_model_limit" + clineCreditsReason = "cline_credits_exhausted" + + // clineLimitRecheck 是无法得知重置时间时的冷却:到期后由下一次请求重新确认 + // (管理员可能调高上限、订阅可能续费、滚动窗口逐步释放)。 + clineLimitRecheck = time.Hour + // ClinePass 7 天 / 30 天窗口没有用量快照时的冷却:窗口滚动释放,时间点无从得知, + // 冷却较长但远短于窗口,由下一次请求或用量探测重新确认。 + clinePassWeeklyRecheck = 6 * time.Hour + clinePassMonthlyRecheck = 24 * time.Hour + // clinePassMaxCooldown 限制按快照重置时间冷却的上限(最长的 30 天窗口)。 + clinePassMaxCooldown = 31 * 24 * time.Hour + // clineFreeModelMaxCooldown 是从免费模型文案解析出的重置时长上限(每日额度)。 + clineFreeModelMaxCooldown = 25 * time.Hour +) + +var ( + // clinePassLimitPattern 与 Cline 客户端的判定一致:"you have reached your" 开头、 + // "please try again later." 结尾,中间含 "clinepass limit"。 + clinePassLimitPattern = regexp.MustCompile(`(?is)you have reached your(.*?)clinepass limit.*?please try again later\.`) + clineTryAgainPattern = regexp.MustCompile(`(?i)\btry\s+again\s+in\s+`) +) + +// parseClineError 返回错误分类与命中的限额消息,后者用于解析窗口和重试时间。 +func parseClineError(body []byte) (clineErrorKind, string) { + if !gjson.ValidBytes(body) { + return clineErrorNone, "" + } + err := gjson.GetBytes(body, "error") + if !err.IsObject() { + return clineErrorNone, "" + } + if code := err.Get("code"); code.Type == gjson.String { + if kind := classifyClineErrorCode(code.String()); kind != clineErrorNone { + return kind, "" + } + } + // 官方客户端也识别 details.code 中的组织花费上限。 + if code := err.Get("details.code"); code.Type == gjson.String && classifyClineErrorCode(code.String()) == clineSpendLimit { + return clineSpendLimit, "" + } + for _, path := range []string{"message", "details.message"} { + message := err.Get(path) + if message.Type != gjson.String { + continue + } + if kind := classifyClineLimitMessage(message.String()); kind != clineErrorNone { + return kind, message.String() + } + } + return clineErrorNone, "" +} + +func classifyClineErrorCode(code string) clineErrorKind { + switch strings.ToLower(strings.TrimSpace(code)) { + case "insufficient_credits": + return clineCreditsExhausted + case "spend_limit_exceeded": + return clineSpendLimit + case "inference_cap_error": + return clineInferenceCap + default: + return clineErrorNone + } +} + +func classifyClineLimitMessage(message string) clineErrorKind { + text := strings.ToLower(message) + switch { + case strings.Contains(text, "organization accounts cannot use individual model inference subscriptions"), + strings.Contains(text, "the user is not subscribed to required model plan"): + return clinePassUnavailable + case strings.Contains(text, "free limit reached on model"): + return clineFreeModelLimit + case clinePassLimitPattern.MatchString(message): + return clinePassLimit + default: + return clineErrorNone + } +} + +// clineErrorWallet 返回错误本身指明的钱包冷却键,用于请求模型未知的情况。 +func clineErrorWallet(kind clineErrorKind) string { + switch kind { + case clinePassLimit, clinePassUnavailable, clineInferenceCap: + return clinePassRateLimitKey + case clineCreditsExhausted, clineSpendLimit: + return clineCreditsRateLimitKey + default: + return "" + } +} + +// clinePassWindow 返回 ClinePass 超限文案所指的窗口(5h / weekly / monthly)。 +func clinePassWindow(body []byte) string { + match := clinePassLimitPattern.FindSubmatch(body) + if match == nil { + return "5h" + } + window := strings.ToLower(string(match[1])) + switch { + case strings.Contains(window, "month"): + return "monthly" + case strings.Contains(window, "week"): + return "weekly" + default: + return "5h" + } +} + +// clinePassLimitCooldown 冷却到用量快照中该窗口的重置时间;没有可用快照时按窗口估计。 +func clinePassLimitCooldown(account *Account, window string, now time.Time) time.Duration { + if account != nil && account.Extra != nil { + resetKey := map[string]string{ + "5h": cnExtraSuffix5hReset, + "weekly": cnExtraSuffixWeeklyReset, + "monthly": cnExtraSuffixMonthlyReset, + }[window] + if resetAt := parseSchedulingResetAt(account.Extra[cnExtraKey(PlatformCline, resetKey)]); resetAt != nil { + if wait := resetAt.Sub(now); wait > 0 && wait <= clinePassMaxCooldown { + return wait + } + } + } + switch window { + case "monthly": + return clinePassMonthlyRecheck + case "weekly": + return clinePassWeeklyRecheck + default: + return clineLimitRecheck + } +} + +// clineFreeModelResetAfter 解析 "try again in 3h 20m";无法解析或超出上限时返回 0。 +func clineFreeModelResetAfter(body []byte) time.Duration { + text := string(body) + loc := clineTryAgainPattern.FindStringIndex(text) + if loc == nil { + return 0 + } + after := parseOpenCodeGoUsageLimitResetDuration("resets in " + text[loc[1]:]) + if after <= 0 || after > clineFreeModelMaxCooldown { + return 0 + } + return after +} + +// handleClineError 处理已识别的 Cline 上游错误:冷却对应钱包(或免费模型本身),不停 +// 整个账号。handled=false 时由调用方继续走默认逻辑。 +func (s *RateLimitService) handleClineError( + ctx context.Context, + account *Account, + statusCode int, + responseBody []byte, + upstreamMsg string, +) bool { + kind, limitMessage := parseClineError(responseBody) + if kind == clineErrorNone && statusCode == http.StatusPaymentRequired { + // 402 只会来自积分不足(ClinePass 超限走 429 等其他状态码)。 + kind = clineCreditsExhausted + } + if kind == clineErrorNone { + return false + } + now := time.Now() + modelKey := modelRateLimitKeyForUpstreamModelNotFound(ctx, account, tempUnschedulableModel(ctx, nil)) + + scope := modelKey + if kind != clineFreeModelLimit { + if scope = clineWalletRateLimitKey(modelKey); scope == "" { + scope = clineErrorWallet(kind) + } + } + if scope == "" { + return false + } + + var cooldown time.Duration + var reason string + switch kind { + case clineCreditsExhausted: + cooldown, reason = s.cnBalanceCooldownDuration(), clineCreditsReason + case clineSpendLimit: + cooldown, reason = clineLimitRecheck, clineSpendLimitReason + case clineInferenceCap: + cooldown, reason = clineLimitRecheck, clineInferenceCapReason + if scope == clinePassRateLimitKey { + cooldown = clinePassLimitCooldown(account, "5h", now) + } + case clinePassLimit: + cooldown, reason = clinePassLimitCooldown(account, clinePassWindow([]byte(limitMessage)), now), clinePassLimitReason + case clinePassUnavailable: + cooldown, reason = clineLimitRecheck, clinePassUnavailableReason + case clineFreeModelLimit: + cooldown, reason = clineFreeModelResetAfter([]byte(limitMessage)), clineFreeModelLimitReason + if cooldown <= 0 { + cooldown = clineLimitRecheck + } + } + until := now.Add(cooldown) + if msg := strings.TrimSpace(upstreamMsg); msg != "" { + reason += ": " + msg + } + + setAccountModelRateLimitSnapshot(account, scope, until, reason, now) + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, scope, until, reason); err != nil { + slog.Warn("cline_limit_set_failed", "account_id", account.ID, "scope", scope, "error", err) + return false + } + slog.Info("cline_limit", "account_id", account.ID, "scope", scope, "model", modelKey, + "status_code", statusCode, "until", until.UTC()) + return true +} diff --git a/backend/internal/service/ratelimit_cline_test.go b/backend/internal/service/ratelimit_cline_test.go new file mode 100644 index 000000000000..548dde15479c --- /dev/null +++ b/backend/internal/service/ratelimit_cline_test.go @@ -0,0 +1,146 @@ +//go:build unit + +package service + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" +) + +func TestClassifyClineErrorRequiresStructuredFields(t *testing.T) { + for name, body := range map[string]string{ + "code in model message": `{"error":{"code":"invalid_request_error","message":"Model INFERENCE_CAP_ERROR not found"}}`, + "code suffix": `{"error":{"code":"model-SPEND_LIMIT_EXCEEDED","message":"Invalid model"}}`, + "credits message without code": `{"error":{"message":"Insufficient credits"}}`, + "spend message without code": `{"error":{"message":"SPEND_LIMIT_EXCEEDED"}}`, + "metadata code": `{"error":{"code":429,"message":"Rate limit exceeded","metadata":{"code":"SPEND_LIMIT_EXCEEDED"}}}`, + "metadata pass message": `{"error":{"message":"Invalid model","metadata":{"message":"You have reached your monthly ClinePass limit. Please try again later."}}}`, + "metadata free message": `{"error":{"message":"Invalid model","metadata":{"message":"Free limit reached on model x. Try again in 3h 20m"}}}`, + "root request code": `{"error":{"message":"Invalid model"},"request":{"model":"insufficient_credits"}}`, + "root message": `{"message":"You have reached your monthly ClinePass limit. Please try again later."}`, + "error string": `{"error":"SPEND_LIMIT_EXCEEDED"}`, + "plain code": `Model SPEND_LIMIT_EXCEEDED not found`, + "plain pass message": `You have reached your weekly ClinePass limit. Please try again later.`, + "malformed json": `{"error":{"code":"insufficient_credits"}`, + } { + t.Run(name, func(t *testing.T) { + kind, _ := parseClineError([]byte(body)) + require.Equal(t, clineErrorNone, kind) + }) + } +} + +func TestHandleUpstreamError_Cline400DoesNotCoolWallet(t *testing.T) { + for name, body := range map[string]string{ + "credits model echo": `{"error":{"code":"invalid_request_error","message":"Model insufficient_credits not found"}}`, + "spend model echo": `{"error":{"message":"Model SPEND_LIMIT_EXCEEDED not found"}}`, + "pass model echo": `{"error":{"message":"Model cline-pass/INFERENCE_CAP_ERROR not found"}}`, + "metadata echo": `{"error":{"code":400,"message":"Invalid model","metadata":{"model":"SPEND_LIMIT_EXCEEDED"}}}`, + "plain echo": `Model SPEND_LIMIT_EXCEEDED not found`, + "business code": clineInferenceCapBody, + "pass message": clinePassMonthlyBody, + } { + t.Run(name, func(t *testing.T) { + account := clineTestAccount(400) + repo := &commandCodeRateLimitRepo{} + shouldDisable := NewRateLimitService(repo, nil, &config.Config{}, nil, nil).HandleUpstreamError( + context.Background(), account, http.StatusBadRequest, http.Header{}, []byte(body), "cline-pass/INFERENCE_CAP_ERROR") + + require.False(t, shouldDisable) + require.Empty(t, repo.modelLimits) + require.Zero(t, repo.setErrorCalls) + require.Zero(t, repo.tempCalls) + require.False(t, account.isModelRateLimitedWithContext(context.Background(), "cline-pass/glm-5.3")) + require.False(t, account.isModelRateLimitedWithContext(context.Background(), "anthropic/claude-sonnet-4-6")) + }) + } +} + +func TestHandleUpstreamError_Cline429EchoedKeywordsDoNotCoolWallet(t *testing.T) { + for name, body := range map[string]string{ + "message without code": `{"error":{"message":"Model INFERENCE_CAP_ERROR not found"}}`, + "metadata model": `{"error":{"code":429,"message":"Rate limit exceeded","metadata":{"model":"SPEND_LIMIT_EXCEEDED"}}}`, + "request model": `{"error":{"code":429,"message":"Rate limit exceeded"},"request":{"model":"insufficient_credits"}}`, + "plain text": `Model SPEND_LIMIT_EXCEEDED not found`, + } { + t.Run(name, func(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + NewRateLimitService(repo, nil, &config.Config{}, nil, nil).HandleUpstreamError(context.Background(), + clineTestAccount(429), http.StatusTooManyRequests, http.Header{}, []byte(body), "deepseek/deepseek-v4-flash") + require.Empty(t, repo.modelLimits) + }) + } +} + +func TestHandleUpstreamError_ClineStructuredErrorsPreserveCooldown(t *testing.T) { + for _, tc := range []struct { + name, model, body, scope, reason string + status int + wait time.Duration + }{ + { + name: "credits code before echoed metadata", model: "deepseek/deepseek-v4-flash", + body: `{"error":{"code":"insufficient_credits","message":"Insufficient credits","metadata":{"model":"SPEND_LIMIT_EXCEEDED"}}}`, + scope: clineCreditsRateLimitKey, reason: clineCreditsReason, status: http.StatusPaymentRequired, wait: cnBalanceCheckCooldown(nil), + }, + { + name: "spend code before pass wording", model: "deepseek/deepseek-v4-flash", + body: `{"error":{"code":"SPEND_LIMIT_EXCEEDED","message":"You have reached your monthly ClinePass limit. Please try again later."}}`, + scope: clineCreditsRateLimitKey, reason: clineSpendLimitReason, status: http.StatusTooManyRequests, wait: clineLimitRecheck, + }, + { + name: "details spend code", model: "deepseek/deepseek-v4-flash", + body: `{"error":{"code":429,"message":"Rate limit exceeded","details":{"code":"SPEND_LIMIT_EXCEEDED"}}}`, + scope: clineCreditsRateLimitKey, reason: clineSpendLimitReason, status: http.StatusTooManyRequests, wait: clineLimitRecheck, + }, + { + name: "inference code", model: "cline-pass/glm-5.3", body: clineInferenceCapBody, + scope: clinePassRateLimitKey, reason: clineInferenceCapReason, status: http.StatusTooManyRequests, wait: clineLimitRecheck, + }, + { + name: "pass message ignores metadata window", model: "cline-pass/glm-5.3", + body: `{"error":{"code":429,"metadata":{"message":"You have reached your monthly ClinePass limit. Please try again later."},"message":"You have reached your 5-hour ClinePass limit. Please try again later."}}`, + scope: clinePassRateLimitKey, reason: clinePassLimitReason, status: http.StatusTooManyRequests, wait: clineLimitRecheck, + }, + { + name: "details pass message", model: "cline-pass/glm-5.3", + body: `{"error":{"code":429,"message":"Rate limit exceeded","details":{"message":"You have reached your monthly ClinePass limit. Please try again later."}}}`, + scope: clinePassRateLimitKey, reason: clinePassLimitReason, status: http.StatusTooManyRequests, wait: clinePassMonthlyRecheck, + }, + { + name: "details entitlement message", model: "cline-pass/glm-5.3", + body: `{"error":{"code":403,"message":"Forbidden","details":{"message":"The user is not subscribed to required model plan"}}}`, + scope: clinePassRateLimitKey, reason: clinePassUnavailableReason, status: http.StatusForbidden, wait: clineLimitRecheck, + }, + { + name: "free message ignores metadata retry", model: "cline-free/kimi-k3", + body: `{"error":{"code":429,"metadata":{"message":"Try again in 20h"},"message":"Free limit reached on model cline-free/kimi-k3. Try again in 5m"}}`, + scope: "cline-free/kimi-k3", reason: clineFreeModelLimitReason, status: http.StatusTooManyRequests, wait: 5 * time.Minute, + }, + { + name: "details free message", model: "cline-free/kimi-k3", + body: `{"error":{"code":429,"message":"Rate limit exceeded","details":{"message":"Free limit reached on model cline-free/kimi-k3. Try again in 3h 20m"}}}`, + scope: "cline-free/kimi-k3", reason: clineFreeModelLimitReason, status: http.StatusTooManyRequests, wait: 3*time.Hour + 20*time.Minute, + }, + } { + t.Run(tc.name, func(t *testing.T) { + repo := &commandCodeRateLimitRepo{} + account := clineTestAccount(430) + shouldDisable := NewRateLimitService(repo, nil, &config.Config{}, nil, nil).HandleUpstreamError( + context.Background(), account, tc.status, http.Header{}, []byte(tc.body), tc.model) + + require.False(t, shouldDisable) + require.Len(t, repo.modelLimits, 1) + require.WithinDuration(t, time.Now().Add(tc.wait), repo.modelLimits[tc.scope], time.Second) + require.True(t, strings.HasPrefix(repo.reasons[tc.scope], tc.reason), repo.reasons[tc.scope]) + require.Zero(t, repo.setErrorCalls) + require.Zero(t, repo.tempCalls) + }) + } +} diff --git a/backend/internal/service/ratelimit_cn_providers.go b/backend/internal/service/ratelimit_cn_providers.go index bb222a9f9bc8..226f7bdb0f66 100644 --- a/backend/internal/service/ratelimit_cn_providers.go +++ b/backend/internal/service/ratelimit_cn_providers.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/tidwall/gjson" ) @@ -153,9 +154,17 @@ func (s *RateLimitService) handleCNProviderInsufficientBalance( // 默认 20 分钟)。周期任务会在余额恢复后提前清除,故此处只需保证冷却覆盖到下一次 // 周期检测即可。 func (s *RateLimitService) cnBalanceCooldownDuration() time.Duration { + if s == nil { + return cnBalanceCheckCooldown(nil) + } + return cnBalanceCheckCooldown(s.cfg) +} + +// cnBalanceCheckCooldown 返回余额类冷却时长(= 2× 余额检测周期,默认周期 10 分钟)。 +func cnBalanceCheckCooldown(cfg *config.Config) time.Duration { minutes := 10 - if s != nil && s.cfg != nil { - if cfgMin := s.cfg.Gateway.CNProviders.BalanceCheckIntervalMinutes; cfgMin > 0 { + if cfg != nil { + if cfgMin := cfg.Gateway.CNProviders.BalanceCheckIntervalMinutes; cfgMin > 0 { minutes = cfgMin } } @@ -175,12 +184,20 @@ func cnProviderQuotaSnapshotReset(account *Account, now time.Time) *time.Time { if account == nil || len(account.Extra) == 0 { return nil } - if !account.IsOpenCodeGo() && (!account.IsCNProvider() || !account.IsCodingPlan()) { + switch { + case account.IsOpenCodeGo(): + case account.IsCommandCode(): + // 有充值积分时上游跳过窗口检查,429 与窗口无关。 + if !commandCodeWindowsBinding(account.Extra) { + return nil + } + case account.IsCNProvider() && account.IsCodingPlan(): + default: return nil } provider := account.Platform suffixes := []string{cnExtraSuffix5hReset, cnExtraSuffixWeeklyReset} - if account.IsOpenCodeGo() { + if cnQuotaHasMonthlyWindow(provider) { suffixes = append(suffixes, cnExtraSuffixMonthlyReset) } var earliest *time.Time diff --git a/backend/internal/service/ratelimit_command_code.go b/backend/internal/service/ratelimit_command_code.go new file mode 100644 index 000000000000..4b8d95d01d23 --- /dev/null +++ b/backend/internal/service/ratelimit_command_code.go @@ -0,0 +1,174 @@ +package service + +import ( + "context" + "log/slog" + "net/http" + "regexp" + "strings" + "time" +) + +// Command Code 用量超限的响应式处理,依据官方错误说明 +// (https://commandcode.ai/docs/troubleshooting/errors): +// +// - Rate limit:短时间请求过多,很快恢复,走默认 429 逻辑; +// - Usage exceeded,三种来源: +// 1. 5 小时 / 每周滚动窗口用满,文案如 +// "You've reached your 5-hour usage limit. Resets in 2h 41m (3:00 PM)"; +// 2. 月度积分耗尽; +// 3. 组织为成员设的花费上限,分成员总额度与成员某个模型的额度两种,后者文案如 +// "You've reached the $10.00/mo spending limit your organization set for GLM-5. It resets in 3 days." +// +// 文档没有给出状态码与响应体结构,因此按文案分类,402 / 403 / 429 同样处理; +// 页面路径中的 usage_exceeded 可能是错误码,一并识别。未识别的错误交回默认逻辑。 +// +// 窗口用满写带原因的临时停调(而非 SetRateLimited):充值积分不受滚动窗口限制, +// 额度探测发现充值积分后只清除这一类停调(见 queryCommandCodeUsage)。 + +type commandCodeUsageErrorKind int + +const ( + commandCodeUsageErrorNone commandCodeUsageErrorKind = iota + // commandCodeUsageWindowLimit 滚动窗口用满:冷却到窗口重置。 + commandCodeUsageWindowLimit + // commandCodeUsageCreditsExhausted 积分耗尽:可恢复停调,积分恢复后由周期检测解除。 + commandCodeUsageCreditsExhausted + // commandCodeUsageSpendLimit 组织花费上限:模型级只限当前模型,成员总额度按账号冷却。 + commandCodeUsageSpendLimit +) + +const ( + // commandCodeUsageLimitReason 是窗口用满临时停调的原因前缀,充值后据此只清除这一类停调。 + commandCodeUsageLimitReason = "command_code_usage_limit" + commandCodeSpendLimitReason = "command_code_spend_limit" + // 从文案解析出的重置时长上限:窗口最长为每周,花费上限最长按月。 + commandCodeMaxWindowCooldown = 8 * 24 * time.Hour + commandCodeMaxSpendLimitCooldown = 32 * 24 * time.Hour + // commandCodeAccountSpendLimitRecheck 是账号级花费上限(或作用域不明)的最长冷却: + // 管理员可能随时调高上限,到期后由下一次请求重新确认。 + commandCodeAccountSpendLimitRecheck = time.Hour +) + +// commandCodeModelSpendLimitPattern 匹配组织为成员某个模型设的花费上限 +// ("... spending limit your organization set for GLM-5.");成员总额度没有 "for <模型>"。 +var commandCodeModelSpendLimitPattern = regexp.MustCompile(`(?i)spending limit your organization set for\s+([^.\s][^.]*)`) + +// commandCodeSpendLimitIsModelScoped 报告花费上限是否只针对单个模型。 +func commandCodeSpendLimitIsModelScoped(body []byte) bool { + match := commandCodeModelSpendLimitPattern.FindSubmatch(body) + if match == nil { + return false + } + target := strings.ToLower(strings.TrimSpace(string(match[1]))) + return target != "" && !strings.HasPrefix(target, "you") +} + +func classifyCommandCodeUsageError(body []byte) commandCodeUsageErrorKind { + text := strings.ToLower(string(body)) + switch { + case strings.Contains(text, "spending limit"): + return commandCodeUsageSpendLimit + case strings.Contains(text, "usage limit"): + return commandCodeUsageWindowLimit + case cnProviderResponseIndicatesInsufficientBalance(body), + strings.Contains(text, "insufficient credit"), + strings.Contains(text, "out of credits"), + strings.Contains(text, "usage_exceeded"), + strings.Contains(text, "usage exceeded"): + return commandCodeUsageCreditsExhausted + default: + return commandCodeUsageErrorNone + } +} + +// commandCodeResetAt 解析文案中的 "Resets in 2h 41m" / "resets in 3 days";无法解析 +// 或超过 maxWait 时返回 nil。 +func commandCodeResetAt(body []byte, now time.Time, maxWait time.Duration) *time.Time { + after := parseOpenCodeGoUsageLimitResetDuration(extractUpstreamErrorMessage(body)) + if after <= 0 { + after = parseOpenCodeGoUsageLimitResetDuration(string(body)) + } + if after <= 0 || after > maxWait { + return nil + } + until := now.Add(after) + return &until +} + +// handleCommandCodeUsageError 处理已识别的 Command Code 用量超限;handled=false 时由 +// 调用方继续走默认逻辑。shouldDisable 与同类状态码的既有口径一致:429 不置位; +// 只限单个模型时也不置位,避免调用方把整个账号拉黑。 +func (s *RateLimitService) handleCommandCodeUsageError( + ctx context.Context, + account *Account, + statusCode int, + responseBody []byte, + upstreamMsg string, +) (handled, shouldDisable bool) { + accountWide := statusCode != http.StatusTooManyRequests + now := time.Now() + switch classifyCommandCodeUsageError(responseBody) { + case commandCodeUsageWindowLimit: + until := commandCodeResetAt(responseBody, now, commandCodeMaxWindowCooldown) + if until == nil { + until = cnProviderQuotaSnapshotReset(account, now) + } + if until == nil { + return false, false + } + reason := commandCodeUsageLimitReason + if msg := strings.TrimSpace(upstreamMsg); msg != "" { + reason += ": " + msg + } + s.notifyAccountSchedulingBlocked(account, *until, commandCodeUsageLimitReason) + if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, *until, reason); err != nil { + slog.Warn("command_code_usage_limit_set_failed", "account_id", account.ID, "error", err) + return false, false + } + slog.Info("command_code_usage_limit", "account_id", account.ID, "status_code", statusCode, "reset_at", until.UTC()) + return true, accountWide + case commandCodeUsageCreditsExhausted: + s.handleCNProviderInsufficientBalance(ctx, account, upstreamMsg) + return true, accountWide + case commandCodeUsageSpendLimit: + return true, s.handleCommandCodeSpendLimit(ctx, account, statusCode, responseBody, now) + default: + return false, false + } +} + +// handleCommandCodeSpendLimit 处理组织花费上限:文案指明模型且请求模型已知时只限该 +// 模型,冷却到文案中的重置时间;成员总额度或作用域不明时整个账号冷却,但最长 +// commandCodeAccountSpendLimitRecheck,避免管理员调高上限后账号仍停用数天,也不会 +// 因累积 403 被永久禁用。 +func (s *RateLimitService) handleCommandCodeSpendLimit(ctx context.Context, account *Account, statusCode int, responseBody []byte, now time.Time) (shouldDisable bool) { + reset := commandCodeResetAt(responseBody, now, commandCodeMaxSpendLimitCooldown) + modelKey := modelRateLimitKeyForUpstreamModelNotFound(ctx, account, tempUnschedulableModel(ctx, nil)) + if modelKey != "" && commandCodeSpendLimitIsModelScoped(responseBody) { + until := reset + if until == nil { + fallback, ok := s.get429FallbackCooldown(ctx, account) + if !ok || fallback <= 0 { + fallback = time.Duration(defaultRateLimit429CooldownSeconds) * time.Second + } + next := now.Add(fallback) + until = &next + } + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *until, commandCodeSpendLimitReason); err != nil { + slog.Warn("command_code_spend_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err) + } + slog.Info("command_code_spend_limit", "account_id", account.ID, "model", modelKey, "status_code", statusCode, "reset_at", until.UTC()) + return false + } + until := now.Add(commandCodeAccountSpendLimitRecheck) + if reset != nil && reset.Before(until) { + until = *reset + } + s.notifyAccountSchedulingBlocked(account, until, commandCodeSpendLimitReason) + if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, commandCodeSpendLimitReason); err != nil { + slog.Warn("command_code_spend_limit_set_failed", "account_id", account.ID, "error", err) + } + slog.Info("command_code_spend_limit", "account_id", account.ID, "status_code", statusCode, "until", until.UTC()) + return statusCode != http.StatusTooManyRequests +} diff --git a/backend/internal/service/ratelimit_service.go b/backend/internal/service/ratelimit_service.go index f6c89e7a3dde..c4a32feefdd0 100644 --- a/backend/internal/service/ratelimit_service.go +++ b/backend/internal/service/ratelimit_service.go @@ -439,6 +439,28 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc upstreamMsg = truncateForLog([]byte(upstreamMsg), 512) } + // Command Code 用量超限按文案区分、与状态码无关(见 ratelimit_command_code.go); + // 未识别的错误(含普通频率限制)继续走下面的默认逻辑。 + if account.IsCommandCode() && (statusCode == http.StatusPaymentRequired || + statusCode == http.StatusForbidden || statusCode == http.StatusTooManyRequests) { + if handled, disable := s.handleCommandCodeUsageError(ctx, account, statusCode, responseBody, upstreamMsg); handled { + return disable + } + } + // Cline 积分、花费上限与 ClinePass 超限按结构化错误区分,只冷却对应的钱包。 + // 400 是请求错误,不能因回显的模型名或请求内容触发钱包冷却。 + if account.IsCline() && (statusCode == http.StatusPaymentRequired || + statusCode == http.StatusForbidden || statusCode == http.StatusTooManyRequests) { + if s.handleClineError(ctx, account, statusCode, responseBody, upstreamMsg) { + return false + } + // 402 只会是积分不足:冷却没写进去也不能落到下面的 402 默认分支——那里永久置 + // status=error,而周期探测只覆盖激活账号,充值后无法自动恢复。 + if statusCode == http.StatusPaymentRequired { + return false + } + } + switch statusCode { case 400: // "organization has been disabled" → 永久禁用 @@ -555,9 +577,10 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc shouldDisable = true } case 402: - // 国产供应商:余额不足是可恢复状态(充值/检测恢复后由周期任务自动解除), - // 不能走 handleAuthError 永久置 status=error。改为可恢复的临时停调。 - if account.IsCNProvider() || account.IsOpenCodeZen() { + // 国产供应商 / OpenCode Zen / Command Code:余额(积分)不足是可恢复状态 + // (充值/检测恢复后由周期任务自动解除),不能走 handleAuthError 永久置 + // status=error。改为可恢复的临时停调。 + if account.IsCNProvider() || account.IsOpenCodeZen() || account.IsCommandCode() { s.handleCNProviderInsufficientBalance(ctx, account, upstreamMsg) shouldDisable = true break @@ -1046,7 +1069,7 @@ func (s *RateLimitService) handle403(ctx context.Context, account *Account, upst // 国产供应商与 openai 同口径:HTML 403(CDN/代理拦截页)不构成账号失效证据, // 且 403 在 failover 状态集里会被逐账号重放——直接 SetError 会让一个坏请求/ // 一层坏代理连环永久禁用整组账号。走 HTML 豁免 + N 次累计 + 临时冷却。 - if account.Platform == PlatformOpenAI || IsCNProvider(account.Platform) || account.IsOpenCodeGo() { + if account.Platform == PlatformOpenAI || account.IsMultiProtocolAPIKey() { return s.handleOpenAI403(ctx, account, upstreamMsg, responseBody) } // 非 Antigravity 平台:保持原有行为 diff --git a/backend/internal/service/reasoning_recovery_setting.go b/backend/internal/service/reasoning_recovery_setting.go deleted file mode 100644 index 8250e12dee37..000000000000 --- a/backend/internal/service/reasoning_recovery_setting.go +++ /dev/null @@ -1,170 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "sync" - "sync/atomic" - "time" - - infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" -) - -// Reasoning recovery has one switch for every OpenAI account. It is one JSON -// setting; the forwarding path reads an in-memory value, so deciding whether a -// rejected reasoning ciphertext may be recovered never queries the database. - -// SettingKeyReasoningRecoveryConfig stores ReasoningRecoveryConfig as JSON. -// Without a stored value recovery is enabled. -const SettingKeyReasoningRecoveryConfig = "reasoning_recovery_config" - -const ( - reasoningRecoveryRefreshInterval = time.Minute - reasoningRecoveryRetryInterval = 5 * time.Second - reasoningRecoveryStoreTimeout = 5 * time.Second - reasoningRecoveryWarnInterval = time.Minute -) - -var ErrReasoningRecoveryUnavailable = infraerrors.ServiceUnavailable("REASONING_RECOVERY_UNAVAILABLE", "reasoning recovery configuration is unavailable") - -// ReasoningRecoveryConfig is the stored switch and its admin view. -type ReasoningRecoveryConfig struct { - Enabled bool `json:"enabled"` -} - -// DecodeReasoningRecoveryConfig accepts exactly {"enabled": }. -func DecodeReasoningRecoveryConfig(raw []byte) (ReasoningRecoveryConfig, error) { - var fields struct { - Enabled *bool `json:"enabled"` - } - if err := DecodeSwitchSettings(raw, &fields, "enabled"); err != nil { - return ReasoningRecoveryConfig{}, err - } - if fields.Enabled == nil { - return ReasoningRecoveryConfig{}, errors.New("enabled is required") - } - return ReasoningRecoveryConfig{Enabled: *fields.Enabled}, nil -} - -// ReasoningRecoveryService owns the switch and its in-memory value. -type ReasoningRecoveryService struct { - settings SettingRepository - - // storeMu orders stored reads and writes, so a refresh that began before a - // save cannot publish the older value after it. - storeMu sync.Mutex - // The zero value is the default: enabled. - disabled atomic.Bool - nextRefresh atomic.Int64 - refreshing atomic.Bool - warnedAt atomic.Int64 -} - -func NewReasoningRecoveryService(settings SettingRepository) *ReasoningRecoveryService { - return &ReasoningRecoveryService{settings: settings} -} - -// ProvideReasoningRecoveryService loads the switch once at startup. A failed -// read stops startup: a stored "off" must not silently become the enabled -// default. -func ProvideReasoningRecoveryService(settings SettingRepository) (*ReasoningRecoveryService, error) { - svc := NewReasoningRecoveryService(settings) - ctx, cancel := context.WithTimeout(context.Background(), reasoningRecoveryStoreTimeout) - defer cancel() - if _, err := svc.Get(ctx); err != nil { - return nil, err - } - return svc, nil -} - -// Enabled reports the switch for one forwarding decision. It reads memory only -// and never waits: when the value is stale a background refresh picks up what -// another instance saved. Without a service the default applies. -func (s *ReasoningRecoveryService) Enabled() bool { - if s == nil { - return true - } - if s.settings != nil && time.Now().UnixNano() >= s.nextRefresh.Load() && s.refreshing.CompareAndSwap(false, true) { - go func() { - defer s.refreshing.Store(false) - ctx, cancel := context.WithTimeout(context.Background(), reasoningRecoveryStoreTimeout) - defer cancel() - if _, err := s.Get(ctx); err != nil { - s.warn("reasoning_recovery.refresh_failed", "error", err) - } - }() - } - return !s.disabled.Load() -} - -// Get returns the stored switch and makes it this instance's value. A failed -// read keeps the value this instance already applies. -func (s *ReasoningRecoveryService) Get(ctx context.Context) (ReasoningRecoveryConfig, error) { - if s == nil || s.settings == nil { - return ReasoningRecoveryConfig{}, ErrReasoningRecoveryUnavailable - } - s.storeMu.Lock() - defer s.storeMu.Unlock() - config, err := s.read(ctx) - if err != nil { - s.nextRefresh.Store(time.Now().Add(reasoningRecoveryRetryInterval).UnixNano()) - return ReasoningRecoveryConfig{}, err - } - s.publish(config) - return config, nil -} - -// Save stores the switch and applies it to this instance at once. -func (s *ReasoningRecoveryService) Save(ctx context.Context, config ReasoningRecoveryConfig) (ReasoningRecoveryConfig, error) { - if s == nil || s.settings == nil { - return ReasoningRecoveryConfig{}, ErrReasoningRecoveryUnavailable - } - raw, err := json.Marshal(config) - if err != nil { - return ReasoningRecoveryConfig{}, err - } - s.storeMu.Lock() - defer s.storeMu.Unlock() - dbCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), reasoningRecoveryStoreTimeout) - defer cancel() - if err := s.settings.Set(dbCtx, SettingKeyReasoningRecoveryConfig, string(raw)); err != nil { - return ReasoningRecoveryConfig{}, err - } - s.publish(config) - return config, nil -} - -// read returns the stored switch; a missing row is the enabled default. -func (s *ReasoningRecoveryService) read(ctx context.Context) (ReasoningRecoveryConfig, error) { - dbCtx, cancel := context.WithTimeout(ctx, reasoningRecoveryStoreTimeout) - defer cancel() - raw, err := s.settings.GetValue(dbCtx, SettingKeyReasoningRecoveryConfig) - if errors.Is(err, ErrSettingNotFound) { - return ReasoningRecoveryConfig{Enabled: true}, nil - } - if err != nil { - return ReasoningRecoveryConfig{}, fmt.Errorf("read reasoning recovery configuration: %w", err) - } - config, err := DecodeReasoningRecoveryConfig([]byte(raw)) - if err != nil { - return ReasoningRecoveryConfig{}, fmt.Errorf("stored reasoning recovery configuration is invalid: %w", err) - } - return config, nil -} - -func (s *ReasoningRecoveryService) publish(config ReasoningRecoveryConfig) { - s.disabled.Store(!config.Enabled) - s.nextRefresh.Store(time.Now().Add(reasoningRecoveryRefreshInterval).UnixNano()) -} - -func (s *ReasoningRecoveryService) warn(msg string, args ...any) { - now := time.Now().UnixNano() - last := s.warnedAt.Load() - if now-last < int64(reasoningRecoveryWarnInterval) || !s.warnedAt.CompareAndSwap(last, now) { - return - } - slog.Warn(msg, args...) -} diff --git a/backend/internal/service/reasoning_recovery_setting_test.go b/backend/internal/service/reasoning_recovery_setting_test.go deleted file mode 100644 index d6877fbf00f7..000000000000 --- a/backend/internal/service/reasoning_recovery_setting_test.go +++ /dev/null @@ -1,187 +0,0 @@ -package service - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "sync" - "testing" - "time" - - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" -) - -// newReasoningRecoverySwitchForTest returns a switch with a fixed value and no -// store behind it. -func newReasoningRecoverySwitchForTest(enabled bool) *ReasoningRecoveryService { - recovery := NewReasoningRecoveryService(nil) - recovery.publish(ReasoningRecoveryConfig{Enabled: enabled}) - return recovery -} - -type reasoningRecoverySettingStore struct { - SettingRepository - mu sync.Mutex - values map[string]string - reads int - readErr error - // A non-nil gate holds the answer of every read until it is closed. The - // value is read before that, as a query that is still in flight has. - gate chan struct{} -} - -func (s *reasoningRecoverySettingStore) GetValue(_ context.Context, key string) (string, error) { - s.mu.Lock() - s.reads++ - value, ok := s.values[key] - err, gate := s.readErr, s.gate - s.mu.Unlock() - if gate != nil { - <-gate - } - if err != nil { - return "", err - } - if !ok { - return "", ErrSettingNotFound - } - return value, nil -} - -func (s *reasoningRecoverySettingStore) Set(_ context.Context, key, value string) error { - s.mu.Lock() - defer s.mu.Unlock() - s.values[key] = value - return nil -} - -func (s *reasoningRecoverySettingStore) put(value string) { - s.mu.Lock() - defer s.mu.Unlock() - s.values[SettingKeyReasoningRecoveryConfig] = value -} - -func (s *reasoningRecoverySettingStore) readCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return s.reads -} - -func (s *reasoningRecoverySettingStore) hold() chan struct{} { - gate := make(chan struct{}) - s.mu.Lock() - defer s.mu.Unlock() - s.gate = gate - return gate -} - -func TestReasoningRecoveryServiceDefaultsToEnabledWithoutRow(t *testing.T) { - store := &reasoningRecoverySettingStore{values: map[string]string{}} - recovery, err := ProvideReasoningRecoveryService(store) - require.NoError(t, err) - require.True(t, recovery.Enabled()) - config, err := recovery.Get(context.Background()) - require.NoError(t, err) - require.Equal(t, ReasoningRecoveryConfig{Enabled: true}, config) - require.Empty(t, store.values, "reading the default stores nothing") - require.True(t, (*ReasoningRecoveryService)(nil).Enabled(), "a gateway without the service applies the default") -} - -func TestReasoningRecoveryServiceSaveAppliesToTheNextRequestWithoutDatabaseRead(t *testing.T) { - gin.SetMode(gin.TestMode) - store := &reasoningRecoverySettingStore{values: map[string]string{}} - recovery, err := ProvideReasoningRecoveryService(store) - require.NoError(t, err) - svc := &OpenAIGatewayService{} - svc.SetReasoningRecoveryService(recovery) - account := &Account{ID: 31, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} - requestEnabled := func() bool { - c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) - return svc.newOpenAIReasoningRecoveryState(context.Background(), c, account, "source-token").enabled - } - require.True(t, requestEnabled()) - - saved, err := recovery.Save(context.Background(), ReasoningRecoveryConfig{Enabled: false}) - require.NoError(t, err) - require.Equal(t, ReasoningRecoveryConfig{Enabled: false}, saved) - require.Equal(t, map[string]string{SettingKeyReasoningRecoveryConfig: `{"enabled":false}`}, store.values) - reads := store.readCount() - require.False(t, requestEnabled(), "the very next request applies the saved switch") - require.False(t, recovery.Enabled()) - require.Equal(t, reads, store.readCount(), "the forwarding path reads memory only") - - stored, err := recovery.Get(context.Background()) - require.NoError(t, err) - require.Equal(t, ReasoningRecoveryConfig{Enabled: false}, stored) - - _, err = recovery.Save(context.Background(), ReasoningRecoveryConfig{Enabled: true}) - require.NoError(t, err) - require.Equal(t, map[string]string{SettingKeyReasoningRecoveryConfig: `{"enabled":true}`}, store.values) - require.True(t, requestEnabled()) -} - -func TestReasoningRecoveryServiceKeepsItsValueWhenTheStoredOneIsUnreadable(t *testing.T) { - store := &reasoningRecoverySettingStore{values: map[string]string{SettingKeyReasoningRecoveryConfig: `{"enabled":false}`}} - recovery, err := ProvideReasoningRecoveryService(store) - require.NoError(t, err) - require.False(t, recovery.Enabled()) - for _, raw := range []string{`true`, `{"enabled":"false"}`, `{"enabled":null}`, `{"enabled":false,"other":true}`, `{}`, ``} { - store.put(raw) - _, err := recovery.Get(context.Background()) - require.Error(t, err, raw) - require.False(t, recovery.Enabled(), "a stored off never becomes the enabled default") - } - store.put(`{"enabled":false}`) - store.readErr = errors.New("database unavailable") - _, err = recovery.Get(context.Background()) - require.Error(t, err) - require.False(t, recovery.Enabled()) - _, err = ProvideReasoningRecoveryService(store) - require.Error(t, err, "startup does not guess the switch") -} - -func TestReasoningRecoveryServiceConvergesOnAnotherInstancesSave(t *testing.T) { - store := &reasoningRecoverySettingStore{values: map[string]string{}} - recovery, err := ProvideReasoningRecoveryService(store) - require.NoError(t, err) - store.put(`{"enabled":false}`) - require.True(t, recovery.Enabled(), "a fresh value is served from memory") - - gate := store.hold() - recovery.nextRefresh.Store(0) - require.True(t, recovery.Enabled(), "a stale read returns at once and refreshes in the background") - close(gate) - require.Eventually(t, func() bool { return !recovery.Enabled() }, time.Second, time.Millisecond) -} - -// A refresh that read the stored value before a save must not publish it after -// the save: the saving instance keeps what it saved. -func TestReasoningRecoveryServiceSaveIsNotReplacedByAnEarlierRead(t *testing.T) { - store := &reasoningRecoverySettingStore{values: map[string]string{}} - recovery, err := ProvideReasoningRecoveryService(store) - require.NoError(t, err) - reads := store.readCount() - gate := store.hold() - recovery.nextRefresh.Store(0) - require.True(t, recovery.Enabled()) - require.Eventually(t, func() bool { return store.readCount() > reads }, time.Second, time.Millisecond, "the refresh has read the stored default") - - saved := make(chan error, 1) - go func() { - _, err := recovery.Save(context.Background(), ReasoningRecoveryConfig{Enabled: false}) - saved <- err - }() - select { - case err := <-saved: - t.Fatalf("the save did not wait for the read that began before it: %v", err) - case <-time.After(30 * time.Millisecond): - } - close(gate) - require.NoError(t, <-saved) - require.False(t, recovery.Enabled()) - require.Eventually(t, func() bool { return !recovery.refreshing.Load() }, time.Second, time.Millisecond) - require.False(t, recovery.Enabled(), "the earlier read cannot replace the saved value") -} diff --git a/backend/internal/service/scheduler_snapshot_service.go b/backend/internal/service/scheduler_snapshot_service.go index 3c911e3e4e89..980672594e33 100644 --- a/backend/internal/service/scheduler_snapshot_service.go +++ b/backend/internal/service/scheduler_snapshot_service.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" ) @@ -608,10 +609,10 @@ func (s *SchedulerSnapshotService) handleBulkAccountEvent(ctx context.Context, p continue } accountGroupIDs := s.normalizeGroupIDs(account.GroupIDs) - switch account.Platform { - case PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe: + switch { + case account.Platform != PlatformAntigravity && isConcreteRequestPlatform(account.Platform): addPlatformGroups(account.Platform, accountGroupIDs) - case PlatformAntigravity: + case account.Platform == PlatformAntigravity: // 批量更新可能刚关闭 mixed_scheduling,仍需清理两个兼容平台的旧快照。 addPlatformGroups(PlatformAntigravity, accountGroupIDs) addPlatformGroups(PlatformAnthropic, accountGroupIDs) @@ -824,8 +825,9 @@ func (s *SchedulerSnapshotService) rebuildByAccount(ctx context.Context, account return s.rebuildBuckets(ctx, buckets, reason) } -func schedulerSnapshotPlatforms() [11]string { - return [11]string{PlatformAnthropic, PlatformGemini, PlatformOpenAI, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, PlatformTypeSafe} +// schedulerSnapshotPlatforms 返回需要维护调度快照的全部具体平台(平台清单)。 +func schedulerSnapshotPlatforms() []string { + return domain.CompositePrecedencePlatformIDs() } // 生命周期辅助函数有意排除 group0;full rebuild 构造 group0 canonical 集时必须显式调用 canonical helper。 diff --git a/backend/internal/service/system_prompt.go b/backend/internal/service/system_prompt.go index 0e3abb7b4e7d..ce7fb51ef1d6 100644 --- a/backend/internal/service/system_prompt.go +++ b/backend/internal/service/system_prompt.go @@ -74,6 +74,7 @@ var systemPromptIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$ var SystemPromptPlatforms = []string{ PlatformAnthropic, PlatformOpenAI, PlatformGemini, PlatformAntigravity, PlatformGrok, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, + PlatformCommandCode, PlatformCline, } // AccountTakesSystemPrompt reports whether the account's platform has an diff --git a/backend/internal/service/system_prompt_openai.go b/backend/internal/service/system_prompt_openai.go index d6f909871404..afca851b729b 100644 --- a/backend/internal/service/system_prompt_openai.go +++ b/backend/internal/service/system_prompt_openai.go @@ -2,7 +2,6 @@ package service import ( "bytes" - "errors" "fmt" "io" "net/http" @@ -174,26 +173,9 @@ func normalizeSystemPromptRejectedFieldRetryBody(c *gin.Context, status int, cle return normalizeOpenAIResponsesRejectedFieldRetryBody(status, clean, out) } -var errReasoningWireProjection = errors.New("reasoning recovery edit cannot be applied to the clean request") - // prepareReasoningRecoveryRequest lets reasoning recovery inspect the exact // wire body produced by the builder, then repeats its edits on the clean body // that later retries rebuild from. -func prepareReasoningRecoveryRequest(recovery *openAIReasoningRecoveryState, req *http.Request, clean []byte, proxyURL string) (*http.Request, []byte, []byte, error) { - wire, err := finalWireRequestBody(req, clean) - if err != nil { - return nil, nil, nil, err - } - prepared, final, err := recovery.PrepareRequest(req, wire, proxyURL) - if err != nil { - return nil, nil, nil, err - } - clean, err = projectReasoningRecoveryEdits(clean, wire, final) - if err != nil { - return nil, nil, nil, err - } - return prepared, clean, final, nil -} // finalWireRequestBody reads the body a builder attached to req. func finalWireRequestBody(req *http.Request, fallback []byte) ([]byte, error) { @@ -227,71 +209,6 @@ func finalWirePromptCacheKey(wire []byte, seed string) string { // relative order the system prompt never changes, and the id of an item it // has stripped: with the ciphertext, or afterwards from the item left behind. // Any other difference is rejected instead of becoming the next retry's source. -func projectReasoningRecoveryEdits(clean, before, after []byte) ([]byte, error) { - if bytes.Equal(before, after) { - return clean, nil - } - wireItems, cleanItems := openAIReasoningCipherItems(before), openAIReasoningCipherItems(clean) - if len(wireItems) != len(cleanItems) { - return nil, errReasoningWireProjection - } - var afterItems []gjson.Result - if input := gjson.GetBytes(after, "input"); input.IsArray() { - afterItems = input.Array() - } - lost := func(index int, field string) bool { - return index >= len(afterItems) || !afterItems[index].Get(field).Exists() - } - var wireEdit, cleanEdit openAIReasoningRecoveryEdit - for index, item := range wireItems { - if item.hash != cleanItems[index].hash { - return nil, errReasoningWireProjection - } - if !lost(item.index, "encrypted_content") { - continue - } - wireEdit.cipher = append(wireEdit.cipher, item.index) - cleanEdit.cipher = append(cleanEdit.cipher, cleanItems[index].index) - if item.id != "" && lost(item.index, "id") { - wireEdit.ids = append(wireEdit.ids, item.index) - cleanEdit.ids = append(cleanEdit.ids, cleanItems[index].index) - } - } - // An item left behind without ciphertext has no hash to pair it by. It is - // the same item in both bodies when it is the same occurrence of its id. - wireBare, cleanBare := openAIReasoningBareItemsByID(before), openAIReasoningBareItemsByID(clean) - for id, indices := range wireBare { - for occurrence, index := range indices { - if !lost(index, "id") { - continue - } - if occurrence >= len(cleanBare[id]) { - return nil, errReasoningWireProjection - } - wireEdit.ids = append(wireEdit.ids, index) - cleanEdit.ids = append(cleanEdit.ids, cleanBare[id][occurrence]) - } - } - expected, err := applyOpenAIReasoningRecoveryEdit(before, wireEdit) - if err != nil || !bytes.Equal(expected, after) { - return nil, errReasoningWireProjection - } - return applyOpenAIReasoningRecoveryEdit(clean, cleanEdit) -} // openAIReasoningBareItemsByID lists, for each id, the input positions of the // reasoning items that carry it and have no ciphertext. -func openAIReasoningBareItemsByID(body []byte) map[string][]int { - out := map[string][]int{} - input := gjson.GetBytes(body, "input") - if !input.IsArray() { - return out - } - for index, item := range input.Array() { - id := item.Get("id") - if item.Get("type").String() == "reasoning" && !openAIReasoningItemHasCipher(item) && id.Type == gjson.String && id.String() != "" { - out[id.String()] = append(out[id.String()], index) - } - } - return out -} diff --git a/backend/internal/service/system_prompt_test.go b/backend/internal/service/system_prompt_test.go index ddc9bf7765a5..3fc4d345a61c 100644 --- a/backend/internal/service/system_prompt_test.go +++ b/backend/internal/service/system_prompt_test.go @@ -33,6 +33,19 @@ func TestSystemPromptSetBindingsSkipsAccountsWithoutInsertionPoint(t *testing.T) require.Empty(t, repo.bulkUpdateIDs) } +func TestSystemPromptNewPlatformsUseInsertionPoints(t *testing.T) { + repo := &accountRepoStubForBulkUpdate{getByIDsAccounts: []*Account{ + {ID: 11, Platform: PlatformCommandCode, Type: AccountTypeAPIKey}, + {ID: 12, Platform: PlatformCline, Type: AccountTypeAPIKey}, + {ID: 13, Platform: PlatformTypeSafe, Type: AccountTypeAPIKey}, + }} + service := NewSystemPromptService(nil, repo, nil) + updated, err := service.SetBindings(context.Background(), []int64{11, 12, 13}, SystemPromptBinding{Mode: SystemPromptModeOff}) + require.NoError(t, err) + require.Equal(t, int64(2), updated) + require.Equal(t, []int64{11, 12}, repo.bulkUpdateIDs) +} + // Besides the binding endpoint only account creation stores a binding, and only // for an account that can take a prompt: data import and duplicate create from // a stored extra. The key-level writers of extra store none on any platform. diff --git a/backend/internal/service/testdata/record_upstream_protocol_routing_main.py b/backend/internal/service/testdata/record_upstream_protocol_routing_main.py new file mode 100644 index 000000000000..dd12d90e46b4 --- /dev/null +++ b/backend/internal/service/testdata/record_upstream_protocol_routing_main.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Record the routing golden from the pinned main source, never the checkout.""" + +import io +import json +import os +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile + + +def format_golden(fixture): + """Keep observations readable and wrap case indexes in compact rows.""" + metadata = {key: value for key, value in fixture.items() if key != "results"} + prefix = json.dumps(metadata, ensure_ascii=False, indent=2)[:-2] + indexes = fixture["results"] + rows = [" " + ", ".join(map(str, indexes[i:i + 32])) for i in range(0, len(indexes), 32)] + return (prefix + ',\n "results": [\n' + ",\n".join(rows) + "\n ]\n}\n").encode("utf-8") + + +def main(): + repo = Path(__file__).resolve().parents[4] + driver = repo / "backend/internal/service/upstream_protocol_routing_matrix_test.go" + source = driver.read_text() + revision = re.search(r'const routingMatrixMainRevision = "([0-9a-f]{40})"', source).group(1) + subprocess.run(["git", "cat-file", "-e", revision + "^{commit}"], cwd=repo, check=True) + archive = subprocess.check_output(["git", "archive", revision, "backend"], cwd=repo) + + with tempfile.TemporaryDirectory(prefix="sub2api-routing-main-") as scratch: + root = Path(scratch) + with tarfile.open(fileobj=io.BytesIO(archive)) as files: + files.extractall(root, filter="data") + recording_driver = root / "backend/internal/service/upstream_protocol_routing_matrix_test.go" + recording_driver.write_text(source + r''' + +func TestRecordUpstreamProtocolRoutingMainGolden(t *testing.T) { + gin.SetMode(gin.TestMode) + data, err := json.MarshalIndent(captureRoutingMatrixGolden(), "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(os.Getenv("ROUTING_MATRIX_RECORD_OUTPUT"), append(data, '\n'), 0o644)) +} +''') + output = root / "routing_main_golden.json" + environment = dict(os.environ, ROUTING_MATRIX_RECORD_OUTPUT=str(output)) + subprocess.run( + ["go", "test", "-tags", "unit", "./internal/service/", "-run", "^TestRecordUpstreamProtocolRoutingMainGolden$", "-count=1"], + cwd=root / "backend", env=environment, check=True, + ) + data = output.read_bytes() + fixture = json.loads(data) + if fixture["source_revision"] != revision: + raise RuntimeError("Recorded source revision differs from the pinned revision") + destination = driver.parent / "testdata/upstream_protocol_routing_main_golden.json" + destination.write_bytes(format_golden(fixture)) + print(f"Recorded {fixture['case_count']} cases from {revision} into {destination}") + + +if __name__ == "__main__": + main() diff --git a/backend/internal/service/testdata/upstream_protocol_routing_main_golden.json b/backend/internal/service/testdata/upstream_protocol_routing_main_golden.json new file mode 100644 index 000000000000..6db675bb42cd --- /dev/null +++ b/backend/internal/service/testdata/upstream_protocol_routing_main_golden.json @@ -0,0 +1,3931 @@ +{ + "source_revision": "5fc0e486c3f6a8a191b8bd140f39b60457f611cf", + "case_count": 63600, + "case_keys_sha256": "639577ac93cdf9b12c3ae7070ff272d44415fdd0a03de6b783ae70bb53c21862", + "observations": [ + { + "upstream": [ + { + "url": "https://api.moonshot.cn/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.moonshot.cn/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "error": "unsupported account type: upstream", + "status": 200 + }, + { + "error": "get access token: unsupported account type: upstream", + "status": 200 + }, + { + "error": "access_token not found in credentials", + "status": 200 + }, + { + "error": "get access token: access_token not found in credentials", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.moonshot.cn/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.moonshot.cn/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.moonshot.cn/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.moonshot.cn/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "error": "account 801 missing api_key", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.kimi.com/coding/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "kimi-k2" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " kimi-k2 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/paas/v4/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/paas/v4/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/coding/paas/v4/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/coding/paas/v4/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://open.bigmodel.cn/api/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " glm-4.7 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " glm-4.7 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-4.7" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " glm-4.7 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.deepseek.com/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "deepseek-chat" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " deepseek-chat " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.minimaxi.com/anthropic/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " minimax-m3 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.openai.com/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.openai.com/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " gpt-5 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " gpt-5 " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.openai.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.openai.com/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "error": "opencode unsupported model: gemini-3.8-flash", + "status": 400, + "client_error_type": "invalid_request_error", + "client_error_code": "model_not_supported", + "client_error_message": "Model 'gemini-3.8-flash' is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)" + }, + { + "error": "opencode unsupported model: jev-1.13", + "status": 400, + "client_error_type": "invalid_request_error", + "client_error_code": "model_not_supported", + "client_error_message": "Model 'jev-1.13' is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)" + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "error": "opencode unsupported model: gemini-3.8-flash", + "status": 400, + "client_error_type": "invalid_request_error", + "client_error_message": "Model 'gemini-3.8-flash' is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)" + }, + { + "error": "opencode unsupported model: jev-1.13", + "status": 400, + "client_error_type": "invalid_request_error", + "client_error_message": "Model 'jev-1.13' is not supported on OpenCode standard gateway (gemini models require Google SDK endpoint, jev models require System One endpoint)" + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://responses.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://cc.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://anthropic.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/chat/completions", + "authorization": "Bearer sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/messages", + "api_key": "sk-test", + "body_kind": "messages", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/go/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": " opencode/gpt-5.6-luna " + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "opencode/gpt-5.6-luna" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "minimax-m3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "glm-5.3" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "claude-sonnet-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-max" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://opencode.ai/zen/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "qwen3.8-flash" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "https://api.x.ai/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "grok-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "upstream": [ + { + "url": "http://base.example/v1/responses", + "authorization": "Bearer sk-test", + "body_kind": "input", + "model": "grok-4" + } + ], + "error": "upstream error: 502 (failover)", + "status": 200 + }, + { + "error": "get access token: credential failure: grok_oauth_provider_config (failover)", + "status": 200 + } + ], + "results": [ + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 20, 21, 8, 8, 20, 20, 22, 23, 20, 21, 2, 2, 20, 20, 22, 23, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 20, 21, 8, 8, 20, 20, 22, 23, 20, 21, 2, 2, 20, 20, 22, 23, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 20, 21, 8, 8, 20, 20, 22, 23, 20, 21, 2, 2, 20, 20, 22, 23, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 10, 11, 0, 0, 10, 10, 12, 13, 10, 11, 2, 2, 10, 10, 12, 13, 14, 15, 16, 16, 14, 14, 17, 18, 14, 15, 16, 16, 14, 14, 17, 18, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, + 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, + 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, 22, 22, 22, 22, 22, 22, 22, 23, 24, 24, 24, 24, 24, 24, 24, 25, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, 12, 12, 12, 12, 12, 12, 12, 13, 24, 24, 24, 24, 24, 24, 24, 25, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, + 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, + 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, 20, 21, 20, 20, 20, 20, 20, 20, 26, 27, 26, 26, 26, 26, 26, 26, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, 10, 11, 10, 10, 10, 10, 10, 10, 26, 27, 26, 26, 26, 26, 26, 26, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, 8, 8, 8, 8, 9, 9, 8, 8, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 2, 2, 2, 3, 3, 2, 2, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 32, 32, 32, 32, 32, 32, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 32, 32, 32, 32, 32, 32, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 32, 32, 32, 32, 32, 32, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 28, 28, 28, 28, 28, 28, 34, 35, 30, 30, 30, 30, 30, 30, 34, 35, 36, 36, 36, 36, 36, 36, 37, 38, 36, 36, 36, 36, 36, 36, 37, 38, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, 4, 4, 4, 4, 4, 4, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, 6, 6, 6, 6, 6, 6, 19, 19, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, 34, 34, 34, 34, 34, 34, 34, 35, 39, 39, 39, 39, 39, 39, 39, 40, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, 32, 32, 32, 32, 33, 33, 32, 32, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, 28, 28, 28, 28, 29, 29, 28, 28, 30, 30, 30, 30, 31, 31, 30, 30, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 45, 46, 41, 41, 45, 45, 47, 48, 45, 46, 43, 43, 45, 45, 47, 48, 49, 50, 51, 51, 49, 49, 52, 53, 49, 50, 51, 51, 49, 49, 52, 53, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, 47, 47, 47, 47, 47, 47, 47, 48, 54, 54, 54, 54, 54, 54, 54, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, 45, 46, 45, 45, 45, 45, 45, 45, 56, 57, 56, 56, 56, 56, 56, 56, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, 41, 41, 41, 41, 42, 42, 41, 41, 43, 43, 43, 43, 44, 44, 43, 43, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 62, 63, 58, 58, 62, 62, 64, 65, 62, 63, 60, 60, 62, 62, 64, 65, 66, 67, 68, 68, 66, 66, 69, 70, 66, 67, 68, 68, 66, 66, 69, 70, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, 4, 4, 4, 4, 5, 5, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, 6, 6, 6, 6, 7, 7, 19, 19, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, 64, 64, 64, 64, 64, 64, 64, 65, 71, 71, 71, 71, 71, 71, 71, 72, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, 62, 63, 62, 62, 62, 62, 62, 62, 73, 74, 73, 73, 73, 73, 73, 73, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, 58, 58, 58, 58, 59, 59, 58, 58, 60, 60, 60, 60, 61, 61, 60, 60, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, 75, 76, 75, 75, 75, 75, 75, 75, 77, 78, 77, 77, 77, 77, 77, 77, + 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, 79, 79, 79, 79, 80, 80, 79, 79, 81, 81, 81, 81, 82, 82, 81, 81, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, + 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, + 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, + 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 129, 114, 115, 122, + 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, + 122, 117, 123, 93, 94, 130, 129, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, + 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, + 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, + 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 93, 94, 130, 129, 93, 129, 114, 115, 122, + 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, 122, 117, 123, 89, 90, 130, 129, 89, 129, 114, 115, + 122, 117, 123, 93, 94, 130, 129, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, + 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, + 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, + 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, + 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, + 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, + 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 93, 94, 126, 125, 93, 125, 95, 85, 86, + 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, 86, 97, 98, 89, 90, 126, 125, 89, 125, 95, 85, + 86, 97, 98, 93, 94, 126, 125, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, + 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, + 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 89, 90, 128, 127, 89, 127, 60, 99, 100, 101, 102, 93, 94, 128, 127, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, + 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, + 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, + 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, + 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 4, + 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, + 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, + 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, + 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 142, 143, 121, 116, + 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, + 116, 144, 118, 93, 94, 146, 142, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, + 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, + 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, + 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 93, 94, 146, 142, 93, 142, 143, 121, 116, + 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, 116, 144, 118, 89, 90, 145, 142, 89, 142, 143, 121, + 116, 144, 118, 93, 94, 146, 142, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, + 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, + 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, + 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, + 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, + 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, + 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 93, 94, 134, 131, 93, 131, 84, 96, 132, + 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, 132, 87, 88, 89, 90, 133, 131, 89, 131, 84, 96, + 132, 87, 88, 93, 94, 134, 131, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, + 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, + 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 89, 90, 140, 135, 89, 135, 71, 136, 137, 138, 139, 93, 94, 141, 135, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, + 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, + 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, + 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, + 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, + 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, + 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, + 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, + 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, + 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, + 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 89, 90, 19, 19, 89, 19, 19, 19, 19, 19, 19, 93, 94, 19, 19, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, + 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, + 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, + 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 113, 159, 160, 161, + 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, + 161, 162, 163, 93, 94, 120, 113, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, + 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, + 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 113, 159, 160, 161, 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, + 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 93, 94, 120, 113, 93, 113, 159, 160, 161, + 162, 163, 89, 90, 119, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, 161, 162, 163, 89, 90, 120, 113, 89, 113, 159, 160, + 161, 162, 163, 93, 94, 120, 113, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, + 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, + 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, + 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, + 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, + 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 83, 147, 148, 149, 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, + 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 93, 94, 92, 83, 93, 83, 147, 148, 149, + 150, 151, 89, 90, 91, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, 149, 150, 151, 89, 90, 92, 83, 89, 83, 147, 148, + 149, 150, 151, 93, 94, 92, 83, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, + 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, 152, 73, 153, 154, 155, 156, 89, 90, 157, 152, 89, 152, + 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 89, 90, 158, 152, 89, 152, 73, 153, 154, 155, 156, 93, 94, 158, 152, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, + 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, + 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, + 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, + 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, + 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, + 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, + 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, + 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, + 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, 4, 4, 4, 4, 4, 4, 89, 90, 4, 4, 89, 5, + 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 89, 90, 5, 5, 89, 5, 5, 5, 5, 5, 5, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 113, 114, 115, 116, 117, 118, 89, 90, 119, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, 89, 113, 114, 115, 116, 117, 118, 89, 90, 120, 113, + 89, 113, 114, 115, 116, 117, 118, 93, 94, 120, 113, 93, 113, 114, 121, 122, 117, 123, 89, 90, 119, 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, + 113, 89, 113, 114, 121, 122, 117, 123, 89, 90, 120, 113, 89, 113, 114, 121, 122, 117, 123, 93, 94, 120, 113, 93, 113, 60, 99, 116, 101, 118, 89, 90, + 119, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 89, 90, 120, 113, 89, 113, 60, 99, 116, 101, 118, 93, + 94, 120, 113, 93, 113, 60, 121, 100, 101, 102, 89, 90, 119, 113, 89, 113, 60, 121, 100, 101, 102, 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, + 89, 90, 120, 113, 89, 113, 60, 121, 100, 101, 102, 93, 94, 120, 113, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, + 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 68, 104, 124, 111, 107, 89, 90, 108, 103, 89, 103, 68, 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, + 104, 124, 111, 107, 89, 90, 109, 103, 89, 103, 68, 104, 124, 111, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, + 19, 89, 90, 5, 5, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 4, + 4, 19, 4, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, 89, 83, 84, 85, 86, 87, 88, 89, 90, 92, 83, + 89, 83, 84, 85, 86, 87, 88, 93, 94, 92, 83, 93, 83, 95, 96, 86, 97, 98, 89, 90, 91, 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, + 83, 89, 83, 95, 96, 86, 97, 98, 89, 90, 92, 83, 89, 83, 95, 96, 86, 97, 98, 93, 94, 92, 83, 93, 83, 84, 99, 100, 87, 88, 89, 90, + 91, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 89, 90, 92, 83, 89, 83, 84, 99, 100, 87, 88, 93, + 94, 92, 83, 93, 83, 60, 96, 100, 101, 102, 89, 90, 91, 83, 89, 83, 60, 96, 100, 101, 102, 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, + 89, 90, 92, 83, 89, 83, 60, 96, 100, 101, 102, 93, 94, 92, 83, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, + 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, + 111, 112, 89, 90, 108, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, + 105, 111, 112, 93, 94, 109, 103, 93, 103, 69, 104, 105, 106, 107, 89, 90, 108, 103, 89, 103, 69, 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, + 104, 105, 106, 107, 89, 90, 109, 103, 89, 103, 69, 104, 105, 106, 107, 93, 94, 109, 103, 93, 103, 68, 110, 105, 111, 112, 89, 90, 108, 103, 89, 103, + 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 89, 90, 109, 103, 89, 103, 68, 110, 105, 111, 112, 93, 94, 109, 103, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, + 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, + 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, + 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, + 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, + 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, + 19, 89, 90, 5, 5, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, + 19, 19, 89, 90, 4, 4, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, + 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, 19, 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, + 4, 4, 19, 19, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, 4, 19, 19, 19, 19, 19, 89, 90, 4, 4, 89, 5, + 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 4, 19, 4, 4, 4, 89, 90, 5, 5, 89, 5, 19, 19, 19, 19, 19, 93, 94, 5, 5, 93, + 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 19, 19, 19, 19, 19, 19, 19, 19, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, + 166, 166, 166, 166, 166, 166, 166, 166, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 19, 19, 19, 19, 19, 19, 19, 19, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 166, 166, 166, 166, 166, 166, 166, 166, + 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, + 19, 19, 19, 19, 19, 19, 19, 19, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 164, 164, 165, 165, 164, 164, 165, 165, + 164, 164, 165, 165, 164, 164, 165, 165, 19, 19, 19, 19, 19, 19, 19, 19, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, + 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 19, 19, 19, 19, 19, 19, 19, 19, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, + 166, 166, 166, 166, 166, 166, 166, 166, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 164, 164, 165, 165, 19, 19, 19, 19, 19, 19, 19, 19, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 166, 166, 166, 166, 166, 166, 166, 166, + 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166 + ] +} diff --git a/backend/internal/service/upstream_billing_probe.go b/backend/internal/service/upstream_billing_probe.go index adf3d25277fb..d9a769e3c48a 100644 --- a/backend/internal/service/upstream_billing_probe.go +++ b/backend/internal/service/upstream_billing_probe.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/Wei-Shaw/sub2api/internal/domain" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" @@ -626,7 +627,7 @@ func (s *UpstreamBillingProbeService) probeLoadedAccount(ctx context.Context, ac return s.persistProbeFailure(ctx, account, intervalMinutes, now, 0, "missing_api_key", 0) } baseURL := account.GetCredential("base_url") - if account.IsCNProvider() && account.IsAdaptiveAPIProtocol() { + if account.RoutesProtocolByInbound() && account.IsAdaptiveAPIProtocol() { baseURL = account.GetCNProtocolBaseURL(APIProtocolChatCompletions) } if account.Platform == PlatformOpenAI { @@ -1008,17 +1009,7 @@ func decodeUpstreamBillingProbeSnapshot(extra map[string]any) *UpstreamBillingPr // type=apikey by the admin form, so only pre-existing type=upstream rows // cannot turn the probe on. func IsUpstreamBillingProbeIdentity(platform, accountType string) bool { - if accountType != AccountTypeAPIKey { - return false - } - switch platform { - case PlatformOpenAI, PlatformAnthropic, PlatformGemini, PlatformAntigravity, PlatformGrok, - PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax, PlatformOpenCodeGo, - PlatformTypeSafe: - return true - default: - return false - } + return accountType == AccountTypeAPIKey && domain.IsConcretePlatform(platform) } func isUpstreamBillingProbeAccount(account *Account) bool { @@ -1054,6 +1045,8 @@ var upstreamBillingProbeOfficialAPIDomains = []string{ "deepseek.com", "opencode.ai", "typesafe.ai", + "commandcode.ai", + "cline.bot", } func upstreamBillingProbeTargetIsOfficialAPI(baseURL string) bool { @@ -1105,6 +1098,12 @@ func (s *UpstreamBillingProbeService) currentTime() time.Time { } func nextProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time.Duration { + if retryAfterDuration < 0 { + retryAfterDuration = 0 + } + if retryAfterDuration > upstreamBillingProbeMaxDelay { + retryAfterDuration = upstreamBillingProbeMaxDelay + } interval := time.Duration(intervalMinutes) * time.Minute if interval < upstreamBillingProbeMinIntervalMinutes*time.Minute { interval = upstreamBillingProbeMinIntervalMinutes * time.Minute @@ -1120,8 +1119,8 @@ func nextProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time. interval += time.Duration(rand.Int64N(int64(jitterRange)*2+1)) - jitterRange } if retryAfterDuration > interval { - // Retry-After is an explicit upstream instruction; do not shorten it - // with the local maximum delay. + // Retry-After is an explicit upstream instruction; honor it over the + // local backoff interval, clamped to the same 24h ceiling. return retryAfterDuration } if interval > upstreamBillingProbeMaxDelay { @@ -1133,8 +1132,7 @@ func nextProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time. // unsupportedProbeDelay 拉长 unsupported 账号的重探间隔,让无效候选自然退出 // 热队列,不再和真正接入 sub2api 的中转账号抢每周期的探测名额。 // 仍按 upstreamBillingProbeMaxDelay 封顶,保证上游后来接入 sub2api 时最迟一天 -// 内会被重新发现;base 本身已达上限(例如 Retry-After 明确要求更久)时原样返回, -// 不缩短上游指令。 +// 内会被重新发现;base 本身亦受 24h 封顶保护。 func unsupportedProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time.Duration { base := nextProbeDelay(intervalMinutes, retryAfterDuration) if base >= upstreamBillingProbeMaxDelay { diff --git a/backend/internal/service/upstream_billing_probe_test.go b/backend/internal/service/upstream_billing_probe_test.go index 9a5f774f4f77..68df096efe65 100644 --- a/backend/internal/service/upstream_billing_probe_test.go +++ b/backend/internal/service/upstream_billing_probe_test.go @@ -767,13 +767,13 @@ func TestUpstreamBillingProbeFailurePreservesLastSuccessAndRetryAfter(t *testing require.Equal(t, initialRate, *account.RateMultiplier) } -func TestUpstreamBillingProbeRetryAfterIsNotShortened(t *testing.T) { +func TestUpstreamBillingProbeRetryAfterIsCappedAtMaxDelay(t *testing.T) { delay := nextProbeDelay(30, 48*time.Hour) - require.Equal(t, 48*time.Hour, delay) + require.Equal(t, upstreamBillingProbeMaxDelay, delay) } // unsupported 的重探间隔明显长于普通失败,但始终有上界:上游后来接入 sub2api -// 时最迟一天内会被重新发现,且不会缩短上游 Retry-After 指令。 +// 时最迟一天内会被重新发现,且上游超长 Retry-After 亦受 24h 封顶保护。 func TestUpstreamBillingProbeUnsupportedDelayIsStretchedAndBounded(t *testing.T) { // 默认 30 分钟 interval:普通失败 24~36 分钟,unsupported 为其 8 倍。 stretched := unsupportedProbeDelay(30, 0) @@ -785,8 +785,8 @@ func TestUpstreamBillingProbeUnsupportedDelayIsStretchedAndBounded(t *testing.T) require.LessOrEqual(t, unsupportedProbeDelay(upstreamBillingProbeMaxIntervalMinutes, 0), upstreamBillingProbeMaxDelay) require.Positive(t, unsupportedProbeDelay(upstreamBillingProbeMinIntervalMinutes, 0)) - // Retry-After 更长时原样保留,不被封顶缩短;更短时至少不早于该指令。 - require.Equal(t, 48*time.Hour, unsupportedProbeDelay(30, 48*time.Hour)) + // Retry-After 更长时受封顶保护,不超过 upstreamBillingProbeMaxDelay;更短时至少不早于该指令。 + require.Equal(t, upstreamBillingProbeMaxDelay, unsupportedProbeDelay(30, 48*time.Hour)) require.GreaterOrEqual(t, unsupportedProbeDelay(30, time.Hour), time.Hour) } diff --git a/backend/internal/service/upstream_models.go b/backend/internal/service/upstream_models.go index c57266bfdbb8..c4d865226ade 100644 --- a/backend/internal/service/upstream_models.go +++ b/backend/internal/service/upstream_models.go @@ -835,7 +835,7 @@ func upstreamModelRegistryBaseURL(account *Account) string { return "" } switch { - case account.IsOpenAI() || account.IsCNProvider() || account.IsOpenCodeGo(): + case account.IsOpenAI() || account.IsMultiProtocolAPIKey(): return account.GetOpenAIFormatBaseURL() case account.IsGrok(): return account.GetGrokBaseURL() @@ -1099,9 +1099,8 @@ func (s *AccountTestService) buildUpstreamModelsRequest(ctx context.Context, acc return s.buildAntigravityAPIKeyModelsRequest(ctx, account) case account.IsGrok(): return s.buildGrokUpstreamModelsRequest(ctx, account) - case account.IsOpenAI() || account.IsCNProvider() || account.IsOpenCodeGo(): - // 国产 OpenAI 兼容供应商(kimi/zhipu/deepseek)与 OpenCode Go - // 复用 OpenAI /v1/models 探测。 + case account.IsOpenAI() || account.IsMultiProtocolAPIKey(): + // 多协议 API Key 供应商(国产厂商与聚合平台)复用 OpenAI /v1/models 探测。 return s.buildOpenAIUpstreamModelsRequest(ctx, account) case account.IsGemini(): return s.buildGeminiUpstreamModelsRequest(ctx, account) diff --git a/backend/internal/service/upstream_protocol_routing.go b/backend/internal/service/upstream_protocol_routing.go new file mode 100644 index 000000000000..dff4cb99054c --- /dev/null +++ b/backend/internal/service/upstream_protocol_routing.go @@ -0,0 +1,139 @@ +package service + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" +) + +// resolveUpstreamProtocol 决定 OpenAI 网关本次请求发往上游的协议 +// (chat_completions / responses / anthropic),是 /v1/responses、 +// /v1/chat/completions、/v1/messages 三个入站入口共用的唯一判定。 +// 只做判断,不改写请求体、不发请求;各入口再按「入站 × 上游」选择转换路径。 +// +// inbound 为入站协议;/v1/chat/completions 收到 Responses 形状请求体时按 +// responses 计。model 仅按模型分流的账号使用,取 upstreamRoutingModel 的结果。 +// catalog 为上游模型目录给出的该模型支持的协议(见 model_protocol_catalog.go),未知时为 nil。 +// +// 优先级: +// 1. 按模型分流的供应商(多模型聚合平台,如 OpenCode、Command Code):显式 +// api_protocol 直接生效;否则先得到该模型支持的协议集合(见 modelProtocolSet), +// 入站协议在集合中时同协议直通,不在时走集合首项(首选协议)。 +// 2. adaptive(按入站协议分流):供应商提供入站协议的原生端点则走同协议, +// 否则回落 Chat Completions。 +// 3. 显式 anthropic 协议 → Anthropic。 +// 4. shouldForwardOpenAIResponsesViaRawChatCompletions(显式 chat_completions、 +// 探针确认不支持 Responses 等)→ Chat Completions。 +// 5. 其余 → Responses。 +// +// Grok 在 Responses / Chat Completions 入口有专属链路,须在调用本函数前分流。 +func resolveUpstreamProtocol(account *Account, inbound, model string, catalog []string) string { + if account.routesByModel() { + return account.resolveModelRoutedProtocolFor(model, inbound, catalog) + } + if account.IsAdaptiveAPIProtocol() { + if account.providerSupportsProtocol(inbound) { + return inbound + } + return APIProtocolChatCompletions + } + if account.IsAnthropicProtocol() { + return APIProtocolAnthropic + } + if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { + return APIProtocolChatCompletions + } + return APIProtocolResponses +} + +// resolveModelRoutedProtocol 返回按模型分流账号对该模型的首选上游协议(不考虑入站 +// 协议与上游模型目录),供连接测试等没有入站协议的场景使用。 +func (a *Account) resolveModelRoutedProtocol(model string) string { + return a.resolveModelRoutedProtocolFor(model, "", nil) +} + +// resolveModelRoutedProtocolFor 按模型名解析按模型分流账号的上游协议:显式 pinned +// 协议直接生效;否则取该模型支持的协议集合,入站协议在其中时同协议直通(省去一次 +// 协议转换),不在时走首选协议。 +func (a *Account) resolveModelRoutedProtocolFor(model, inbound string, catalog []string) string { + switch proto := a.GetAPIProtocol(); proto { + case APIProtocolChatCompletions, APIProtocolAnthropic, APIProtocolResponses: + return proto + } + set := a.modelProtocolSet(model, catalog) + if slices.Contains(set, inbound) { + return inbound + } + return set[0] +} + +// modelProtocolSet 返回模型支持的上游协议集合(首项为首选),来源依次为: +// 1. 账号 protocol_rules 中命中的规则(管理员显式配置); +// 2. 上游模型目录(catalog):首选沿用内置规则对该模型的首选(在集合内时),否则取目录首项; +// 3. profile 内置规则——仅在账号未配置 protocol_rules 时,已配置但未命中视为有意不走内置规则; +// 4. Chat Completions,避免落入 Responses 转换链。 +func (a *Account) modelProtocolSet(model string, catalog []string) []string { + rules, configured := a.configuredProtocolRules() + if configured { + if set, ok := matchProtocolRuleSet(model, rules); ok { + return set + } + } + profileRules := a.providerProfile().Endpoints(a.GetCredential("account_mode")).ProtocolRules + if set := a.supportedCatalogProtocols(catalog); len(set) > 0 { + preferred := matchProtocolRules(model, profileRules) + if slices.Contains(set, preferred) { + return normalizeProtocolSet(preferred, set) + } + return set + } + if !configured { + if set, ok := matchProtocolRuleSet(model, profileRules); ok { + return set + } + } + return []string{APIProtocolChatCompletions} +} + +// supportedCatalogProtocols 过滤出账号接入模式确有原生端点的目录协议。 +func (a *Account) supportedCatalogProtocols(catalog []string) []string { + out := make([]string, 0, len(catalog)) + for _, protocol := range catalog { + if isNativeUpstreamProtocol(protocol) && a.providerSupportsProtocol(protocol) && !slices.Contains(out, protocol) { + out = append(out, protocol) + } + } + return out +} + +// upstreamRoutingModel 返回参与协议分流的模型名:仅按模型分流的账号需要,按账号 +// 模型映射与上游归一后的名字匹配规则;其余账号的分流与模型无关,返回空串。 +func upstreamRoutingModel(account *Account, body []byte, defaultMappedModel string) string { + if !account.routesByModel() { + return "" + } + return resolveMappedUpstreamModel(account, body, defaultMappedModel) +} + +// convertResponsesShapedChatBody 把发到 /v1/chat/completions 的 Responses 形状 +// 请求体(Cursor 兼容)转换为 Chat Completions 请求体。 +func (s *OpenAIGatewayService) convertResponsesShapedChatBody(body []byte) ([]byte, error) { + var responsesReq apicompat.ResponsesRequest + if err := json.Unmarshal(body, &responsesReq); err != nil { + return nil, fmt.Errorf("parse responses-shaped chat completions request: %w", err) + } + chatReq, err := apicompat.ResponsesToChatCompletionsRequestWithOptions( + &responsesReq, + &apicompat.ResponsesToChatOptions{ReasoningContentByID: s.reasoningContentByID}, + ) + if err != nil { + return nil, fmt.Errorf("convert responses-shaped chat completions request: %w", err) + } + chatBody, err := json.Marshal(chatReq) + if err != nil { + return nil, fmt.Errorf("marshal converted chat completions request: %w", err) + } + return chatBody, nil +} diff --git a/backend/internal/service/upstream_protocol_routing_matrix_test.go b/backend/internal/service/upstream_protocol_routing_matrix_test.go new file mode 100644 index 000000000000..2a9a3d2ffeca --- /dev/null +++ b/backend/internal/service/upstream_protocol_routing_matrix_test.go @@ -0,0 +1,350 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/openai_compat" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// The fixture is captured through the real gateway entry points on this main +// revision, before provider profiles and resolveUpstreamProtocol were added. +// Expected results never call routing helpers from the implementation under test. +// Regenerate with python3 testdata/record_upstream_protocol_routing_main.py, which exports +// this pinned revision before compiling and recording the same matrix driver. +const routingMatrixMainRevision = "5fc0e486c3f6a8a191b8bd140f39b60457f611cf" +const routingMatrixGoldenPath = "testdata/upstream_protocol_routing_main_golden.json" + +type routingMatrixIngress struct { + name string + path string + responsesBody bool + forward func(*OpenAIGatewayService, *gin.Context, *Account, []byte) error +} + +func routingMatrixIngresses() []routingMatrixIngress { + return []routingMatrixIngress{ + { + name: "responses", path: "/v1/responses", responsesBody: true, + forward: func(svc *OpenAIGatewayService, c *gin.Context, account *Account, body []byte) error { + _, err := svc.Forward(context.Background(), c, account, body) + return err + }, + }, + { + name: "chat", path: "/v1/chat/completions", + forward: func(svc *OpenAIGatewayService, c *gin.Context, account *Account, body []byte) error { + _, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + return err + }, + }, + { + name: "chat_responses_shape", path: "/v1/chat/completions", responsesBody: true, + forward: func(svc *OpenAIGatewayService, c *gin.Context, account *Account, body []byte) error { + _, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + return err + }, + }, + { + name: "messages", path: "/v1/messages", + forward: func(svc *OpenAIGatewayService, c *gin.Context, account *Account, body []byte) error { + _, err := svc.ForwardAsAnthropic(context.Background(), c, account, body, "", "") + return err + }, + }, + } +} + +type routingMatrixCase struct { + platform string + apiProtocol string + mode string + accountType string + probe string + addresses string + rules bool + ingress routingMatrixIngress + model string +} + +func (tc routingMatrixCase) key() string { + return fmt.Sprintf("%s|proto=%s|mode=%s|type=%s|probe=%s|addresses=%s|rules=%t|%s|%s", + tc.platform, tc.apiProtocol, tc.mode, tc.accountType, tc.probe, tc.addresses, tc.rules, tc.ingress.name, tc.model) +} + +func (tc routingMatrixCase) account() *Account { + credentials := map[string]any{"api_key": "sk-test"} + if tc.addresses == "base" || tc.addresses == "both" { + credentials["base_url"] = "http://base.example" + } + if tc.addresses == "split" || tc.addresses == "both" { + credentials["api_base_urls"] = map[string]any{ + APIProtocolChatCompletions: "http://cc.example", + APIProtocolResponses: "http://responses.example", + APIProtocolAnthropic: "http://anthropic.example", + } + } + if tc.apiProtocol != "" { + credentials["api_protocol"] = tc.apiProtocol + } + if tc.mode != "" { + credentials["account_mode"] = tc.mode + } + if tc.rules { + credentials["protocol_rules"] = []any{ + map[string]any{"pattern": "glm-*", "protocol": APIProtocolAnthropic}, + map[string]any{"pattern": "gpt-*", "protocol": APIProtocolResponses}, + } + } + if tc.platform == PlatformOpenCodeGo { + credentials["model_mapping"] = map[string]any{ + "mapped-gpt": "gpt-5.6-luna", + "mapped-unsupported": "gemini-3.8-flash", + } + } + extra := map[string]any{} + switch tc.probe { + case "yes": + extra[openai_compat.ExtraKeyResponsesSupported] = true + case "no": + extra[openai_compat.ExtraKeyResponsesSupported] = false + } + return &Account{ + ID: 801, Name: "routing-matrix", Platform: tc.platform, Type: tc.accountType, + Concurrency: 1, Credentials: credentials, Extra: extra, + } +} + +func (tc routingMatrixCase) body() []byte { + switch { + case tc.ingress.responsesBody: + return []byte(fmt.Sprintf(`{"model":%q,"input":"hello","max_output_tokens":32,"stream":false}`, tc.model)) + case tc.ingress.name == "messages": + return []byte(fmt.Sprintf(`{"model":%q,"max_tokens":32,"messages":[{"role":"user","content":"hello"}],"stream":false}`, tc.model)) + default: + return []byte(fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"hello"}],"stream":false}`, tc.model)) + } +} + +func routingMatrixCases() []routingMatrixCase { + var cases []routingMatrixCase + protocols := []string{"", APIProtocolAdaptive, APIProtocolChatCompletions, APIProtocolAnthropic, APIProtocolResponses, "invalid"} + types := []string{AccountTypeAPIKey, AccountTypeUpstream, AccountTypeOAuth} + probes := []string{"", "yes", "no"} + addresses := []string{"none", "base", "split", "both"} + add := func(platform string, modes, models []string, rulesOptions []bool, probeOptions, typeOptions []string, ingresses []routingMatrixIngress) { + for _, proto := range protocols { + for _, mode := range modes { + for _, accountType := range typeOptions { + for _, probe := range probeOptions { + for _, address := range addresses { + for _, rules := range rulesOptions { + for _, ingress := range ingresses { + for _, model := range models { + cases = append(cases, routingMatrixCase{ + platform: platform, apiProtocol: proto, mode: mode, accountType: accountType, + probe: probe, addresses: address, rules: rules, ingress: ingress, model: model, + }) + } + } + } + } + } + } + } + } + } + all := routingMatrixIngresses() + modes := []string{"", AccountModePayG, AccountModeCoding, "invalid"} + for _, provider := range []struct{ platform, model string }{ + {PlatformKimi, "kimi-k2"}, {PlatformZhipu, "glm-4.7"}, + {PlatformDeepseek, "deepseek-chat"}, {PlatformMiniMax, "minimax-m3"}, + } { + add(provider.platform, modes, []string{provider.model, " " + provider.model + " "}, []bool{false}, probes, types, all) + } + add(PlatformOpenAI, []string{""}, []string{"gpt-5", " gpt-5 "}, []bool{false}, probes, types, all) + add(PlatformOpenCodeGo, []string{"", AccountModeGo, AccountModeZen, "invalid"}, + []string{"gpt-5.6-luna", "minimax-m3", "glm-5.3", "claude-sonnet-4", "qwen3.8-max", "qwen3.8-flash", + "gemini-3.8-flash", "jev-1.13", " opencode/gpt-5.6-luna ", "mapped-gpt", "mapped-unsupported"}, + []bool{false, true}, []string{"", "no"}, []string{AccountTypeAPIKey, AccountTypeUpstream}, all) + // Grok has its own Responses and Chat Completions implementations; only the + // Messages entry uses the routing logic covered by this compatibility matrix. + add(PlatformGrok, []string{""}, []string{"grok-4", " grok-4 "}, []bool{false}, probes, types, all[3:]) + return cases +} + +type routingUpstreamRequest struct { + URL string `json:"url"` + Authorization string `json:"authorization,omitempty"` + APIKey string `json:"api_key,omitempty"` + AnthropicVersion string `json:"anthropic_version,omitempty"` + BodyKind string `json:"body_kind"` + Model string `json:"model"` +} + +type routingObservation struct { + Requests []routingUpstreamRequest `json:"upstream,omitempty"` + Error string `json:"error,omitempty"` + Panic string `json:"panic,omitempty"` + Status int `json:"status"` + ClientErrorType string `json:"client_error_type,omitempty"` + ClientErrorCode string `json:"client_error_code,omitempty"` + ClientErrorMessage string `json:"client_error_message,omitempty"` +} + +func (o routingObservation) String() string { + data, _ := json.Marshal(o) + return string(data) +} + +func observeRouting(tc routingMatrixCase) (obs routingObservation) { + upstream := &httpUpstreamRecorder{err: errors.New("stop after capture")} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + body := tc.body() + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, tc.ingress.path, bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + defer func() { + if r := recover(); r != nil { + obs.Panic = fmt.Sprint(r) + } + for i, req := range upstream.requests { + kind := "other" + switch { + case gjson.GetBytes(upstream.bodies[i], "messages").Exists(): + kind = "messages" + case gjson.GetBytes(upstream.bodies[i], "input").Exists(): + kind = "input" + } + obs.Requests = append(obs.Requests, routingUpstreamRequest{ + URL: req.URL.String(), Authorization: req.Header.Get("Authorization"), + APIKey: req.Header.Get("X-Api-Key"), AnthropicVersion: req.Header.Get("Anthropic-Version"), + BodyKind: kind, Model: gjson.GetBytes(upstream.bodies[i], "model").String(), + }) + } + obs.Status = c.Writer.Status() + obs.ClientErrorType = gjson.Get(recorder.Body.String(), "error.type").String() + obs.ClientErrorCode = gjson.Get(recorder.Body.String(), "error.code").String() + obs.ClientErrorMessage = gjson.Get(recorder.Body.String(), "error.message").String() + }() + err := tc.ingress.forward(svc, c, tc.account(), body) + if err != nil { + obs.Error = err.Error() + } + return obs +} + +// Cases share many outcomes. Store each distinct observation once, then one +// observation index per case in routingMatrixCases order. The key digest detects +// any change to case identity or ordering, so matrix edits require a new capture. +type routingMatrixGolden struct { + SourceRevision string `json:"source_revision"` + CaseCount int `json:"case_count"` + CaseKeysSHA256 string `json:"case_keys_sha256"` + Observations []routingObservation `json:"observations"` + Results []int `json:"results"` +} + +func routingMatrixCaseDigest(cases []routingMatrixCase) string { + var keys strings.Builder + for _, tc := range cases { + keys.WriteString(tc.key()) + keys.WriteByte('\n') + } + return fmt.Sprintf("%x", sha256.Sum256([]byte(keys.String()))) +} + +func captureRoutingMatrixGolden() routingMatrixGolden { + cases := routingMatrixCases() + golden := routingMatrixGolden{SourceRevision: routingMatrixMainRevision, CaseCount: len(cases), CaseKeysSHA256: routingMatrixCaseDigest(cases)} + indexByObservation := make(map[string]int) + for _, tc := range cases { + obs := observeRouting(tc) + key := obs.String() + index, ok := indexByObservation[key] + if !ok { + index = len(golden.Observations) + golden.Observations = append(golden.Observations, obs) + indexByObservation[key] = index + } + golden.Results = append(golden.Results, index) + } + return golden +} + +// Zhipu has no Responses endpoint. The refactor routes adaptive /responses to +// Chat Completions even for unsupported upstream/oauth account types. Both +// versions reject these accounts before transport, with these exact errors. +// Only this complete before/after observation is permitted; other no-upstream +// results and new panics still fail the golden comparison. +func matchesKnownZhipuNonAPIKeyDifference(tc routingMatrixCase, want, got routingObservation) bool { + if tc.platform != PlatformZhipu || tc.apiProtocol != APIProtocolAdaptive || tc.ingress.name != "responses" { + return false + } + var oldError string + switch tc.accountType { + case AccountTypeUpstream: + oldError = "unsupported account type: upstream" + case AccountTypeOAuth: + oldError = "access_token not found in credentials" + default: + return false + } + return reflect.DeepEqual(want, routingObservation{Error: oldError, Status: http.StatusOK}) && + reflect.DeepEqual(got, routingObservation{Error: "account 801 missing api_key", Status: http.StatusOK}) +} + +func TestUpstreamProtocolRoutingMatrixMatchesMainGolden(t *testing.T) { + gin.SetMode(gin.TestMode) + data, err := os.ReadFile(routingMatrixGoldenPath) + require.NoError(t, err) + var golden routingMatrixGolden + require.NoError(t, json.Unmarshal(data, &golden)) + cases := routingMatrixCases() + require.Equal(t, routingMatrixMainRevision, golden.SourceRevision) + require.Equal(t, len(cases), golden.CaseCount) + require.Equal(t, routingMatrixCaseDigest(cases), golden.CaseKeysSHA256) + require.Len(t, golden.Results, len(cases)) + + var differences []string + mismatched := 0 + knownDifferences := 0 + for i, tc := range cases { + index := golden.Results[i] + require.GreaterOrEqual(t, index, 0) + require.Less(t, index, len(golden.Observations)) + want := golden.Observations[index] + got := observeRouting(tc) + if reflect.DeepEqual(want, got) { + continue + } + if matchesKnownZhipuNonAPIKeyDifference(tc, want, got) { + knownDifferences++ + continue + } + mismatched++ + if len(differences) < 12 { + differences = append(differences, fmt.Sprintf("%s\nmain: %s\ncurrent: %s", tc.key(), want, got)) + } + } + require.Zero(t, mismatched, "routing differs from main %s in %d cases:\n%s", golden.SourceRevision, mismatched, strings.Join(differences, "\n")) + // 2 account types × 4 modes × 3 probe states × 4 address configurations × + // 2 model spellings. A disappeared or expanded exception requires review. + require.Equal(t, 192, knownDifferences, "unexpected change to the declared Zhipu non-API-Key difference") +} diff --git a/backend/internal/service/upstream_protocol_routing_test.go b/backend/internal/service/upstream_protocol_routing_test.go new file mode 100644 index 000000000000..1572a7551a4d --- /dev/null +++ b/backend/internal/service/upstream_protocol_routing_test.go @@ -0,0 +1,165 @@ +//go:build unit + +package service + +import ( + "context" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/openai_compat" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func routingTestAccount(platform, accountType string, credentials, extra map[string]any) *Account { + if credentials == nil { + credentials = map[string]any{} + } + credentials["api_key"] = "sk-test" + return &Account{Platform: platform, Type: accountType, Credentials: credentials, Extra: extra} +} + +func TestResolveUpstreamProtocol(t *testing.T) { + t.Parallel() + + const ( + cc = APIProtocolChatCompletions + responses = APIProtocolResponses + anthropic = APIProtocolAnthropic + ) + adaptive := func(platform string) *Account { + return routingTestAccount(platform, AccountTypeAPIKey, map[string]any{"api_protocol": APIProtocolAdaptive}, nil) + } + pinned := func(platform, protocol string) *Account { + return routingTestAccount(platform, AccountTypeAPIKey, map[string]any{"api_protocol": protocol}, nil) + } + openCode := func(mode string, credentials map[string]any) *Account { + if credentials == nil { + credentials = map[string]any{} + } + credentials["account_mode"] = mode + return routingTestAccount(PlatformOpenCodeGo, AccountTypeAPIKey, credentials, nil) + } + openAIWithProbe := func(accountType string, supported any) *Account { + extra := map[string]any{} + if supported != nil { + extra[openai_compat.ExtraKeyResponsesSupported] = supported + } + return routingTestAccount(PlatformOpenAI, accountType, nil, extra) + } + + cases := []struct { + name string + account *Account + inbound string + model string + want string + }{ + // 按入站协议分流(国产单一厂商 adaptive) + {"kimi adaptive anthropic inbound", adaptive(PlatformKimi), anthropic, "", anthropic}, + {"kimi adaptive cc inbound", adaptive(PlatformKimi), cc, "", cc}, + {"kimi adaptive responses inbound", adaptive(PlatformKimi), responses, "", responses}, + {"zhipu adaptive responses inbound falls back to cc", adaptive(PlatformZhipu), responses, "", cc}, + {"zhipu adaptive anthropic inbound", adaptive(PlatformZhipu), anthropic, "", anthropic}, + + // 显式固定协议,与入站无关 + {"kimi pinned anthropic", pinned(PlatformKimi, anthropic), cc, "", anthropic}, + {"kimi pinned cc", pinned(PlatformKimi, cc), responses, "", cc}, + {"kimi pinned responses", pinned(PlatformKimi, responses), anthropic, "", responses}, + {"zhipu pinned responses is unsupported and stays cc", pinned(PlatformZhipu, responses), responses, "", cc}, + + // 按模型分流(多模型聚合平台),与入站无关 + {"opencode go gpt", openCode(AccountModeGo, nil), anthropic, "gpt-5.6-luna", responses}, + {"opencode go minimax", openCode(AccountModeGo, nil), responses, "minimax-m3", anthropic}, + {"opencode go glm falls back to cc", openCode(AccountModeGo, nil), anthropic, "glm-5.3", cc}, + {"opencode go claude has no rule", openCode(AccountModeGo, nil), anthropic, "claude-sonnet-4", cc}, + {"opencode zen claude", openCode(AccountModeZen, nil), cc, "claude-sonnet-4", anthropic}, + {"opencode account rules override defaults", openCode(AccountModeGo, map[string]any{ + "protocol_rules": []any{map[string]any{"pattern": "glm-*", "protocol": anthropic}}, + }), cc, "glm-5.3", anthropic}, + {"opencode account rules miss falls back to cc", openCode(AccountModeGo, map[string]any{ + "protocol_rules": []any{map[string]any{"pattern": "glm-*", "protocol": anthropic}}, + }), cc, "gpt-5.6-luna", cc}, + {"opencode pinned protocol beats rules", openCode(AccountModeGo, map[string]any{"api_protocol": cc}), anthropic, "gpt-5.6-luna", cc}, + {"opencode ignores responses probe", func() *Account { + account := openCode(AccountModeGo, nil) + account.Extra = map[string]any{openai_compat.ExtraKeyResponsesSupported: false} + return account + }(), cc, "gpt-5.6-luna", responses}, + + // 无 profile 的账号按探针决定 Responses / CC + {"openai apikey unknown probe", openAIWithProbe(AccountTypeAPIKey, nil), cc, "", responses}, + {"openai apikey probe unsupported", openAIWithProbe(AccountTypeAPIKey, false), anthropic, "", cc}, + {"openai oauth ignores probe", openAIWithProbe(AccountTypeOAuth, false), cc, "", responses}, + } + for _, tc := range cases { + require.Equal(t, tc.want, resolveUpstreamProtocol(tc.account, tc.inbound, tc.model, nil), tc.name) + } +} + +func TestRoutesByModelOnlyForAggregators(t *testing.T) { + t.Parallel() + + aggregators := map[string]bool{PlatformOpenCodeGo: true, PlatformCommandCode: true} + for platform, profile := range providerProfiles { + account := &Account{Platform: platform} + require.Equal(t, aggregators[platform], account.routesByModel(), platform) + require.Equal(t, aggregators[platform], profile.Routing == ProviderRoutingByModel, platform) + } + require.False(t, (&Account{Platform: PlatformOpenAI}).routesByModel()) + require.False(t, (*Account)(nil).routesByModel()) +} + +func TestUpstreamRoutingModelOnlyForModelRoutedAccounts(t *testing.T) { + t.Parallel() + + body := []byte(`{"model":"gpt-5.6-luna","input":"hi"}`) + openCode := routingTestAccount(PlatformOpenCodeGo, AccountTypeAPIKey, map[string]any{ + "model_mapping": map[string]any{"gpt-5.6-luna": "glm-5.3"}, + }, nil) + require.Equal(t, "glm-5.3", upstreamRoutingModel(openCode, body, "")) + require.Empty(t, upstreamRoutingModel(routingTestAccount(PlatformKimi, AccountTypeAPIKey, nil, nil), body, "")) +} + +// /v1/responses 转 Anthropic 上游时,计费模型名与上游模型名都应是去除首尾空白的 +// 请求模型;按模型分流(OpenCode)与显式 anthropic 协议(Kimi)账号行为一致。 +func TestResponsesToNativeAnthropicTrimsBillingModel(t *testing.T) { + gin.SetMode(gin.TestMode) + + openCode := &Account{ + ID: 703, + Name: "opencode-go", + Platform: PlatformOpenCodeGo, + Type: AccountTypeAPIKey, + Concurrency: 1, + Credentials: map[string]any{ + "api_key": "sk-test", + "account_mode": AccountModeGo, + "api_base_urls": map[string]any{ + APIProtocolAnthropic: "http://anthropic.example", + }, + }, + } + for _, tc := range []struct { + name string + account *Account + model string + }{ + {"opencode by model", openCode, "minimax-m3"}, + {"kimi pinned anthropic", nativeAnthropicTestAccount(), "kimi-k2"}, + } { + t.Run(tc.name, func(t *testing.T) { + body := []byte(`{"model":" ` + tc.model + ` ","input":"hello","stream":false}`) + upstream := &httpUpstreamRecorder{resp: nativeAnthropicStreamResponse()} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + + result, err := svc.Forward(context.Background(), adaptiveProtocolTestContext("/v1/responses", body), tc.account, body) + require.NoError(t, err) + require.True(t, strings.HasSuffix(upstream.lastReq.URL.Path, "/v1/messages"), upstream.lastReq.URL.String()) + require.Equal(t, tc.model, gjson.GetBytes(upstream.lastBody, "model").String()) + require.Equal(t, tc.model, result.BillingModel) + }) + } +} diff --git a/backend/internal/service/wire.go b/backend/internal/service/wire.go index f50386bf90e1..a33942243c85 100644 --- a/backend/internal/service/wire.go +++ b/backend/internal/service/wire.go @@ -75,7 +75,6 @@ func ProvideOpenAIGatewayService( settingService *SettingService, userPlatformQuotaRepo UserPlatformQuotaRepository, systemPrompts *SystemPromptService, - reasoningRecovery *ReasoningRecoveryService, ) *OpenAIGatewayService { svc := NewOpenAIGatewayService( accountRepo, usageLogRepo, usageBillingRepo, userRepo, userSubRepo, @@ -85,7 +84,6 @@ func ProvideOpenAIGatewayService( channelService, balanceNotifyService, settingService, userPlatformQuotaRepo, ) svc.SetSystemPromptService(systemPrompts) - svc.SetReasoningRecoveryService(reasoningRecovery) return svc } @@ -1019,7 +1017,6 @@ var ProviderSet = wire.NewSet( NewAdminService, ProvideGatewayService, ProvideSystemPromptService, - ProvideReasoningRecoveryService, ProvideCodexGatewayBorrowService, ProvidePelicanTestRunner, NewCodexGatewayBorrowPreviewService, diff --git a/backend/internal/web/embed_on.go b/backend/internal/web/embed_on.go index 454b3bed51ac..ef589e53902a 100644 --- a/backend/internal/web/embed_on.go +++ b/backend/internal/web/embed_on.go @@ -362,10 +362,26 @@ func shouldBypassEmbeddedFrontend(path string) bool { strings.HasPrefix(trimmed, "/setup/") || trimmed == "/health" || trimmed == "/models" || + strings.HasPrefix(trimmed, "/models/") || trimmed == "/responses" || strings.HasPrefix(trimmed, "/responses/") || + trimmed == "/chat/completions" || + trimmed == "/embeddings" || + trimmed == "/messages/count_tokens" || trimmed == "/alpha/search" || + trimmed == "/web_search" || + trimmed == "/x_search" || + trimmed == "/tts" || + trimmed == "/stt" || + trimmed == "/realtime" || + trimmed == "/custom-voices" || + strings.HasPrefix(trimmed, "/custom-voices/") || + trimmed == "/contents/generations/tasks" || + strings.HasPrefix(trimmed, "/contents/generations/tasks/") || + trimmed == "/v3/contents/generations/tasks" || + strings.HasPrefix(trimmed, "/v3/contents/generations/tasks/") || strings.HasPrefix(trimmed, "/images/") || + trimmed == "/videos" || strings.HasPrefix(trimmed, "/videos/") } diff --git a/backend/internal/web/embed_test.go b/backend/internal/web/embed_test.go index 3eea464c609a..965ee6cb0eb6 100644 --- a/backend/internal/web/embed_test.go +++ b/backend/internal/web/embed_test.go @@ -530,6 +530,9 @@ func TestFrontendServer_Middleware(t *testing.T) { "/health", "/responses", "/responses/compact", + "/chat/completions", + "/models/gpt-5.5", + "/v3/contents/generations/tasks/task-123", } for _, path := range apiPaths { @@ -689,6 +692,40 @@ func TestEmbeddedFrontendBypassesBareVideoAPIRoutes(t *testing.T) { } } +func TestEmbeddedFrontendBypassesBareAPIAliases(t *testing.T) { + for _, path := range []string{ + "/chat/completions", + "/embeddings", + "/messages/count_tokens", + "/models/gpt-5.5", + "/videos", + "/tts", + "/stt", + "/custom-voices", + "/custom-voices/voice-123", + "/custom-voices/voice-123/audio", + "/realtime", + "/web_search", + "/x_search", + "/contents/generations/tasks", + "/contents/generations/tasks/task-123", + "/v3/contents/generations/tasks", + "/v3/contents/generations/tasks/task-123", + } { + require.True(t, shouldBypassEmbeddedFrontend(path), "path=%s", path) + } + + for _, path := range []string{ + "/model-plaza", + "/custom/page-1", + "/monitor", + "/setup", + "/v3/other", + } { + require.False(t, shouldBypassEmbeddedFrontend(path), "path=%s", path) + } +} + func TestNewFrontendServer(t *testing.T) { t.Run("creates_server_successfully", func(t *testing.T) { provider := &mockSettingsProvider{ @@ -792,6 +829,9 @@ func TestServeEmbeddedFrontend(t *testing.T) { "/health", "/responses", "/responses/compact", + "/chat/completions", + "/models/gpt-5.5", + "/v3/contents/generations/tasks/task-123", } for _, path := range apiPaths { diff --git a/backend/migrations/242_drop_platform_check_constraints.sql b/backend/migrations/242_drop_platform_check_constraints.sql new file mode 100644 index 000000000000..7307be715e51 --- /dev/null +++ b/backend/migrations/242_drop_platform_check_constraints.sql @@ -0,0 +1,18 @@ +-- 平台白名单改由应用层的平台清单(internal/domain/platforms.go)统一校验, +-- 新增平台不再需要修改数据库约束的迁移。 +-- +-- 1. user_platform_quotas.platform:admin 接口 / 设置校验 service.AllowedQuotaPlatforms, +-- repository 写入前校验 domain.IsConcretePlatform,ent schema Validate 同源。 +-- 2. composite_model_routes.target_platform:admin 接口 binding、service +-- compositeRouteFromInput 与 ent schema Validate 均校验为已登记的具体平台。 +-- +-- channel_monitors / channel_monitor_request_templates 的 provider CHECK 表示 +-- 渠道监控已实现的探测能力,不属于平台清单,保留不动。 +-- +-- DROP ... IF EXISTS 保证可重入。 + +ALTER TABLE user_platform_quotas + DROP CONSTRAINT IF EXISTS user_platform_quotas_platform_check; + +ALTER TABLE composite_model_routes + DROP CONSTRAINT IF EXISTS composite_model_routes_target_platform_check; diff --git a/backend/migrations/277_drop_legacy_platform_check_constraints.sql b/backend/migrations/277_drop_legacy_platform_check_constraints.sql new file mode 100644 index 000000000000..397e06929054 --- /dev/null +++ b/backend/migrations/277_drop_legacy_platform_check_constraints.sql @@ -0,0 +1,9 @@ +-- Official 242 runs before immutable downstream migrations 251, 260 and 265. +-- Those historical migrations restore the quota whitelist on a fresh install. +-- Finish with the official application-catalog validation contract in both +-- fresh installations and existing databases, without rewriting old checksums. +ALTER TABLE user_platform_quotas + DROP CONSTRAINT IF EXISTS user_platform_quotas_platform_check; + +ALTER TABLE composite_model_routes + DROP CONSTRAINT IF EXISTS composite_model_routes_target_platform_check; diff --git a/backend/migrations/platform_check_constraints_drop_migration_test.go b/backend/migrations/platform_check_constraints_drop_migration_test.go new file mode 100644 index 000000000000..739f66382162 --- /dev/null +++ b/backend/migrations/platform_check_constraints_drop_migration_test.go @@ -0,0 +1,47 @@ +package migrations + +import ( + "io/fs" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDropPlatformCheckConstraintsMigration(t *testing.T) { + content, err := FS.ReadFile("242_drop_platform_check_constraints.sql") + require.NoError(t, err) + + sql := strings.Join(strings.Fields(string(content)), " ") + require.Contains(t, sql, + "ALTER TABLE user_platform_quotas DROP CONSTRAINT IF EXISTS user_platform_quotas_platform_check;") + require.Contains(t, sql, + "ALTER TABLE composite_model_routes DROP CONSTRAINT IF EXISTS composite_model_routes_target_platform_check;") + require.NotContains(t, sql, "ADD CONSTRAINT") + require.NotContains(t, sql, "DROP CONSTRAINT IF EXISTS channel_monitors_provider_check") + require.NotContains(t, sql, "DROP CONSTRAINT IF EXISTS channel_monitor_request_templates_provider_check") +} + +// Immutable downstream migrations 251, 260 and 265 predate this integration. +// Check the final action across the complete migration sequence, including 277, +// so fresh installs cannot finish with a restored database whitelist. +func TestLaterMigrationsDoNotRestorePlatformCheckConstraints(t *testing.T) { + entries, err := fs.ReadDir(FS, ".") + require.NoError(t, err) + comments := regexp.MustCompile(`(?ms)/\*.*?\*/|--[^\n]*`) + constraintAction := regexp.MustCompile(`(?i)\b(ADD|DROP)\s+CONSTRAINT(?:\s+IF\s+EXISTS)?\s+"?(user_platform_quotas_platform_check|composite_model_routes_target_platform_check)\b`) + lastAction := map[string]string{} + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") { + continue + } + content, err := FS.ReadFile(entry.Name()) + require.NoError(t, err) + for _, action := range constraintAction.FindAllSubmatch(comments.ReplaceAll(content, nil), -1) { + lastAction[strings.ToLower(string(action[2]))] = strings.ToUpper(string(action[1])) + } + } + require.Equal(t, "DROP", lastAction["user_platform_quotas_platform_check"]) + require.Equal(t, "DROP", lastAction["composite_model_routes_target_platform_check"]) +} diff --git a/frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts b/frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts index b9b00f9a4ca4..2f703e66e29a 100644 --- a/frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts +++ b/frontend/src/api/__tests__/settings.authSourceDefaults.spec.ts @@ -11,21 +11,18 @@ import { type UpdateSettingsRequest, type DefaultPlatformQuotasMap, } from "@/api/admin/settings"; +import { listPlatformIds } from "@/constants/platformCatalog"; -/** 全 null 的全部 concrete 平台 map,用于断言归一化默认值 */ -const allNullQuotas: DefaultPlatformQuotasMap = { - anthropic: { daily: null, weekly: null, monthly: null }, - openai: { daily: null, weekly: null, monthly: null }, - gemini: { daily: null, weekly: null, monthly: null }, - antigravity: { daily: null, weekly: null, monthly: null }, - grok: { daily: null, weekly: null, monthly: null }, - kimi: { daily: null, weekly: null, monthly: null }, - zhipu: { daily: null, weekly: null, monthly: null }, - deepseek: { daily: null, weekly: null, monthly: null }, - minimax: { daily: null, weekly: null, monthly: null }, - opencode_go: { daily: null, weekly: null, monthly: null }, - typesafe: { daily: null, weekly: null, monthly: null }, -} +/** 与后端 AllowedQuotaPlatforms 一致的全部具体平台(平台清单)。 */ +const quotaPlatforms = [ + "anthropic", "openai", "gemini", "antigravity", "grok", + "kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe", "command_code", "cline", +]; + +/** 全部平台全 null 的 map,用于断言归一化默认值 */ +const allNullQuotas: DefaultPlatformQuotasMap = Object.fromEntries( + quotaPlatforms.map((platform) => [platform, { daily: null, weekly: null, monthly: null }]), +) describe("admin settings auth source defaults helpers", () => { it("builds auth source defaults state from flat settings fields", () => { @@ -251,12 +248,14 @@ describe("normalizePlatformQuotasMap", () => { expect(result.gemini).toEqual({ daily: null, weekly: null, monthly: null }); expect(result.antigravity).toEqual({ daily: null, weekly: null, monthly: null }); expect(result.grok).toEqual({ daily: null, weekly: null, monthly: null }); + expect(result.kimi).toEqual({ daily: null, weekly: null, monthly: null }); + expect(result.opencode_go).toEqual({ daily: null, weekly: null, monthly: null }); }); - it("无参数时返回全部 concrete 平台全 null", () => { + it("无参数时返回平台清单中的全部平台全 null", () => { const result = normalizePlatformQuotasMap(); - expect(Object.keys(result)).toHaveLength(11); - expect(result).toEqual(allNullQuotas); + expect(listPlatformIds()).toEqual(quotaPlatforms); + expect(Object.keys(result)).toEqual(quotaPlatforms); for (const v of Object.values(result)) { expect(v).toEqual({ daily: null, weekly: null, monthly: null }); } @@ -304,8 +303,7 @@ describe("sanitizePlatformQuotasMap", () => { it("缺失平台填充为全 null", () => { const result = sanitizePlatformQuotasMap({}); - expect(Object.keys(result)).toHaveLength(11); - expect(result).toEqual(allNullQuotas); + expect(Object.keys(result)).toEqual(quotaPlatforms); for (const v of Object.values(result)) { expect(v).toEqual({ daily: null, weekly: null, monthly: null }); } @@ -315,12 +313,12 @@ describe("sanitizePlatformQuotasMap", () => { describe("MiniMax scheduling threshold compatibility", () => { it("initializes and saves the MiniMax threshold", () => { expect(SCHEDULING_THRESHOLD_PLATFORMS).toEqual([ - "openai", "anthropic", "grok", "kimi", "zhipu", "minimax", "opencode_go", + "openai", "anthropic", "grok", "kimi", "zhipu", "minimax", "opencode_go", "command_code", ]); expect(normalizeAccountSchedulingThresholdsMap().minimax).toBe(100); const thresholds = normalizeAccountSchedulingThresholdsMap({ openai: 92, minimax: 73 }); expect(sanitizeAccountSchedulingThresholdsMap(thresholds)).toEqual({ - openai: 92, anthropic: 100, grok: 100, kimi: 100, zhipu: 100, minimax: 73, opencode_go: 100, + openai: 92, anthropic: 100, grok: 100, kimi: 100, zhipu: 100, minimax: 73, opencode_go: 100, command_code: 100, }); }); }); diff --git a/frontend/src/api/admin/cnProviders.ts b/frontend/src/api/admin/cnProviders.ts index 7a4d3f18c5b2..ee0c7df0db08 100644 --- a/frontend/src/api/admin/cnProviders.ts +++ b/frontend/src/api/admin/cnProviders.ts @@ -12,7 +12,7 @@ export interface CNQuotaTier { reset_at?: string } -/** Coding Plan 额度探测结果(kimi / zhipu),对齐后端 CNProviderQuotaProbeResult。 */ +/** 额度探测结果(Coding Plan / OpenCode Go / Command Code),对齐后端 CNProviderQuotaProbeResult。 */ export interface CNProviderQuotaProbeResult { provider: string source?: string @@ -24,6 +24,8 @@ export interface CNProviderQuotaProbeResult { fetched_at: number persisted: boolean error?: string + /** 同一次探测得到的余额(Command Code 积分与窗口同源),其余供应商为空。 */ + balance?: CNProviderBalanceResult } /** 单币种余额明细(deepseek 双币种账号含 CNY + USD 两条)。 */ diff --git a/frontend/src/api/admin/index.ts b/frontend/src/api/admin/index.ts index 5b70512dbdf7..fa1a41d05032 100644 --- a/frontend/src/api/admin/index.ts +++ b/frontend/src/api/admin/index.ts @@ -37,7 +37,6 @@ import adminComplianceAPI from './compliance' import auditAPI from './audit' import pluginsAPI from './plugins' import systemPromptsAPI from './systemPrompts' -import reasoningRecoveryAPI from './reasoningRecovery' import accountJobsAPI from './accountJobs' /** @@ -78,7 +77,6 @@ export const adminAPI = { audit: auditAPI, plugins: pluginsAPI, systemPrompts: systemPromptsAPI, - reasoningRecovery: reasoningRecoveryAPI, accountJobs: accountJobsAPI } @@ -117,7 +115,6 @@ export { auditAPI, pluginsAPI, systemPromptsAPI, - reasoningRecoveryAPI, accountJobsAPI } @@ -137,4 +134,3 @@ export type { PluginTestResult } from './plugins' export type { SystemPrompt, SystemPromptBinding, SystemPromptConfig, SystemPromptState } from './systemPrompts' -export type { ReasoningRecoveryConfig } from './reasoningRecovery' diff --git a/frontend/src/api/admin/ops.ts b/frontend/src/api/admin/ops.ts index 1476026df7e1..cbd2611745a6 100644 --- a/frontend/src/api/admin/ops.ts +++ b/frontend/src/api/admin/ops.ts @@ -64,6 +64,14 @@ export interface OpsDashboardOverview { avg: number } + output_tps?: { + p5: number | null + p10: number | null + p50: number | null + avg: number | null + sample_count: number + } | null + duration: OpsPercentiles ttft: OpsPercentiles } diff --git a/frontend/src/api/admin/reasoningRecovery.ts b/frontend/src/api/admin/reasoningRecovery.ts deleted file mode 100644 index a6d42e92342c..000000000000 --- a/frontend/src/api/admin/reasoningRecovery.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { apiClient } from '../client' - -export interface ReasoningRecoveryConfig { - enabled: boolean -} - -const basePath = '/admin/reasoning-recovery' - -export const reasoningRecoveryAPI = { - get: async () => (await apiClient.get(basePath)).data, - save: async (config: ReasoningRecoveryConfig) => (await apiClient.put(basePath, config)).data -} - -export default reasoningRecoveryAPI diff --git a/frontend/src/api/admin/settings.ts b/frontend/src/api/admin/settings.ts index 05804b48322c..d30b46211a9c 100644 --- a/frontend/src/api/admin/settings.ts +++ b/frontend/src/api/admin/settings.ts @@ -4,7 +4,7 @@ */ import { apiClient } from "../client"; -import { CONCRETE_PLATFORM_VALUES } from "@/constants/platforms"; +import { listPlatformIds } from "@/constants/platformCatalog"; import type { AccountPlatform, CustomEndpoint, @@ -51,6 +51,7 @@ export interface DefaultSubscriptionSetting { } // ── 平台限额类型 ────────────────────────────────────────────────── +/** 可设置默认限额的平台:平台清单中的全部具体平台(与后端 AllowedQuotaPlatforms 同源)。 */ export type PlatformType = AccountPlatform export type QuotaWindowType = "daily" | "weekly" | "monthly" @@ -64,7 +65,6 @@ export interface PlatformQuotaLimits { /** 全平台默认限额 map(key = PlatformType) */ export type DefaultPlatformQuotasMap = Partial> -const PLATFORMS: readonly PlatformType[] = CONCRETE_PLATFORM_VALUES export type SchedulingThresholdPlatformType = | "openai" @@ -74,11 +74,12 @@ export type SchedulingThresholdPlatformType = | "zhipu" | "minimax" | "opencode_go" + | "command_code" export type AccountSchedulingThresholdsMap = Record // 与后端 AllowedSchedulingThresholdPlatforms 保持一致(deepseek 为余额型, -// 走余额检测而非用量阈值;minimax Coding/Token Plan 与 OpenCode GO 有滚动窗口)。 +// 走余额检测而非用量阈值;minimax Coding/Token Plan、OpenCode GO 与 Command Code 有滚动窗口)。 export const SCHEDULING_THRESHOLD_PLATFORMS: SchedulingThresholdPlatformType[] = [ "openai", "anthropic", @@ -87,6 +88,7 @@ export const SCHEDULING_THRESHOLD_PLATFORMS: SchedulingThresholdPlatformType[] = "zhipu", "minimax", "opencode_go", + "command_code", ] export function normalizeAccountSchedulingThresholdsMap( @@ -108,10 +110,10 @@ export function sanitizeAccountSchedulingThresholdsMap( return normalizeAccountSchedulingThresholdsMap(input) } -/** 归一化为全部 concrete 平台 × 3 窗口(缺失填 null),供模板非空绑定 */ +/** 归一化为全部平台 × 3 窗口(缺失填 null),供模板非空绑定 */ export function normalizePlatformQuotasMap(input?: DefaultPlatformQuotasMap | null): DefaultPlatformQuotasMap { const result: DefaultPlatformQuotasMap = {} - for (const p of PLATFORMS) { + for (const p of listPlatformIds()) { const src = input?.[p] result[p] = { daily: typeof src?.daily === "number" ? src.daily : null, @@ -122,11 +124,11 @@ export function normalizePlatformQuotasMap(input?: DefaultPlatformQuotasMap | nu return result } -/** 提交前清洗:非有限数/负数/空字符串 → null(保留 0 = 显式禁用),返回全平台嵌套 map */ +/** 提交前清洗:非有限数/负数/空字符串 → null(保留 0 = 显式禁用),返回全部平台嵌套 map */ export function sanitizePlatformQuotasMap(input?: DefaultPlatformQuotasMap | null): DefaultPlatformQuotasMap { const clean = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null) const result: DefaultPlatformQuotasMap = {} - for (const p of PLATFORMS) { + for (const p of listPlatformIds()) { const src = input?.[p] result[p] = { daily: clean(src?.daily), weekly: clean(src?.weekly), monthly: clean(src?.monthly) } } diff --git a/frontend/src/api/admin/users.ts b/frontend/src/api/admin/users.ts index 5624e8727ebb..d943508e7979 100644 --- a/frontend/src/api/admin/users.ts +++ b/frontend/src/api/admin/users.ts @@ -4,8 +4,8 @@ */ import { apiClient } from '../client' +import { listPlatformIds } from '@/constants/platformCatalog' import type { AccountPlatform, AdminUser, UpdateUserRequest, PaginatedResponse, ApiKey } from '@/types' -import { CONCRETE_PLATFORM_VALUES } from '@/constants/platforms' export interface AdminBindAuthIdentityChannelRequest { channel: string @@ -331,10 +331,11 @@ export async function bindUserAuthIdentity( /** * Platform quota types */ +// 与后端 AllowedQuotaPlatforms 同源:平台清单中的全部具体平台。 +export function platformQuotaPlatforms(): PlatformQuotaPlatform[] { + return listPlatformIds() +} export type PlatformQuotaPlatform = AccountPlatform - -// Downstream: every concrete account platform supports platform quotas. -export const PLATFORM_QUOTA_PLATFORMS = [...CONCRETE_PLATFORM_VALUES] as PlatformQuotaPlatform[] export type PlatformQuotaWindow = 'daily' | 'weekly' | 'monthly' export interface PlatformQuotaItem { diff --git a/frontend/src/components/account/AccountUsageCell.vue b/frontend/src/components/account/AccountUsageCell.vue index 9d8206e3cee5..618788f82a9c 100644 --- a/frontend/src/components/account/AccountUsageCell.vue +++ b/frontend/src/components/account/AccountUsageCell.vue @@ -517,8 +517,9 @@ - - diff --git a/frontend/src/views/admin/ReasoningRecoveryView.vue b/frontend/src/views/admin/ReasoningRecoveryView.vue deleted file mode 100644 index ace8ba5fd39c..000000000000 --- a/frontend/src/views/admin/ReasoningRecoveryView.vue +++ /dev/null @@ -1,80 +0,0 @@ - - - diff --git a/frontend/src/views/admin/SettingsView.vue b/frontend/src/views/admin/SettingsView.vue index 6351e4a9592a..0489100eb022 100644 --- a/frontend/src/views/admin/SettingsView.vue +++ b/frontend/src/views/admin/SettingsView.vue @@ -4084,7 +4084,7 @@ - + {{ p }} @@ -4421,7 +4421,7 @@ - + {{ p }} @@ -9073,6 +9073,7 @@ import ObservabilitySettingsPanel from '@/components/admin/ObservabilitySettingsPanel.vue' import OfficialModelCatalogPanel from '@/components/admin/OfficialModelCatalogPanel.vue' import { ref, reactive, computed, onMounted, watch, nextTick } from "vue"; +import { listPlatformIds } from "@/constants/platformCatalog"; import { useI18n } from "vue-i18n"; import { adminAPI } from "@/api"; import { @@ -9125,7 +9126,6 @@ import PaymentProviderDialog from "@/components/payment/PaymentProviderDialog.vu import GroupBadge from "@/components/common/GroupBadge.vue"; import GroupOptionItem from "@/components/common/GroupOptionItem.vue"; import Toggle from "@/components/common/Toggle.vue"; -import { CONCRETE_PLATFORM_VALUES } from "@/constants/platforms"; import ProxySelector from "@/components/common/ProxySelector.vue"; import ImageUpload from "@/components/common/ImageUpload.vue"; import BackupSettings from "@/views/admin/BackupView.vue"; @@ -9866,13 +9866,18 @@ type SettingsForm = Omit< payment_recharge_bonus_tiers: RechargeBonusTierDraft[]; payment_recharge_bonus_mode: RechargeBonusMode; payment_recharge_bonus_notice: string; - // 系统全局平台限额 map;form 内始终归一化为全部 concrete 平台对象。 + // 系统全局平台限额 map;form 内始终归一化为全部平台对象(模板非空绑定依赖此不变量) default_platform_quotas: DefaultPlatformQuotasMap; account_scheduling_thresholds: ReturnType; }; const schedulingThresholdPlatforms = SCHEDULING_THRESHOLD_PLATFORMS; -const platformQuotaPlatforms = CONCRETE_PLATFORM_VALUES; + +// 平台限额表格的行:平台清单顺序中、已在归一化 map 里的平台(清单晚于设置加载时 +// 不渲染尚未归一化的平台,保持模板非空绑定)。 +function platformQuotaRows(map: DefaultPlatformQuotasMap | undefined): string[] { + return listPlatformIds().filter((platform) => !!map?.[platform]); +} const form = reactive({ registration_enabled: true, diff --git a/frontend/src/views/admin/__tests__/BackupView.spec.ts b/frontend/src/views/admin/__tests__/BackupView.spec.ts index 8c1e1563bc02..933e70228ce5 100644 --- a/frontend/src/views/admin/__tests__/BackupView.spec.ts +++ b/frontend/src/views/admin/__tests__/BackupView.spec.ts @@ -108,6 +108,27 @@ describe('admin BackupView', () => { document.body.innerHTML = '' }) + it.each(['backup', 'restore'])('does not resume %s polling after navigation during initial loading', async (operation) => { + vi.useFakeTimers() + let finish!: (value: object) => void + listBackups.mockImplementationOnce(() => new Promise(resolve => { finish = resolve })) + const wrapper = mountBackupView() + try { + await flushPromises() + wrapper.unmount() + finish({ items: [{ ...baseRecord('pending'), + status: operation === 'backup' ? 'running' : 'completed', + restore_status: operation === 'restore' ? 'running' : undefined, + }] }) + await flushPromises() + expect(vi.getTimerCount()).toBe(0) + } finally { + wrapper.unmount() + vi.clearAllTimers() + vi.useRealTimers() + } + }) + it('显示分卷数并在下载时列出每个分卷链接', async () => { listBackups.mockResolvedValue({ items: [baseRecord('split', [{ index: 1 }, { index: 2 }, { index: 3 }])], diff --git a/frontend/src/views/admin/__tests__/ChannelsView.modelSync.spec.ts b/frontend/src/views/admin/__tests__/ChannelsView.modelSync.spec.ts new file mode 100644 index 000000000000..fc3abad7cae6 --- /dev/null +++ b/frontend/src/views/admin/__tests__/ChannelsView.modelSync.spec.ts @@ -0,0 +1,155 @@ +import { defineComponent } from 'vue' +import { flushPromises, mount } from '@vue/test-utils' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + BUILTIN_PLATFORM_CATALOG, + resetPlatformCatalog, + setPlatformCatalog +} from '@/constants/platformCatalog' +import ChannelsView from '@/views/admin/ChannelsView.vue' + +const { listChannels, getGroups, syncPricingModels, showSuccess, showError } = vi.hoisted(() => ({ + listChannels: vi.fn(), + getGroups: vi.fn(), + syncPricingModels: vi.fn(), + showSuccess: vi.fn(), + showError: vi.fn() +})) + +vi.mock('@/api/admin', () => ({ + adminAPI: { + channels: { list: listChannels, syncPricingModels }, + groups: { getAll: getGroups }, + settings: { getWebSearchEmulationConfig: vi.fn().mockResolvedValue({ enabled: false }) } + } +})) + +vi.mock('@/stores/app', () => ({ useAppStore: () => ({ showSuccess, showError }) })) + +vi.mock('vue-i18n', async () => ({ + ...await vi.importActual('vue-i18n'), + useI18n: () => ({ t: (key: string) => key }) +})) + +function mountView() { + return mount(ChannelsView, { + global: { + stubs: { + AppLayout: defineComponent({ template: '
' }), + TablePageLayout: defineComponent({ + template: '
' + }), + BaseDialog: defineComponent({ + props: ['show'], + template: '
' + }), + PricingEntryCard: defineComponent({ + props: ['entry'], + template: '
{{ entry.models.join(", ") }}
' + }), + DataTable: true, + Pagination: true, + ConfirmDialog: true, + EmptyState: true, + Select: true, + Icon: true, + PlatformIcon: true, + Toggle: true + } + } + }) +} + +async function openDialog(wrapper: ReturnType) { + await flushPromises() + await wrapper.findAll('button').find(button => button.text() === 'admin.channels.createChannel')!.trigger('click') + await flushPromises() +} + +async function enablePlatform(wrapper: ReturnType, platform: string) { + const label = wrapper.findAll('label').find(label => label.text() === `admin.groups.platforms.${platform}`) + expect(label).toBeDefined() + await label!.get('input[type="checkbox"]').setValue(true) + await wrapper.findAll('button.channel-tab').find(button => button.text() === `admin.groups.platforms.${platform}`)!.trigger('click') +} + +function visibleSyncButtons(wrapper: ReturnType) { + return wrapper.findAll('button').filter(button => button.isVisible() && [ + 'admin.channels.form.syncLatestModels', + 'admin.channels.form.syncingModels' + ].includes(button.text())) +} + +describe('ChannelsView pricing model sync', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + listChannels.mockResolvedValue({ items: [], total: 0 }) + getGroups.mockResolvedValue([]) + syncPricingModels.mockReset() + }) + + afterEach(() => { + resetPlatformCatalog() + }) + + it.each([ + ['openai', true], + ['opencode_go', true], + ['typesafe', true], + ['command_code', false], + ['cline', false] + ] as const)('offers pricing model sync for %s only when its pricing catalog exists', async (platform, expected) => { + const wrapper = mountView() + await openDialog(wrapper) + await enablePlatform(wrapper, platform) + + expect(visibleSyncButtons(wrapper)).toHaveLength(expected ? 1 : 0) + expect(syncPricingModels).not.toHaveBeenCalled() + wrapper.unmount() + }) + + it('reacts to a new platform gaining a pricing catalog', async () => { + const platform = { id: 'acme_router', display_name: 'Acme Router', gateway: 'openai' as const, cn_provider: false } + setPlatformCatalog({ + ...BUILTIN_PLATFORM_CATALOG, + platforms: [...BUILTIN_PLATFORM_CATALOG.platforms, platform] + }) + const wrapper = mountView() + await openDialog(wrapper) + await enablePlatform(wrapper, platform.id) + expect(visibleSyncButtons(wrapper)).toHaveLength(0) + + setPlatformCatalog({ + ...BUILTIN_PLATFORM_CATALOG, + platforms: [...BUILTIN_PLATFORM_CATALOG.platforms, { ...platform, litellm_provider: 'acme' }] + }) + await flushPromises() + expect(visibleSyncButtons(wrapper)).toHaveLength(1) + wrapper.unmount() + }) + + it('blocks additional syncs while one request is in flight and adds its models once', async () => { + let resolveSync!: (value: { models: string[] }) => void + syncPricingModels.mockReturnValue(new Promise(resolve => { resolveSync = resolve })) + const wrapper = mountView() + await openDialog(wrapper) + await enablePlatform(wrapper, 'openai') + await visibleSyncButtons(wrapper)[0].trigger('click') + await visibleSyncButtons(wrapper)[0].trigger('click') + await enablePlatform(wrapper, 'opencode_go') + + expect(visibleSyncButtons(wrapper)[0].attributes('disabled')).toBeDefined() + await visibleSyncButtons(wrapper)[0].trigger('click') + expect(syncPricingModels).toHaveBeenCalledOnce() + expect(syncPricingModels).toHaveBeenCalledWith('openai') + + resolveSync({ models: ['gpt-latest'] }) + await flushPromises() + expect(wrapper.findAll('.pricing-entry')).toHaveLength(1) + expect(wrapper.get('.pricing-entry').text()).toBe('gpt-latest') + expect(visibleSyncButtons(wrapper)[0].attributes('disabled')).toBeUndefined() + expect(showError).not.toHaveBeenCalled() + wrapper.unmount() + }) +}) diff --git a/frontend/src/views/admin/__tests__/PluginsView.spec.ts b/frontend/src/views/admin/__tests__/PluginsView.spec.ts index 3fa4a7ed294d..2504964aa55b 100644 --- a/frontend/src/views/admin/__tests__/PluginsView.spec.ts +++ b/frontend/src/views/admin/__tests__/PluginsView.spec.ts @@ -1,7 +1,11 @@ -import { flushPromises, mount } from '@vue/test-utils' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { enableAutoUnmount, flushPromises, mount } from '@vue/test-utils' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import PluginsView from '../PluginsView.vue' +import BaseDialog from '@/components/common/BaseDialog.vue' +import type { PluginUISession } from '@/api/admin' + +enableAutoUnmount(afterEach) const { listPlugins, @@ -151,6 +155,60 @@ describe('管理员插件页二次验证', () => { expect(enablePlugin).toHaveBeenCalledWith(7, 100, false) }) + it.each([7, 8])('keeps the latest session when reopening configuration for plugin %s', async (id) => { + let resolveOld!: (session: PluginUISession) => void + createUISession.mockReturnValueOnce(new Promise((resolve) => { resolveOld = resolve })) + listPlugins.mockResolvedValue([plugin, { ...plugin, id: 8, name: 'Other Transport' }]) + const wrapper = mountView() + await flushPromises() + const buttons = wrapper.findAll('button').filter(item => item.text() === 'admin.plugins.configure') + await buttons[0].trigger('click') + wrapper.getComponent(BaseDialog).vm.$emit('close') + await buttons[id === 7 ? 0 : 1].trigger('click') + await flushPromises() + const currentURL = wrapper.get('iframe').attributes('src') + + resolveOld({ url: '/obsolete-session', bridge_token: 'old', ui_bridge_version: 1, expires_at: '' }) + await flushPromises() + expect(wrapper.get('iframe').attributes('src')).toBe(currentURL) + }) + + it('ignores an old session error while the reopened configuration is loading', async () => { + let rejectOld!: (error: Error) => void + let resolveCurrent!: (session: PluginUISession) => void + createUISession + .mockReturnValueOnce(new Promise((_, reject) => { rejectOld = reject })) + .mockReturnValueOnce(new Promise((resolve) => { resolveCurrent = resolve })) + const wrapper = mountView() + await flushPromises() + const button = wrapper.findAll('button').find(item => item.text() === 'admin.plugins.configure')! + await button.trigger('click') + wrapper.getComponent(BaseDialog).vm.$emit('close') + await button.trigger('click') + rejectOld(new Error('obsolete failure')) + await flushPromises() + + expect(wrapper.text()).not.toContain('obsolete failure') + expect(wrapper.text()).toContain('admin.plugins.loadingUI') + resolveCurrent({ url: '/current-session', bridge_token: 'current', ui_bridge_version: 1, expires_at: '' }) + await flushPromises() + await wrapper.get('iframe').trigger('load') + expect(wrapper.get('iframe').attributes('src')).toBe('/current-session') + expect(wrapper.text()).not.toContain('admin.plugins.loadingUI') + }) + + it('does not restore a session after the configuration dialog closes', async () => { + let resolveSession!: (session: PluginUISession) => void + createUISession.mockReturnValueOnce(new Promise((resolve) => { resolveSession = resolve })) + const wrapper = mountView() + await flushPromises() + await wrapper.findAll('button').find(item => item.text() === 'admin.plugins.configure')!.trigger('click') + wrapper.getComponent(BaseDialog).vm.$emit('close') + resolveSession({ url: '/closed-session', bridge_token: 'old', ui_bridge_version: 1, expires_at: '' }) + await flushPromises() + expect(wrapper.find('iframe').exists()).toBe(false) + }) + it('上传插件通过 step-up 控制器执行', async () => { const wrapper = mountView() await flushPromises() diff --git a/frontend/src/views/admin/__tests__/ReasoningRecoveryView.spec.ts b/frontend/src/views/admin/__tests__/ReasoningRecoveryView.spec.ts deleted file mode 100644 index 70b81f8fd320..000000000000 --- a/frontend/src/views/admin/__tests__/ReasoningRecoveryView.spec.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { flushPromises, mount } from '@vue/test-utils' - -import ReasoningRecoveryView from '../ReasoningRecoveryView.vue' - -const { get, put, showSuccess, useStepUp } = vi.hoisted(() => ({ - get: vi.fn(), - put: vi.fn(), - showSuccess: vi.fn(), - useStepUp: vi.fn(), -})) - -vi.mock('@/api/client', () => ({ apiClient: { get, put } })) -vi.mock('@/stores/app', () => ({ useAppStore: () => ({ showSuccess }) })) -vi.mock('@/composables/useStepUp', () => ({ useStepUp })) -vi.mock('vue-i18n', async (importOriginal) => ({ - ...(await importOriginal()), - useI18n: () => ({ t: (key: string) => key }), -})) - -const path = '/admin/reasoning-recovery' - -async function mountView() { - const wrapper = mount(ReasoningRecoveryView, { - global: { stubs: { AppLayout: { template: '
' } } }, - }) - await flushPromises() - return wrapper -} - -type View = Awaited> -const toggle = (wrapper: View) => wrapper.get('[data-test="reasoning-recovery-enabled"]') -const saveButton = (wrapper: View) => wrapper.get('[data-test="reasoning-recovery-save"]') -// Every control of the page, in document order. -const controls = (wrapper: View) => wrapper.findAll('button').map(button => button.attributes('data-test')) - -describe('ReasoningRecoveryView', () => { - beforeEach(() => { - get.mockReset().mockResolvedValue({ data: { enabled: true } }) - put.mockReset().mockImplementation(async (_url: string, value: unknown) => ({ data: value })) - showSuccess.mockReset() - useStepUp.mockReset() - }) - - it('shows only Save until the stored value is loaded, then the switch, and enables Save only while the draft differs', async () => { - let answer!: (value: { data: { enabled: boolean } }) => void - get.mockReturnValueOnce(new Promise((resolve) => { answer = resolve })) - const wrapper = await mountView() - - expect(get).toHaveBeenCalledWith(path) - expect(wrapper.text()).toContain('common.loading') - expect(controls(wrapper)).toEqual(['reasoning-recovery-save']) - expect(saveButton(wrapper).element.disabled).toBe(true) - - answer({ data: { enabled: true } }) - await flushPromises() - expect(wrapper.text()).not.toContain('common.loading') - expect(controls(wrapper)).toEqual(['reasoning-recovery-save', 'reasoning-recovery-enabled']) - expect(toggle(wrapper).attributes('aria-checked')).toBe('true') - expect(saveButton(wrapper).element.disabled).toBe(true) - - await toggle(wrapper).trigger('click') - expect(toggle(wrapper).attributes('aria-checked')).toBe('false') - expect(saveButton(wrapper).element.disabled).toBe(false) - - await toggle(wrapper).trigger('click') - expect(saveButton(wrapper).element.disabled).toBe(true) - expect(put).not.toHaveBeenCalled() - }) - - it('saves the draft with one PUT and no step-up', async () => { - const wrapper = await mountView() - await toggle(wrapper).trigger('click') - await saveButton(wrapper).trigger('click') - await flushPromises() - - expect(put).toHaveBeenCalledTimes(1) - expect(put).toHaveBeenCalledWith(path, { enabled: false }) - expect(useStepUp).not.toHaveBeenCalled() - expect(wrapper.findComponent({ name: 'TotpStepUpDialog' }).exists()).toBe(false) - expect(showSuccess).toHaveBeenCalledWith('admin.reasoningRecovery.saved') - expect(toggle(wrapper).attributes('aria-checked')).toBe('false') - expect(saveButton(wrapper).element.disabled).toBe(true) - expect(wrapper.find('[role="alert"]').exists()).toBe(false) - }) - - it('replaces the draft with the value the server stored', async () => { - put.mockResolvedValue({ data: { enabled: true } }) - const wrapper = await mountView() - await toggle(wrapper).trigger('click') - await saveButton(wrapper).trigger('click') - await flushPromises() - - expect(put).toHaveBeenCalledWith(path, { enabled: false }) - expect(toggle(wrapper).attributes('aria-checked')).toBe('true') - expect(saveButton(wrapper).element.disabled).toBe(true) - }) - - it('reports a failed or unusable load without a switch, and reads again only when the page is opened again', async () => { - get.mockRejectedValueOnce({ message: 'load refused' }) - const refused = await mountView() - - expect(refused.get('[role="alert"]').text()).toBe('load refused') - expect(controls(refused)).toEqual(['reasoning-recovery-save']) - expect(saveButton(refused).element.disabled).toBe(true) - expect(get).toHaveBeenCalledTimes(1) - refused.unmount() - - get.mockResolvedValueOnce({ data: {} }) - const unusable = await mountView() - expect(unusable.get('[role="alert"]').text()).toBe('admin.reasoningRecovery.loadFailed') - expect(controls(unusable)).toEqual(['reasoning-recovery-save']) - unusable.unmount() - - get.mockResolvedValueOnce({ data: { enabled: false } }) - const reopened = await mountView() - expect(get).toHaveBeenCalledTimes(3) - expect(reopened.find('[role="alert"]').exists()).toBe(false) - expect(toggle(reopened).attributes('aria-checked')).toBe('false') - }) - - it('reports a failed save and keeps the draft', async () => { - put.mockRejectedValue({ message: 'save refused' }) - const wrapper = await mountView() - await toggle(wrapper).trigger('click') - await saveButton(wrapper).trigger('click') - await flushPromises() - - expect(wrapper.get('[role="alert"]').text()).toBe('save refused') - expect(showSuccess).not.toHaveBeenCalled() - expect(toggle(wrapper).attributes('aria-checked')).toBe('false') - expect(saveButton(wrapper).element.disabled).toBe(false) - }) -}) diff --git a/frontend/src/views/admin/__tests__/SettingsView.spec.ts b/frontend/src/views/admin/__tests__/SettingsView.spec.ts index c1277d11361f..a7c761b06534 100644 --- a/frontend/src/views/admin/__tests__/SettingsView.spec.ts +++ b/frontend/src/views/admin/__tests__/SettingsView.spec.ts @@ -2295,7 +2295,7 @@ describe("admin SettingsView platform quota matrix", () => { expect(html).toContain("typesafe"); }); - it("保存时 updateSettings payload 应包含嵌套 default_platform_quotas 对象(含全 11 平台)", async () => { + it("保存时 updateSettings payload 应包含嵌套 default_platform_quotas 对象(含全 13 平台)", async () => { const wrapper = mountView(); await flushPromises(); await openUsersTab(wrapper); @@ -2311,7 +2311,7 @@ describe("admin SettingsView platform quota matrix", () => { // 应携带嵌套对象,而非扁平字段 expect(payload).toHaveProperty("default_platform_quotas"); const quotas = payload["default_platform_quotas"] as Record; - const platforms = ["anthropic", "openai", "gemini", "antigravity", "grok", "kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe"]; + const platforms = ["anthropic", "openai", "gemini", "antigravity", "grok", "kimi", "zhipu", "deepseek", "minimax", "opencode_go", "typesafe", "command_code", "cline"]; expect(Object.keys(quotas)).toEqual(platforms); for (const p of platforms) { expect(quotas).toHaveProperty(p); @@ -2326,7 +2326,7 @@ describe("admin SettingsView platform quota matrix", () => { expect(payload).not.toHaveProperty("default_platform_quota_openai_weekly"); }); - it("加载后 form.default_platform_quotas 含全 11 平台并保留 MiniMax 数值", async () => { + it("加载后 form.default_platform_quotas 含全 13 平台并保留 MiniMax 数值", async () => { getSettings.mockResolvedValueOnce({ ...baseSettingsResponse, default_platform_quotas: { @@ -2349,7 +2349,7 @@ describe("admin SettingsView platform quota matrix", () => { expect(quotas["anthropic"]?.["daily"]).toBe(5); expect(quotas["openai"]?.["weekly"]).toBe(12.5); - expect(Object.keys(quotas)).toHaveLength(11); + expect(Object.keys(quotas)).toHaveLength(13); expect(quotas["minimax"]).toEqual({ daily: 0, weekly: 20, monthly: 80 }); // 缺失平台应补全为 null expect(quotas["gemini"]).toEqual({ daily: null, weekly: null, monthly: null }); diff --git a/frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts b/frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts index 7ed60eea701d..c29b45d76cdb 100644 --- a/frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts +++ b/frontend/src/views/admin/__tests__/channelPlatformOptions.spec.ts @@ -1,15 +1,17 @@ import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { listPlatformIds } from '@/constants/platformCatalog' describe('Composite channel platform options', () => { it('includes every concrete provider for pricing and model mapping', () => { const source = readFileSync(resolve('src/views/admin/ChannelsView.vue'), 'utf8') - const declaration = source.match(/const compositePlatforms:[^=]+=[^\n]+/)?.[0] - const order = source.match(/const platformOrder:[^=]+=[^\n]+/)?.[0] - const expectedPlatforms = ['anthropic', 'openai', 'gemini', 'antigravity', 'grok', 'kimi', 'zhipu', 'deepseek', 'minimax', 'opencode_go', 'typesafe'] + // 平台列表来自平台清单(后端 domain/platforms.go),组合分组覆盖全部具体平台。 + expect(source).toMatch(/const platformOrder = computed\(\(\) => listPlatformIds\(\)\)/) + expect(source).toContain('const compositePlatforms = platformOrder') - expect(declaration?.match(/'([^']+)'/g)?.map(value => value.slice(1, -1))).toEqual(expectedPlatforms) - expect(order?.match(/'([^']+)'/g)?.map(value => value.slice(1, -1))).toEqual(expectedPlatforms) + expect(listPlatformIds()).toEqual( + expect.arrayContaining(['kimi', 'zhipu', 'deepseek', 'minimax', 'opencode_go']) + ) }) }) diff --git a/frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue b/frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue index 6a58d834c602..172ae9a71880 100644 --- a/frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue +++ b/frontend/src/views/admin/ops/components/OpsAlertEventsCard.vue @@ -1,5 +1,5 @@