Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 14 additions & 11 deletions backend/internal/handler/admin/account_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -2874,8 +2874,19 @@ func (h *AccountHandler) GetAvailableModels(c *gin.Context) {
response.Success(c, models)
}

// accountTestModels lists the models the single-account test can select.
func (h *AccountHandler) accountTestModels(ctx context.Context, account *service.Account) (any, error) {
// accountTestModels lists the models the single-account test can select, each
// row stating the reasoning efforts the test transmits for it. Both the raw
// model list and the test plan view are built from these rows.
func (h *AccountHandler) accountTestModels(ctx context.Context, account *service.Account) ([]map[string]any, error) {
catalog, err := h.accountTestCatalog(ctx, account)
if err != nil {
return nil, err
}
return accountTestReasoningRows(account, catalog)
}

// accountTestCatalog lists the platform's selectable test models.
func (h *AccountHandler) accountTestCatalog(ctx context.Context, account *service.Account) (any, error) {
// Handle OpenAI accounts
if account.IsOpenAI() {
return h.openAIAccountTestModels(ctx, account)
Expand Down Expand Up @@ -3016,15 +3027,7 @@ func (h *AccountHandler) accountTestModels(ctx context.Context, account *service
// openAIAccountTestModels selects request-side model IDs from this account's
// saved configuration. Discovery enumerates unrestricted accounts and wildcard
// candidates; it is not an availability gate for explicitly configured models.
func (h *AccountHandler) openAIAccountTestModels(ctx context.Context, account *service.Account) (result []openai.Model, resultErr error) {
defer func() {
if resultErr != nil {
return
}
for i := range result {
result[i].ReasoningEfforts, result[i].DefaultReasoningEffort = service.AccountTestReasoningOptions(account, result[i].ID)
}
}()
func (h *AccountHandler) openAIAccountTestModels(ctx context.Context, account *service.Account) ([]openai.Model, error) {
mapping := account.GetModelMapping()
if account.IsOpenAIPassthroughEnabled() || len(mapping) == 0 {
return h.discoverOpenAIAccountTestModels(ctx, account)
Expand Down
35 changes: 34 additions & 1 deletion backend/internal/handler/admin/account_test_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ func (h *AccountHandler) accountTestPlan(ctx context.Context, account *service.A
return ordinaryAccountTestPlan(account, raw)
}

func accountTestPlanModels(raw any) ([]map[string]any, error) {
// accountTestModelRows decodes a typed model catalog into its JSON rows.
func accountTestModelRows(raw any) ([]map[string]any, error) {
encoded, err := json.Marshal(raw)
if err != nil {
return nil, err
Expand All @@ -60,6 +61,38 @@ func accountTestPlanModels(raw any) ([]map[string]any, error) {
if json.Unmarshal(encoded, &models) != nil {
return nil, errors.New("invalid account test model catalog")
}
return models, nil
}

// accountTestReasoningRows decodes any platform's catalog into rows and gives
// each row the reasoning efforts the account test transmits for that model,
// under openai.Model's JSON keys. Rows without efforts carry neither key.
func accountTestReasoningRows(account *service.Account, raw any) ([]map[string]any, error) {
models, err := accountTestModelRows(raw)
if err != nil {
return nil, err
}
for _, model := range models {
delete(model, "reasoning_efforts")
delete(model, "default_reasoning_effort")
id, _ := model["id"].(string)
levels, defaultLevel := service.AccountTestReasoningOptions(account, id)
if len(levels) == 0 {
continue
}
model["reasoning_efforts"] = levels
if defaultLevel != "" {
model["default_reasoning_effort"] = defaultLevel
}
}
return models, nil
}

func accountTestPlanModels(raw any) ([]map[string]any, error) {
models, err := accountTestModelRows(raw)
if err != nil {
return nil, err
}
if models == nil {
models = []map[string]any{}
}
Expand Down
46 changes: 46 additions & 0 deletions backend/internal/handler/admin/account_test_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing"

extensionv1 "github.com/Wei-Shaw/sub2api/internal/nativeapi"
"github.com/Wei-Shaw/sub2api/internal/pkg/openai"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/Wei-Shaw/sub2api/internal/testextensions"
"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -135,3 +136,48 @@ func TestAccountTestPlanCoreSelectionSmallMatrix(t *testing.T) {
_, err = ordinaryAccountTestPlan(&service.Account{ID: 42}, []map[string]any{{"id": "same"}, {"id": "same"}})
require.Error(t, err)
}

func TestAccountTestModelsCarryReasoningOnEveryPlatform(t *testing.T) {
gin.SetMode(gin.TestMode)
read := func(account *service.Account, suffix string) json.RawMessage {
h := &AccountHandler{adminService: testPlanAdmin{account: account}}
router := gin.New()
router.GET("/accounts/:id/models", h.GetAvailableModels)
out := httptest.NewRecorder()
router.ServeHTTP(out, httptest.NewRequest(http.MethodGet, "/accounts/42/models"+suffix, nil))
require.Equal(t, http.StatusOK, out.Code, out.Body.String())
var body struct {
Data json.RawMessage `json:"data"`
}
require.NoError(t, json.Unmarshal(out.Body.Bytes(), &body))
return body.Data
}

anthropic := &service.Account{ID: 42, Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"model_mapping": map[string]any{"opus": "claude-opus-4-7", "haiku": "claude-haiku-4-5-20251001"}}}
var plan accountTestPlanView
require.NoError(t, json.Unmarshal(read(anthropic, "?view=account-test-plan-v1"), &plan))
rows := make(map[string]map[string]any, len(plan.Models))
for _, row := range plan.Models {
id, _ := row["id"].(string)
rows[id] = row
}
require.Equal(t, []any{"low", "medium", "high", "xhigh", "max"}, rows["opus"]["reasoning_efforts"])
require.Equal(t, "high", rows["opus"]["default_reasoning_effort"])
require.Contains(t, rows, "haiku")
require.NotContains(t, rows["haiku"], "reasoning_efforts")
require.NotContains(t, rows["haiku"], "default_reasoning_effort")
var raw []map[string]any
require.NoError(t, json.Unmarshal(read(anthropic, ""), &raw))
require.ElementsMatch(t, plan.Models, raw, "the raw list carries the same rows")

// OpenAI rows keep the openai.Model reasoning fields they had before.
openAI := &service.Account{ID: 42, Platform: service.PlatformOpenAI, Type: service.AccountTypeOAuth,
Credentials: map[string]any{"model_mapping": map[string]any{"friendly": "gpt-6-astra"}}}
levels, defaultLevel := service.AccountTestReasoningOptions(openAI, "friendly")
require.NotEmpty(t, levels)
var openAIRows []openai.Model
require.NoError(t, json.Unmarshal(read(openAI, ""), &openAIRows))
require.Equal(t, []openai.Model{{ID: "friendly", Object: "model", Type: "model", DisplayName: "friendly",
ReasoningEfforts: levels, DefaultReasoningEffort: defaultLevel}}, openAIRows)
}
72 changes: 54 additions & 18 deletions backend/internal/pkg/claude/effort_catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,71 @@ var (
effortLowMediumHighXHighMax = []string{"low", "medium", "high", "xhigh", "max"}
)

var effortFamilies = []struct {
type effortFamily struct {
family string
levels []string
}{
// adaptive pairs output_config.effort with thinking {type: "adaptive"}.
// Opus 4.5 thinks only through enabled+budget_tokens; Mythos Preview's
// thinking mode is unverified.
adaptive bool
}

var effortFamilies = []effortFamily{
{family: "claude-mythos-preview", levels: effortLowMediumHighMax},
{family: "claude-mythos-5", levels: effortLowMediumHighXHighMax},
{family: "claude-fable-5", levels: effortLowMediumHighXHighMax},
{family: "claude-sonnet-4-6", levels: effortLowMediumHighMax},
{family: "claude-sonnet-5-5", levels: effortLowMediumHighXHighMax},
{family: "claude-sonnet-5", levels: effortLowMediumHighXHighMax},
{family: "claude-opus-4-8", levels: effortLowMediumHighXHighMax},
{family: "claude-opus-4-7", levels: effortLowMediumHighXHighMax},
{family: "claude-opus-4-6", levels: effortLowMediumHighMax},
{family: "claude-mythos-5", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-fable-5", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-sonnet-4-6", levels: effortLowMediumHighMax, adaptive: true},
{family: "claude-sonnet-5-5", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-sonnet-5", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-opus-4-8", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-opus-4-7", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-opus-4-6", levels: effortLowMediumHighMax, adaptive: true},
{family: "claude-opus-4-5", levels: effortLowMediumHigh},
{family: "claude-opus-5-5", levels: effortLowMediumHighXHighMax},
{family: "claude-opus-5", levels: effortLowMediumHighXHighMax},
{family: "claude-opus-5-5", levels: effortLowMediumHighXHighMax, adaptive: true},
{family: "claude-opus-5", levels: effortLowMediumHighXHighMax, adaptive: true},
}

func findEffortFamily(model string) *effortFamily {
id := normalizeEffortModelID(model)
for i := range effortFamilies {
if entry := &effortFamilies[i]; id == entry.family || strings.HasPrefix(id, entry.family+"-") {
return entry
}
}
return nil
}

// EffortLevelsForModel returns the output_config.effort values accepted by a
// Claude model, ordered from the lightest to the deepest reasoning level.
func EffortLevelsForModel(model string) []string {
id := normalizeEffortModelID(model)
for _, entry := range effortFamilies {
if id == entry.family || strings.HasPrefix(id, entry.family+"-") {
return append([]string(nil), entry.levels...)
}
if entry := findEffortFamily(model); entry != nil {
return append([]string(nil), entry.levels...)
}
return nil
}

// EffortUsesAdaptiveThinking reports whether a Claude model pairs
// output_config.effort with thinking {type: "adaptive"}.
func EffortUsesAdaptiveThinking(model string) bool {
entry := findEffortFamily(model)
return entry != nil && entry.adaptive
}

// DefaultEffortForModel returns the effort a Claude model applies when
// output_config.effort is omitted, or "" when the model has no effort levels.
func DefaultEffortForModel(model string) string {
entry := findEffortFamily(model)
switch {
case entry == nil:
return ""
case entry.family == "claude-opus-5-5":
// Opus 5.5 defaults one level below every other effort family.
return "medium"
default:
return "high"
}
}

// IsOpus55 identifies the fixed Opus 5.5 ID after provider/local suffix normalization.
func IsOpus55(model string) bool {
return normalizeEffortModelID(model) == "claude-opus-5-5"
Expand Down Expand Up @@ -73,9 +108,10 @@ func normalizeEffortModelID(model string) string {
if mapped, ok := ModelIDReverseOverrides[id]; ok {
id = mapped
}
// Dated snapshots: "-YYYYMMDD" (Anthropic API) and "@YYYYMMDD" (Vertex AI).
if len(id) >= 9 {
suffix := id[len(id)-9:]
if suffix[0] == '-' {
if suffix[0] == '-' || suffix[0] == '@' {
digits := true
for _, r := range suffix[1:] {
if !unicode.IsDigit(r) {
Expand Down
10 changes: 10 additions & 0 deletions backend/internal/pkg/claude/effort_catalog_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ func TestEffortLevelsForModel(t *testing.T) {
{model: "claude-sonnet-5-5", want: []string{"low", "medium", "high", "xhigh", "max"}},
{model: "us.anthropic.claude-sonnet-5-5", want: []string{"low", "medium", "high", "xhigh", "max"}},
{model: "claude-opus-4-5-20251101", want: []string{"low", "medium", "high"}},
{model: "claude-opus-4-5@20251101", want: []string{"low", "medium", "high"}},
{model: "claude-haiku-4-5-20251001", want: nil},
{model: "gpt-5.6", want: nil},
}
Expand All @@ -31,6 +32,15 @@ func TestEffortLevelsForModel(t *testing.T) {
}
}

func TestEffortThinkingModeAndDefault(t *testing.T) {
t.Parallel()
require.True(t, EffortUsesAdaptiveThinking("claude-opus-4-7"))
require.False(t, EffortUsesAdaptiveThinking("claude-opus-4-5-20251101"), "Opus 4.5 thinks only with budget_tokens")
require.Equal(t, "medium", DefaultEffortForModel("claude-opus-5-5"))
require.Equal(t, "high", DefaultEffortForModel("claude-opus-4-8"))
require.Empty(t, DefaultEffortForModel("claude-haiku-4-5-20251001"))
}

func TestIsOpus55OpenRouterExactAlias(t *testing.T) {
t.Parallel()
for _, model := range []string{"claude-opus-5-5", "anthropic/claude-opus-5.5"} {
Expand Down
Loading
Loading