diff --git a/src/ask.ts b/src/ask.ts index 3dffb8e..57ef5f9 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -92,6 +92,9 @@ function outOfTime(opts: AskOptions): Asked { return { ok: false, error: `the time budget ran out${opts.what ? ` before ${opts.what}` : ""}` }; } +/** Endpoint+model pairs that refused `temperature`; asked without it from then on. */ +const noTemperature = new Set(); + export async function askModel(opts: AskOptions): Promise { const pauses = opts.overloadBackoffMs ?? ASK_OVERLOAD_BACKOFF_MS; if ((leftMs(opts.deadlineAt) ?? 1) <= 0) return outOfTime(opts); @@ -202,7 +205,7 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { { role: "user", content: opts.prompt }, ], max_tokens: maxTokens, - temperature: 0, + ...(noTemperature.has(`${base} ${opts.model}`) ? {} : { temperature: 0 }), ...(opts.reasoningEffort ? { reasoning: { effort: opts.reasoningEffort } } : {}), ...selfHostedThinking(base, opts.reasoningEffort), ...openRouterProvider(base, opts.model), @@ -210,12 +213,19 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { }), }); if (!res.ok) { - const body = res.status === 429 ? await res.text().catch(() => "") : ""; + const body = await res.text().catch(() => ""); + // Some current models refuse a sampling setting outright (Claude Haiku 5.5: + // "`temperature` is deprecated for this model"). Remember it and ask again + // without, rather than leaving every judge call on that model refused. + if (res.status === 400 && /temperature/i.test(body) && !noTemperature.has(`${base} ${opts.model}`)) { + noTemperature.add(`${base} ${opts.model}`); + return askOnce(opts, maxTokens); + } const resetAt = rateLimitResetAt(body); if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { return { ok: false, error: `the provider's rate limit is reached ${untilText(resetAt)}${opts.what ? ` (${opts.what})` : ""}` }; } - return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}`, ...(res.status === 429 || res.status === 503 ? { transient: true } : {}) }; + return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}${res.status === 400 && body ? `: ${body.slice(0, 300)}` : ""}`, ...(res.status === 429 || res.status === 503 ? { transient: true } : {}) }; } type Reply = { choices?: { message?: { content?: string | null }; finish_reason?: string | null }[]; diff --git a/src/judge.ts b/src/judge.ts index 07fd89e..0648520 100644 --- a/src/judge.ts +++ b/src/judge.ts @@ -16,7 +16,9 @@ export function judgeTarget(worker: T, env: NodeJS.ProcessEnv env.MAAT_JUDGE_KEY?.trim() || (baseUrl === worker.baseUrl ? worker.apiKey : undefined) || // A CLI worker (Grok Build, OpenCode) judged on OpenRouter uses the OpenRouter key. - (/^https:\/\/openrouter\.ai\//.test(baseUrl) ? env.OPENROUTER_API_KEY?.trim() || undefined : undefined); + (/^https:\/\/openrouter\.ai\//.test(baseUrl) ? env.OPENROUTER_API_KEY?.trim() || undefined : undefined) || + // A judge on Anthropic's API (e.g. a Haiku judge for an OpenRouter worker) uses the Anthropic key. + (/^https:\/\/api\.anthropic\.com\//.test(baseUrl) ? env.ANTHROPIC_API_KEY?.trim() || undefined : undefined); const { apiKey: _workerKey, ...rest } = worker; return { ...rest, baseUrl, model, ...(apiKey !== undefined ? { apiKey } : {}) } as T; } diff --git a/test/ask-temperature.test.ts b/test/ask-temperature.test.ts new file mode 100644 index 0000000..5f8fbe8 --- /dev/null +++ b/test/ask-temperature.test.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { askModel } from "../src/ask.js"; + +describe("a model that refuses temperature", () => { + it("is asked again without it, and the error body is kept for other 400s", async () => { + const bodies: Record[] = []; + const fetchFn = (async (_url: string, init: { body: string }) => { + const body = JSON.parse(init.body) as Record; + bodies.push(body); + if ("temperature" in body) { + return new Response(JSON.stringify({ error: { message: "`temperature` is deprecated for this model." } }), { status: 400 }); + } + return new Response(JSON.stringify({ choices: [{ message: { content: "{\"ok\":true}" }, finish_reason: "stop" }] }), { status: 200, headers: { "content-type": "application/json" } }); + }) as unknown as typeof fetch; + const r = await askModel({ baseUrl: "https://api.anthropic.com/v1", apiKey: "k", model: "claude-haiku-5-5", system: "s", prompt: "p", fetchFn }); + assert.ok(r.ok, JSON.stringify(r)); + assert.equal(bodies.length, 2); + assert.ok(!("temperature" in bodies[1]!)); + // Remembered: the next ask goes straight out without it. + await askModel({ baseUrl: "https://api.anthropic.com/v1", apiKey: "k", model: "claude-haiku-5-5", system: "s", prompt: "p", fetchFn }); + assert.equal(bodies.length, 3); + assert.ok(!("temperature" in bodies[2]!)); + }); + + it("puts a 400's body in the error", async () => { + const fetchFn = (async () => new Response("bad tool_choice value", { status: 400 })) as unknown as typeof fetch; + const r = await askModel({ baseUrl: "https://x.test/v1", model: "m", system: "s", prompt: "p", fetchFn }); + assert.ok(!r.ok && /bad tool_choice/.test(r.error)); + }); +}); diff --git a/test/judge.test.ts b/test/judge.test.ts index 5258d84..8114772 100644 --- a/test/judge.test.ts +++ b/test/judge.test.ts @@ -58,3 +58,14 @@ describe("Grok Build as the judge of an HTTP worker", () => { assert.deepEqual(r, { ok: true, text: '{"criteria":[]}', cutOff: false }); }); }); + +describe("a judge on Anthropic's API", () => { + it("takes ANTHROPIC_API_KEY when the worker is elsewhere", () => { + const t = judgeTarget( + { baseUrl: "https://openrouter.ai/api/v1", apiKey: "or-key", model: "deepseek/deepseek-v4-pro" }, + { MAAT_JUDGE_MODEL: "claude-haiku-5-5", MAAT_JUDGE_URL: "https://api.anthropic.com/v1", ANTHROPIC_API_KEY: "sk-ant-x" }, + ); + assert.equal(t.apiKey, "sk-ant-x"); + assert.equal(t.model, "claude-haiku-5-5"); + }); +});