Judge on Anthropic's API: read ANTHROPIC_API_KEY; ask without temperature when a model refuses it

src/ask.ts

@@ -92,6 +92,9 @@ function outOfTime(opts: AskOptions): Asked {
   return { ok: false, error: `the time budget ran out${opts.what ? ` before ${opts.what}` : ""}` };
 }
 
+/** Endpoint+model pairs that refused `temperature`; asked without it from then on. */
+const noTemperature = new Set<string>();
+
 export async function askModel(opts: AskOptions): Promise<Asked> {
   const pauses = opts.overloadBackoffMs ?? ASK_OVERLOAD_BACKOFF_MS;
   if ((leftMs(opts.deadlineAt) ?? 1) <= 0) return outOfTime(opts);
@@ -202,20 +205,27 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise<Asked> {
           { role: "user", content: opts.prompt },
         ],
         max_tokens: maxTokens,
-        temperature: 0,
+        ...(noTemperature.has(`${base} ${opts.model}`) ? {} : { temperature: 0 }),
         ...(opts.reasoningEffort ? { reasoning: { effort: opts.reasoningEffort } } : {}),
         ...selfHostedThinking(base, opts.reasoningEffort),
         ...openRouterProvider(base, opts.model),
         ...(stream ? { stream: true } : {}),
       }),
     });
     if (!res.ok) {
-      const body = res.status === 429 ? await res.text().catch(() => "") : "";
+      const body = await res.text().catch(() => "");
+      // Some current models refuse a sampling setting outright (Claude Haiku 5.5:
+      // "`temperature` is deprecated for this model"). Remember it and ask again
+      // without, rather than leaving every judge call on that model refused.
+      if (res.status === 400 && /temperature/i.test(body) && !noTemperature.has(`${base} ${opts.model}`)) {
+        noTemperature.add(`${base} ${opts.model}`);
+        return askOnce(opts, maxTokens);
+      }
       const resetAt = rateLimitResetAt(body);
       if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) {
         return { ok: false, error: `the provider's rate limit is reached ${untilText(resetAt)}${opts.what ? ` (${opts.what})` : ""}` };
       }
-      return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}`, ...(res.status === 429 || res.status === 503 ? { transient: true } : {}) };
+      return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}${res.status === 400 && body ? `: ${body.slice(0, 300)}` : ""}`, ...(res.status === 429 || res.status === 503 ? { transient: true } : {}) };
     }
     type Reply = {
       choices?: { message?: { content?: string | null }; finish_reason?: string | null }[];

src/judge.ts

@@ -16,7 +16,9 @@ export function judgeTarget<T extends Target>(worker: T, env: NodeJS.ProcessEnv
     env.MAAT_JUDGE_KEY?.trim() ||
     (baseUrl === worker.baseUrl ? worker.apiKey : undefined) ||
     // A CLI worker (Grok Build, OpenCode) judged on OpenRouter uses the OpenRouter key.
-    (/^https:\/\/openrouter\.ai\//.test(baseUrl) ? env.OPENROUTER_API_KEY?.trim() || undefined : undefined);
+    (/^https:\/\/openrouter\.ai\//.test(baseUrl) ? env.OPENROUTER_API_KEY?.trim() || undefined : undefined) ||
+    // A judge on Anthropic's API (e.g. a Haiku judge for an OpenRouter worker) uses the Anthropic key.
+    (/^https:\/\/api\.anthropic\.com\//.test(baseUrl) ? env.ANTHROPIC_API_KEY?.trim() || undefined : undefined);
   const { apiKey: _workerKey, ...rest } = worker;
   return { ...rest, baseUrl, model, ...(apiKey !== undefined ? { apiKey } : {}) } as T;
 }

test/ask-temperature.test.ts

@@ -0,0 +1,31 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { askModel } from "../src/ask.js";
+
+describe("a model that refuses temperature", () => {
+  it("is asked again without it, and the error body is kept for other 400s", async () => {
+    const bodies: Record<string, unknown>[] = [];
+    const fetchFn = (async (_url: string, init: { body: string }) => {
+      const body = JSON.parse(init.body) as Record<string, unknown>;
+      bodies.push(body);
+      if ("temperature" in body) {
+        return new Response(JSON.stringify({ error: { message: "`temperature` is deprecated for this model." } }), { status: 400 });
+      }
+      return new Response(JSON.stringify({ choices: [{ message: { content: "{\"ok\":true}" }, finish_reason: "stop" }] }), { status: 200, headers: { "content-type": "application/json" } });
+    }) as unknown as typeof fetch;
+    const r = await askModel({ baseUrl: "https://api.anthropic.com/v1", apiKey: "k", model: "claude-haiku-5-5", system: "s", prompt: "p", fetchFn });
+    assert.ok(r.ok, JSON.stringify(r));
+    assert.equal(bodies.length, 2);
+    assert.ok(!("temperature" in bodies[1]!));
+    // Remembered: the next ask goes straight out without it.
+    await askModel({ baseUrl: "https://api.anthropic.com/v1", apiKey: "k", model: "claude-haiku-5-5", system: "s", prompt: "p", fetchFn });
+    assert.equal(bodies.length, 3);
+    assert.ok(!("temperature" in bodies[2]!));
+  });
+
+  it("puts a 400's body in the error", async () => {
+    const fetchFn = (async () => new Response("bad tool_choice value", { status: 400 })) as unknown as typeof fetch;
+    const r = await askModel({ baseUrl: "https://x.test/v1", model: "m", system: "s", prompt: "p", fetchFn });
+    assert.ok(!r.ok && /bad tool_choice/.test(r.error));
+  });
+});

test/judge.test.ts

@@ -58,3 +58,14 @@ describe("Grok Build as the judge of an HTTP worker", () => {
     assert.deepEqual(r, { ok: true, text: '{"criteria":[]}', cutOff: false });
   });
 });
+
+describe("a judge on Anthropic's API", () => {
+  it("takes ANTHROPIC_API_KEY when the worker is elsewhere", () => {
+    const t = judgeTarget(
+      { baseUrl: "https://openrouter.ai/api/v1", apiKey: "or-key", model: "deepseek/deepseek-v4-pro" },
+      { MAAT_JUDGE_MODEL: "claude-haiku-5-5", MAAT_JUDGE_URL: "https://api.anthropic.com/v1", ANTHROPIC_API_KEY: "sk-ant-x" },
+    );
+    assert.equal(t.apiKey, "sk-ant-x");
+    assert.equal(t.model, "claude-haiku-5-5");
+  });
+});