From 1e3bbf698968870e8111e1c34d587925aae2634d Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Wed, 14 Oct 2026 01:48:32 -0700 Subject: [PATCH] chore(models): one Gemini judge constant; analyze-reference honours GEMINI_JUDGE_MODEL lib/judge-model.mjs exports JUDGE_MODEL (GEMINI_JUDGE_MODEL || "gemini-3.6-flash", id unchanged). judge-video, judge-rubric, find-references and analyze-reference import it; analyze-reference previously embedded the literal in its URL and ignored the env var, so observer and judge could run on different models. Adds a request-shape test (fake fetch through the real scripts) and wires it into npm run check. NODE-LOOPS.md default corrected to gemini-3.6-flash. Co-Authored-By: Claude Opus 5.5 --- NODE-LOOPS.md | 2 +- analyze-reference.mjs | 3 +- find-references.mjs | 3 +- judge-rubric.mjs | 4 +- judge-video.mjs | 4 +- lib/judge-model.mjs | 3 ++ package.json | 3 +- test/judge-model-request.test.mjs | 63 ++++++++++++++++++++++++++++++ test/support/fake-gemini-fetch.mjs | 8 ++++ 9 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 lib/judge-model.mjs create mode 100644 test/judge-model-request.test.mjs create mode 100644 test/support/fake-gemini-fetch.mjs diff --git a/NODE-LOOPS.md b/NODE-LOOPS.md index c7586a5..4a8cb90 100644 --- a/NODE-LOOPS.md +++ b/NODE-LOOPS.md @@ -58,7 +58,7 @@ ffmpeg palette (`stats_mode=diff` + `lanczos` + `bayer` + `diff_mode=rectangle`) **The JUDGE = a separate video model, not the thing that drove the browser.** [`judge-video.mjs`](judge-video.mjs) sends the rendered MP4 (the pre-palette render, because GIF is not a supported Gemini video MIME) to Gemini video understanding -(`GEMINI_JUDGE_MODEL`, default `gemini-3.5-flash`) with the anti-hero-shot `RUBRIC`. It +(`GEMINI_JUDGE_MODEL`, default `gemini-3.6-flash`) with the anti-hero-shot `RUBRIC`. It returns strict JSON: per-dimension `{score, evidence}` **with timestamps**, a `defects` list (each `{ts, severity, observed, fix}`), and an overall `verdict`. It writes `out/.judge.json` + `out/.judge.md` (a scored table + timestamped defect diff --git a/analyze-reference.mjs b/analyze-reference.mjs index 99a1363..95c5b4e 100644 --- a/analyze-reference.mjs +++ b/analyze-reference.mjs @@ -20,6 +20,7 @@ // --competencies "multi-agent specialisation,context compression,HITL" --label cheiron-demo // import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { JUDGE_MODEL } from "./lib/judge-model.mjs"; const argv = process.argv.slice(2); const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; }; @@ -91,7 +92,7 @@ const call = async (win) => { ? `\nDecide competency_evidence for exactly these contested items: ${competencies}.` : "\nNo contested items were supplied; leave competency_evidence as an empty array rather than inventing items."; } - const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-3.6-flash:generateContent?key=${key()}`, { + const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${JUDGE_MODEL}:generateContent?key=${key()}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ diff --git a/find-references.mjs b/find-references.mjs index 390a17e..645731d 100644 --- a/find-references.mjs +++ b/find-references.mjs @@ -23,6 +23,7 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { JUDGE_MODEL } from "./lib/judge-model.mjs"; const OUT_DIR = "references/video"; const LEDGER = "references/video/ledger.json"; @@ -58,7 +59,7 @@ function videoId(url) { } const MAX_SECONDS = Number.parseInt(process.env.REFERENCE_MAX_SECONDS ?? "180", 10); const PER_QUERY = Number.parseInt(process.env.REFERENCE_PER_QUERY ?? "4", 10); -const MODEL = process.env.GEMINI_JUDGE_MODEL || "gemini-3.6-flash"; +const MODEL = JUDGE_MODEL; const key = () => { for (const k of ["GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY"]) if (process.env[k]) return process.env[k]; diff --git a/judge-rubric.mjs b/judge-rubric.mjs index dfd8f42..f8b0ed7 100644 --- a/judge-rubric.mjs +++ b/judge-rubric.mjs @@ -110,7 +110,7 @@ const run = async () => { const bytes = readFileSync(video); if (bytes.length > 19_000_000) throw new Error(`${(bytes.length / 1048576).toFixed(1)}MB > inline limit — use the Gemini Files API or render a smaller cut`); console.log(`[judge] ${video} — ${(bytes.length / 1048576).toFixed(1)}MB → gemini`); - const model = process.env.GEMINI_JUDGE_MODEL || "gemini-3.6-flash"; + const model = JUDGE_MODEL; const mime = video.endsWith(".webm") ? "video/webm" : video.endsWith(".mov") ? "video/quicktime" : "video/mp4"; const base = video.replace(/\.(mp4|webm|mov)$/i, ""); @@ -241,3 +241,5 @@ const run = async () => { } }; run().catch((e) => { console.error(e.message || e); process.exit(1); }); +// Imports are hoisted; kept at the end so the line anchors in .tours/ and docs/START_HERE.md stay put. +import { JUDGE_MODEL } from "./lib/judge-model.mjs"; diff --git a/judge-video.mjs b/judge-video.mjs index 928e7ef..1b27a71 100644 --- a/judge-video.mjs +++ b/judge-video.mjs @@ -180,7 +180,7 @@ const run = async () => { // did not include it. ListModels is not an existence proof — it was stale for that key, and a // direct generateContent call returns 200 with valid strict JSON. Absence from an index is // absence from an index. - const model = process.env.GEMINI_JUDGE_MODEL || "gemini-3.6-flash"; + const model = JUDGE_MODEL; const mime = video.endsWith(".webm") ? "video/webm" : video.endsWith(".mov") ? "video/quicktime" : "video/mp4"; const call = async (parts) => { const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${key()}`, { @@ -256,3 +256,5 @@ ${JSON.stringify(observed, null, 1)}` }, console.log(md); }; run().catch((e) => { console.error(e.message || e); process.exit(1); }); +// Imports are hoisted; kept at the end so the line anchors in .tours/ and docs/START_HERE.md stay put. +import { JUDGE_MODEL } from "./lib/judge-model.mjs"; diff --git a/lib/judge-model.mjs b/lib/judge-model.mjs new file mode 100644 index 0000000..39731b0 --- /dev/null +++ b/lib/judge-model.mjs @@ -0,0 +1,3 @@ +// The one Gemini judge/observer model id (SPEC-v3 §1.2 `google` wire id). GEMINI_JUDGE_MODEL pins another. +// Pinned rather than floating: a judge whose model changes underneath it produces verdicts that cannot be compared. +export const JUDGE_MODEL = process.env.GEMINI_JUDGE_MODEL || "gemini-3.6-flash"; diff --git a/package.json b/package.json index 9065a49..91b233c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "streamlit" ], "scripts": { - "check": "node check.mjs && npm run test:evidence && npm run proof", + "check": "node check.mjs && npm run test:evidence && npm run test:judge-model && npm run proof", "probe:opening": "node probe-opening-frame.mjs", "probe:maxpath": "node probe-max-path.mjs", "audit:ui": "node audit-ui.mjs", @@ -49,6 +49,7 @@ "retime": "node retime.mjs", "captions": "node apply-captions.mjs", "test:evidence": "node --test test/featureclip-evidence.test.mjs", + "test:judge-model": "node --test test/judge-model-request.test.mjs", "evidence:check": "node scripts/featureclip-evidence.mjs check", "evidence:project": "node scripts/featureclip-evidence.mjs project --write", "proof": "node scripts/featureclip-evidence.mjs proof" diff --git a/test/judge-model-request.test.mjs b/test/judge-model-request.test.mjs new file mode 100644 index 0000000..8104962 --- /dev/null +++ b/test/judge-model-request.test.mjs @@ -0,0 +1,63 @@ +// Request-shape test (model-defaults SPEC Part A §A.0.6 / §A.2.4): run each real Gemini-calling +// script as a subprocess with a fake fetch and assert the model in the request URL, and that the +// keys Gemini 3.6+ rejects are absent. Scenario: one operator sets GEMINI_JUDGE_MODEL to pin an +// older judge; the observer (analyze-reference) must follow the same pin as the judge scripts. +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const preload = pathToFileURL(path.join(root, "test/support/fake-gemini-fetch.mjs")).href; +const DEFAULT = "gemini-3.6-flash"; // SPEC-v3 §1.2 `google` wire id + +const scripts = { + "judge-video.mjs": (dir) => ["judge-video.mjs", path.join(dir, "clip.mp4"), "--no-reference"], + "judge-rubric.mjs": (dir) => ["judge-rubric.mjs", path.join(dir, "clip.mp4")], + "find-references.mjs": () => ["find-references.mjs", "--url=https://youtu.be/abcdefghijk"], + "analyze-reference.mjs": () => ["analyze-reference.mjs", "--url", "https://youtu.be/abcdefghijk"], +}; + +function firstRequest(script, judgeModel) { + const dir = mkdtempSync(path.join(tmpdir(), "fcs-judge-")); + mkdirSync(path.join(dir, "out")); + writeFileSync(path.join(dir, "clip.mp4"), Buffer.alloc(64)); + const log = path.join(dir, "requests.jsonl"); + const env = { ...process.env, GEMINI_API_KEY: "test-key", FAKE_FETCH_LOG: log }; + delete env.GEMINI_JUDGE_MODEL; + if (judgeModel) env.GEMINI_JUDGE_MODEL = judgeModel; + const [file, ...args] = scripts[script](dir); + // exit code is irrelevant: the fake answers "{}", so the script may fail after its first request + spawnSync(process.execPath, ["--import", preload, path.join(root, file), ...args], { cwd: dir, env, timeout: 30000 }); + assert.ok(existsSync(log), `${script} made no Gemini request`); + return JSON.parse(readFileSync(log, "utf8").split("\n")[0]); +} + +for (const script of Object.keys(scripts)) { + test(`${script}: default model is ${DEFAULT} and request carries no rejected keys`, () => { + const { url, body } = firstRequest(script); + assert.match(url, new RegExp(`/models/${DEFAULT}:generateContent\?`)); + const json = JSON.stringify(body); + for (const forbidden of ["thinking_budget", "thinkingBudget", "candidate_count", "candidateCount"]) { + assert.ok(!json.includes(forbidden), `${script} sends ${forbidden}`); + } + }); + test(`${script}: GEMINI_JUDGE_MODEL is honoured`, () => { + const { url } = firstRequest(script, "gemini-3.5-flash"); + assert.match(url, /\/models\/gemini-3\.5-flash:generateContent\?/); + }); +} + +test("lib/judge-model.mjs exports the one constant", async () => { + const saved = process.env.GEMINI_JUDGE_MODEL; + delete process.env.GEMINI_JUDGE_MODEL; + try { + const mod = await import(pathToFileURL(path.join(root, "lib/judge-model.mjs")).href + "?default"); + assert.equal(mod.JUDGE_MODEL, DEFAULT); + } finally { + if (saved !== undefined) process.env.GEMINI_JUDGE_MODEL = saved; + } +}); diff --git a/test/support/fake-gemini-fetch.mjs b/test/support/fake-gemini-fetch.mjs new file mode 100644 index 0000000..39327c7 --- /dev/null +++ b/test/support/fake-gemini-fetch.mjs @@ -0,0 +1,8 @@ +// Preloaded with `node --import` into the real judge scripts: records every Gemini request +// (URL + parsed body) to $FAKE_FETCH_LOG and answers with an empty JSON candidate. No network. +import { appendFileSync } from "node:fs"; +globalThis.fetch = async (url, init = {}) => { + appendFileSync(process.env.FAKE_FETCH_LOG, JSON.stringify({ url: String(url), body: JSON.parse(init.body ?? "null") }) + "\n"); + const payload = { candidates: [{ content: { parts: [{ text: "{}" }] } }] }; + return { ok: true, status: 200, json: async () => payload, text: async () => JSON.stringify(payload) }; +};