diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx new file mode 100644 index 00000000..a9eca2bf --- /dev/null +++ b/evalboard/app/_components/harness-badge.tsx @@ -0,0 +1,37 @@ +import Image from "next/image"; + +// Vendor logo for a run's harness (RunConfig). Renders the recognizable +// vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, +// mirroring the Slack rollup's vendor emoji. A missing harness defaults to +// claude-code (the nightly). This is an internal-only column — the caller +// gates it behind isInternal (see lib/edition.ts). +const HARNESS_LOGO: Record = { + "claude-code": { + src: "/harness/claude-code.png", + label: "Claude Code · Anthropic", + }, + codex: { src: "/harness/codex.png", label: "Codex · OpenAI" }, + antigravity: { + src: "/harness/antigravity.png", + label: "Antigravity · Google Gemini", + }, +}; + +export function HarnessBadge({ harness }: { harness?: string | null }) { + const key = harness ?? "claude-code"; + const logo = HARNESS_LOGO[key]; + // Unknown harness: show the raw id rather than a misleading logo. + if (!logo) { + return {key}; + } + return ( + {logo.label} + ); +} diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index dc57bb89..5cd66b92 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -15,6 +15,7 @@ import { ChipLegend, MergedTagRail } from "./_overview/tag-rail"; import { TableScroll } from "./_components/scroll-table"; import { CollapsibleRail } from "./_components/collapsible-rail"; import { isInternal } from "@/lib/edition"; +import { HarnessBadge } from "@/app/_components/harness-badge"; export const dynamic = "force-dynamic"; @@ -330,6 +331,11 @@ export default async function Page({ Run + {isInternal && ( + + Harness + + )} Pass rate @@ -363,6 +369,13 @@ export default async function Page({ {fmtRunTime(r.id)} + {isInternal && ( + + + + )} {isFiltered diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 9100682a..5fe056d6 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -5,7 +5,7 @@ import { describe, expect, test } from "vitest"; import { PRICING } from "../pricing"; // Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative -// Python table in src/coder_eval/proxy/pricing.py. If the backend reprices a +// Python table in src/coder_eval/pricing.py. If the backend reprices a // model (or adds one) and this mirror isn't updated, the frontend's "estimated" // USD figures silently disagree with the backend's authoritative Cost on the // same tokens. This test parses the Python table and asserts both tables have @@ -13,7 +13,7 @@ import { PRICING } from "../pricing"; // into a failing build. const here = dirname(fileURLToPath(import.meta.url)); -const PY_PATH = resolve(here, "../../../src/coder_eval/proxy/pricing.py"); +const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py"); // Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), const ROW_RE = @@ -36,7 +36,7 @@ function parsePythonTable(): Record< return out; } -describe("pricing.ts ↔ proxy/pricing.py parity", () => { +describe("pricing.ts ↔ pricing.py parity", () => { const py = parsePythonTable(); test("parses a non-trivial Python table", () => { diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index e0d47cc3..5fd0c719 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -15,6 +15,7 @@ import { type ArtifactRef, clearRunCacheDir, extractComponentShas, + extractRunConfig, findMatureSourceRuns, isExcludedArtifact, type MessageEvent, @@ -346,6 +347,41 @@ describe("extractComponentShas", () => { }); }); +describe("extractRunConfig", () => { + test("prefers the run-level RunConfig stamp (harness + environment)", () => { + const out = extractRunConfig({ + environment_info: { + run_config: { + harness: "codex", + model: "gpt-5.4", + environment: "prod", + }, + }, + }); + expect(out.harness).toBe("codex"); + expect(out.environment).toBe("prod"); + }); + + test("falls back to the most common per-task agent_config.type", () => { + const out = extractRunConfig({ + task_results: [ + { task_id: "a", agent_config: { type: "antigravity" } }, + { task_id: "b", agent_config: { type: "antigravity" } }, + { task_id: "c", agent_config: { type: "claude-code" } }, + ], + }); + expect(out.harness).toBe("antigravity"); + // environment is only known from the run-level stamp, never derived. + expect(out.environment).toBeNull(); + }); + + test("null harness when nothing identifies it (legacy runs)", () => { + const out = extractRunConfig({ task_results: [{ task_id: "a" }] }); + expect(out.harness).toBeNull(); + expect(out.environment).toBeNull(); + }); +}); + describe("findMatureSourceRuns", () => { // Newest-first run list with injected run.json readers (the deps test seam). const ids = ["r5", "r4", "r3", "r2", "r1"]; diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index f2cf064f..68c81568 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -95,6 +95,10 @@ export interface RunListingRow { tasksRun: number; totalCostUsd: number | null; taskDurationSeconds: number | null; + // Run-level harness (coder-eval AgentKind) for the Harness column; null on + // legacy runs that predate the RunConfig stamp / carry no agent_config.type. + // Optional so existing test factories stay valid. + harness?: string | null; } // Window-level rollup across every matched run (pre-limit), so the front-page @@ -680,6 +684,7 @@ export async function getRunListing( tasksRun: scopedTasks.length, totalCostUsd: scopedCost, taskDurationSeconds: scopedDur, + harness: overview.harness ?? null, }); } @@ -735,6 +740,7 @@ export function buildAdhocRows( tasksRun: overview.tasks.length, totalCostUsd: overview.totalCostUsd, taskDurationSeconds: overview.taskDurationSeconds, + // Harness is a main-table-only (internal) column; ad-hoc rows omit it. }); } rows.sort( diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 10fada2c..29f27360 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -1,6 +1,6 @@ // Per-million-token prices and cost math. Ported from -// src/coder_eval/proxy/pricing.py — keep in sync when that table changes. -// Source: Anthropic / OpenAI public pricing. +// src/coder_eval/pricing.py — keep in sync when that table changes. +// Source: Anthropic / OpenAI / Google public pricing. // // This is the single source of truth for rates on the frontend: the // cascade-aware thinking-cost simulator (lib/thinkingSim.ts) and the @@ -15,7 +15,7 @@ export interface Pricing { } // Exported so a unit test can assert key-and-rate parity against the -// authoritative Python table (src/coder_eval/proxy/pricing.py) and fail the +// authoritative Python table (src/coder_eval/pricing.py) and fail the // build on drift — this hand-copied mirror is otherwise guarded only by a // comment. Not part of the consumer API; use resolvePricing() instead. export const PRICING: Record = { @@ -41,6 +41,15 @@ export const PRICING: Record = { "gpt-5.3-codex": p(1.75, 14, 1.75, 0.175), "gpt-5.4": p(2.5, 15, 2.5, 0.25), "gpt-5.5": p(5, 30, 5, 0.5), + // Google Gemini (AntigravityAgent). Gemini bills no separate cache-write + // fee (cache_write == input, effectively unused); cache_read is the cached- + // input rate. Pro's >200K-token tier is higher — this flat rate reads low + // for very-large-context runs, fine for typical eval tasks. + "gemini-3-pro-preview": p(2, 12, 2, 0.2), + "gemini-3.1-pro-preview": p(2, 12, 2, 0.2), + "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), + "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), + "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), }; function p( diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 2e874940..26fa767e 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -346,6 +346,10 @@ interface RawTaskResult { // Model the task ran on (e.g. "claude-sonnet-4-6"). Used to price token // buckets as USD. Absent on legacy runs. model_used?: string | null; + // Per-task agent config; `type` is the harness (coder-eval AgentKind, e.g. + // "claude-code" | "codex" | "antigravity"). Used to derive the run's harness + // when the run-level RunConfig stamp is absent (direct coder-eval / legacy runs). + agent_config?: { type?: string | null } | null; // Activation enrichment the dashboard folds onto each case row in the nested // activation run.json (see cli.py _finalize_activation_run). Absent on skills // rows. `prompt` = the case's prompt text; `expected_skill` = the skill the @@ -790,6 +794,12 @@ export interface RunOverview { totalCostUsd: number | null; taskDurationSeconds: number | null; componentShas: ComponentSha[]; + // Run-level harness (coder-eval AgentKind) from the RunConfig stamp + // (environment_info.run_config), falling back to the most common per-task + // agent_config.type. `environment` (alpha/prod) is captured but intentionally + // NOT surfaced in the UI yet. Both optional so test factories predating them stay valid. + harness?: string | null; + environment?: string | null; // run.json `start_time` (ISO wall-clock). Pipeline runs derive their date // from the date-shaped id; ad-hoc ids carry no date, so the ad-hoc listing // orders by this instead. Optional so test factories predating it stay valid. @@ -814,6 +824,57 @@ export function visibleTurnsFromRaw(t: { return null; } +// The run's harness + environment. Prefers the run-level RunConfig stamped by +// eval_runner into environment_info.run_config; falls back to the most common +// per-task agent_config.type (direct coder-eval / legacy runs carry no stamp). +// Returns null harness when nothing identifies it (legacy pre-harness runs). +export function extractRunConfig(data: RawRunJson): { + harness: string | null; + environment: string | null; +} { + const rc = data.environment_info?.run_config; + if (rc && typeof rc === "object" && !Array.isArray(rc)) { + const r = rc as Record; + return { + // A present-but-partial stamp (run_config object with no/empty + // harness) falls back to the per-task agent_config vote rather + // than nulling out — otherwise a real codex/antigravity run is + // mislabeled as the claude-code default. + harness: + typeof r.harness === "string" && r.harness + ? r.harness + : mostCommonAgentType(data.task_results ?? []), + environment: + typeof r.environment === "string" ? r.environment : null, + }; + } + return { + harness: mostCommonAgentType(data.task_results ?? []), + environment: null, + }; +} + +// Most frequent per-task agent_config.type across a run's rows, or null if none +// declare one. A run is single-harness in practice, so this is just a robust +// "the harness these tasks ran on" for runs without the run-level stamp. +function mostCommonAgentType(rows: RawTaskResult[]): string | null { + const counts = new Map(); + for (const r of rows) { + const t = r.agent_config?.type; + if (typeof t === "string" && t) + counts.set(t, (counts.get(t) ?? 0) + 1); + } + let best: string | null = null; + let bestN = 0; + for (const [t, n] of counts) { + if (n > bestN) { + best = t; + bestN = n; + } + } + return best; +} + export async function readRunOverview( id: string, ): Promise { @@ -859,6 +920,7 @@ export async function readRunOverview( ? taskDurationSum : (data.total_duration_seconds ?? null), componentShas: extractComponentShas(data.environment_info), + ...extractRunConfig(data), startedAt: data.start_time ?? null, }; } diff --git a/evalboard/public/harness/antigravity.png b/evalboard/public/harness/antigravity.png new file mode 100644 index 00000000..c46f02e8 Binary files /dev/null and b/evalboard/public/harness/antigravity.png differ diff --git a/evalboard/public/harness/claude-code.png b/evalboard/public/harness/claude-code.png new file mode 100644 index 00000000..469637a8 Binary files /dev/null and b/evalboard/public/harness/claude-code.png differ diff --git a/evalboard/public/harness/codex.png b/evalboard/public/harness/codex.png new file mode 100644 index 00000000..bf3eadc0 Binary files /dev/null and b/evalboard/public/harness/codex.png differ