diff --git a/CHANGELOG.md b/CHANGELOG.md index f7708fc..224693a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Measured-promotion gate + outcome-calibrated routing (`forge route calibrate`)** — + a reusable `src/promote.js` generalizes the risk predictor's kill-criteria: an + advisory signal (a calibrated weight, later a consolidation cluster or hazard + estimate) may become active **only** if it beats the current baseline on held-out + data under a metric+margin — the honesty register (overview §4), never an assertion. + First application: `forge route calibrate` fits an affine correction of the routing + rubric toward a held-out labeled fixture and promotes it only if it lowers held-out + MAE. Advisory by default — routing keeps the rubric until a promotion is adopted + (`calibratedComplexity` mirrors `predictor.riskFor`). Zero deps, fully unit-tested. - **Legacy-store retirement (`FORGE_LEDGER_ONLY`)** — the PCM ledger can now be the *only* store. Since P1 it has been the convergent write store (dual-write) with a merged read (`ledger_read`); with `FORGE_LEDGER_ONLY=1` the legacy files diff --git a/ROADMAP.md b/ROADMAP.md index c6bfbf2..c3e4585 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -66,9 +66,12 @@ confidence only from independent oracles, and merges across teammates conflict-f - **Playwright loop** — still open: interaction checks and feeding verdicts back as oracle evidence on design claims (fingerprinting itself shipped as `forge uicheck visual`). -- **Advisory → gated promotions** — outcome-calibrated routing weights, consolidation - promotion (ʿilm→fahm), M6 hazard estimates: advisory today, become blocking only - once fixtures measure them (overview §4 honesty register). +- **Advisory → gated promotions** — the measured-promotion gate has shipped + (`src/promote.js`, generalizing the risk predictor's kill-criteria): a candidate only + replaces a baseline when it beats it on held-out data, never by assertion. First + application: outcome-calibrated routing (`forge route calibrate`). Remaining + applications of the same gate: consolidation promotion (ʿilm→fahm) and M6 hazard + estimates. ## Later / exploring diff --git a/docs/GUIDE.md b/docs/GUIDE.md index a2a3147..be75885 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -204,6 +204,30 @@ weights). `ANTHROPIC_MODEL` / `FORGE_MODEL` override the tier choice entirely. Run `forge route gateway` to emit a LiteLLM config so the routing happens automatically. +**`forge route calibrate`** is the *advisory → gated promotion* (overview §4): it fits an +affine correction of the rubric's score toward a held-out labeled fixture and reports +whether that calibration **measurably** beats the raw rubric (lower held-out MAE past a +margin) — the same kill-criteria discipline as the risk predictor (`src/predictor.js`), +generalized in `src/promote.js` so any advisory signal (routing weights here; +consolidation and hazard next) can only become active by measurement, never by assertion. +It is advisory: routing keeps the rubric until a promoted calibration is explicitly +adopted. + +```console +$ forge route calibrate +Forge route calibrate — outcome-calibrated routing (measured gate) + + samples: 24 labeled task(s) + held-out MAE: rubric 0.152 · calibrated 0.226 + → keep the rubric — baseline retained — candidate did not beat it by the margin + + advisory — routing stays on the rubric until a promoted calibration is adopted +``` + +Here the gate does exactly its job: the rubric already generalizes well, the affine +calibration would make held-out error *worse*, so it is **refused**. A promotion only +happens when the measurement earns it — an assertion never does. + ### `forge config` — provider setup Shows, switches, and registers model providers, and sets the default model. Forge diff --git a/src/cli.js b/src/cli.js index 53c85df..537f63f 100755 --- a/src/cli.js +++ b/src/cli.js @@ -1133,6 +1133,28 @@ async function run(argv) { ); return; } + if (argv[1] === "calibrate") { + // Advisory → gated promotion (ROADMAP): measure whether an affine calibration of the + // routing rubric beats the raw rubric on the held-out fixture. Advisory — routing + // keeps the rubric unless the gate promotes AND a caller adopts the calibration. + const res = r.calibrateRouting(); + if (argv.includes("--json")) return console.log(JSON.stringify(res, null, 2)); + heading(`${BRAND.brand} route calibrate — outcome-calibrated routing (measured gate)\n`); + console.log(` samples: ${res.n} labeled task(s)`); + if (res.baselineMetric !== undefined) + console.log( + ` held-out MAE: rubric ${res.baselineMetric} · calibrated ${res.candidateMetric}`, + ); + console.log( + res.mode === "candidate" + ? ` → PROMOTE calibration — ${res.reason} (a=${res.model.a.toFixed(3)}, b=${res.model.b.toFixed(3)})` + : ` → keep the rubric — ${res.reason}`, + ); + console.log( + "\n advisory — routing stays on the rubric until a promoted calibration is adopted", + ); + return; + } const json = argv.includes("--json"); const apply = argv.includes("--apply"); const providerIdx = argv.indexOf("--provider"); diff --git a/src/promote.js b/src/promote.js new file mode 100644 index 0000000..4e7ad25 --- /dev/null +++ b/src/promote.js @@ -0,0 +1,98 @@ +// forge promote — the measured-promotion gate (the honesty register, overview §4). An +// advisory signal (a calibrated weight, a consolidation cluster, a hazard estimate) is +// allowed to become a BLOCKING/active signal ONLY if it MEASURABLY beats the current +// baseline on held-out data. This generalizes predictor.js's kill-criteria (`evaluate`) +// so any faculty can promote a candidate the same honest way instead of asserting a +// threshold. Pure, zero-dep — the decision is a metric comparison, never a claim. +import { aucPr } from "./predictor.js"; + +export { aucPr }; + +const round = (x) => Number(x.toFixed(3)); + +/** Mean absolute error over {score,label} pairs — the regression metric (lower is better). */ +export function mae(scored) { + if (!scored.length) return 0; + return scored.reduce((s, x) => s + Math.abs(x.score - x.label), 0) / scored.length; +} + +/** Default split: prequential (train on the past 80%, test on the held-out future 20%). */ +const prequential = (samples) => { + const cut = Math.floor(samples.length * 0.8); + return [samples.slice(0, cut), samples.slice(cut)]; +}; + +/** + * Decide whether a CANDIDATE may replace a BASELINE. Fits the candidate on the train + * split, scores both on the held-out test split, and promotes the candidate only if it + * beats the baseline by `margin` under `metric`. Cold-start (too few labels) or a + * candidate that fails to fit → keep the baseline (fail-safe, never throws). + * + * @template S,M + * @param {S[]} samples + * @param {{ + * baseline: (s: S) => number, + * fit: (train: S[]) => M, + * predict: (model: M, s: S) => number, + * label: (s: S) => number, + * metric?: (scored: {score:number,label:number}[]) => number, + * split?: (samples: S[]) => [S[], S[]], + * minSamples?: number, + * margin?: number, + * lowerIsBetter?: boolean, + * }} spec + * @returns {{mode:"baseline"|"candidate", n:number, reason:string, baselineMetric?:number, candidateMetric?:number, model?:M}} + */ +export function promotionGate(samples, spec) { + const { + baseline, + fit, + predict, + label, + metric = mae, + split = prequential, + minSamples = 20, + margin = 0.02, + lowerIsBetter = true, + } = spec; + + if (!Array.isArray(samples) || samples.length < minSamples) { + return { + mode: "baseline", + n: samples?.length ?? 0, + reason: "cold-start — not enough labels to measure a promotion", + }; + } + const [train, test] = split(samples); + if (!train.length || !test.length) { + return { mode: "baseline", n: samples.length, reason: "no held-out split to measure against" }; + } + + const baselineMetric = metric(test.map((s) => ({ score: baseline(s), label: label(s) }))); + let model; + try { + model = fit(train); + } catch { + return { + mode: "baseline", + n: samples.length, + reason: "candidate failed to fit — baseline retained", + baselineMetric: round(baselineMetric), + }; + } + const candidateMetric = metric(test.map((s) => ({ score: predict(model, s), label: label(s) }))); + const beats = lowerIsBetter + ? candidateMetric <= baselineMetric - margin + : candidateMetric >= baselineMetric + margin; + + return { + mode: beats ? "candidate" : "baseline", + n: samples.length, + reason: beats + ? `candidate beats baseline by ≥${margin} on held-out data` + : "baseline retained — candidate did not beat it by the margin", + baselineMetric: round(baselineMetric), + candidateMetric: round(candidateMetric), + model: beats ? model : undefined, + }; +} diff --git a/src/route.js b/src/route.js index 2b37a79..d2a0737 100644 --- a/src/route.js +++ b/src/route.js @@ -12,6 +12,7 @@ import { mergedLessons } from "./ledger_read.js"; import { setOverlap } from "./math.js"; import { MODELS } from "./model_tiers.js"; import { preflightRepo, referencedEntities } from "./preflight.js"; +import { promotionGate } from "./promote.js"; import { activeProvider, envModelOverride } from "./providers.js"; import { clamp01, contentHash, epochDay } from "./util.js"; @@ -216,6 +217,113 @@ export function rubricComplexity(task = "") { return { score, band, confidence, knn, neighbors, signals: sig, reasons, strongTopicSignal }; } +// --------------------------------------------------------------------------- +// Outcome-calibrated routing (ROADMAP: advisory → gated promotion). The rubric above +// is the advisory baseline. Below: fit an affine correction of its score toward labeled +// complexities and PROMOTE it over the raw rubric ONLY if it beats the rubric on a +// held-out fixture (promote.js measured gate) — never on assertion. recommend() keeps +// the raw rubric unless a caller opts into the returned calibration, so this stays +// advisory until the measurement earns the promotion (overview §4 honesty register). +// --------------------------------------------------------------------------- + +/** + * Held-out labeled complexities, DISTINCT from EXEMPLARS (the k-NN bank), so the gate + * measures generalization, not memorization. Interleaved by tier so any strided split is + * balanced. y matches the recommend() cutoffs: ~0.08 trivial · ~0.42 library-level · + * ~0.78 algorithmic/systems · ~0.85 architectural. + */ +export const CALIBRATION_SAMPLES = [ + { text: "print numbers from 1 to 100", y: 0.08 }, + { text: "implement a fixed-size ring buffer", y: 0.42 }, + { text: "detect a cycle in a directed graph", y: 0.78 }, + { text: "design a multi-tenant billing subsystem", y: 0.85 }, + { text: "trim whitespace from a string", y: 0.08 }, + { text: "group a list of records by a key", y: 0.42 }, + { text: "implement quicksort in place", y: 0.78 }, + { text: "plan a migration from a monolith to services", y: 0.85 }, + { text: "swap two variables", y: 0.08 }, + { text: "flatten a deeply nested array", y: 0.42 }, + { text: "build a thread-safe bounded blocking queue", y: 0.78 }, + { text: "architect an event-sourced order pipeline", y: 0.85 }, + { text: "return the length of an array", y: 0.08 }, + { text: "add pagination to a list query", y: 0.42 }, + { text: "write an lru eviction policy with o(1) operations", y: 0.78 }, + { text: "design cross-region data replication", y: 0.85 }, + { text: "convert a string to uppercase", y: 0.08 }, + { text: "build a simple event emitter class", y: 0.42 }, + { text: "parse arithmetic expressions with operator precedence", y: 0.78 }, + { text: "define the module boundaries for a new platform", y: 0.85 }, + { text: "add two integers", y: 0.08 }, + { text: "validate an email address format", y: 0.42 }, + { text: "coordinate leader election across nodes", y: 0.78 }, + { text: "design an auth system with roles and sessions", y: 0.85 }, +]; + +/** Least-squares affine calibration a·x + b mapping a rubric score x to the label y. Pure. */ +export function fitComplexityCalibration(train) { + const n = train.length; + if (!n) return { a: 1, b: 0 }; + const mx = train.reduce((s, r) => s + r.x, 0) / n; + const my = train.reduce((s, r) => s + r.y, 0) / n; + let num = 0; + let den = 0; + for (const r of train) { + num += (r.x - mx) * (r.y - my); + den += (r.x - mx) ** 2; + } + const a = den ? num / den : 1; + return { a, b: my - a * mx }; +} + +/** Apply an affine calibration, clamped to [0,1]. */ +export const applyCalibration = ({ a, b }, x) => clamp01(a * x + b); + +// Strided split (every 5th sample to the held-out test set): deterministic and, with the +// interleaved fixture, tier-balanced — no RNG (Math.random is unavailable to scripts). +/** @returns {[any[], any[]]} */ +const stridedSplit = (samples) => { + const train = []; + const test = []; + samples.forEach((s, i) => { + (i % 5 === 0 ? test : train).push(s); + }); + return [train, test]; +}; + +/** + * Run the measured-promotion gate on the routing rubric: fit an affine correction on the + * training split and promote it only if it lowers held-out MAE past the margin. + * @param {{text:string,y:number}[]} [samples] labeled tasks (default: the held-out fixture) + * @param {{margin?:number, minSamples?:number}} [opts] + */ +export function calibrateRouting(samples = CALIBRATION_SAMPLES, opts = {}) { + const enriched = samples.map((s) => ({ ...s, x: rubricComplexity(s.text).score })); + return promotionGate(enriched, { + baseline: (s) => s.x, + fit: fitComplexityCalibration, + predict: (model, s) => applyCalibration(model, s.x), + label: (s) => s.y, + split: stridedSplit, + minSamples: opts.minSamples ?? 20, + margin: opts.margin ?? 0.01, + lowerIsBetter: true, + }); +} + +/** + * The live complexity estimate: the calibrated mapping ONLY if the gate blessed it + * (mirrors predictor.riskFor). Falls back to the raw rubric otherwise. + * @param {string} task + * @param {{mode:string, model?:{a:number,b:number}}} [promotion] result of calibrateRouting + */ +export function calibratedComplexity(task, promotion) { + const base = rubricComplexity(task); + if (promotion?.mode === "candidate" && promotion.model) { + return { ...base, score: applyCalibration(promotion.model, base.score), calibrated: true }; + } + return { ...base, calibrated: false }; +} + const WEIGHTS = { files: 0.22, fanout: 0.22, diff --git a/test/promote.test.js b/test/promote.test.js new file mode 100644 index 0000000..bb0dd93 --- /dev/null +++ b/test/promote.test.js @@ -0,0 +1,110 @@ +// Measured-promotion gate (promote.js) + its concrete routing application (route.js). +// The gate promotes a candidate over a baseline ONLY when it measurably wins on held-out +// data — the honesty register's kill-criteria (overview §4), generalized from predictor. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mae, promotionGate } from "../src/promote.js"; +import { + applyCalibration, + CALIBRATION_SAMPLES, + calibratedComplexity, + calibrateRouting, + fitComplexityCalibration, +} from "../src/route.js"; + +test("mae is the mean absolute error; empty → 0", () => { + assert.equal( + mae([ + { score: 0.5, label: 1 }, + { score: 0.5, label: 0 }, + ]), + 0.5, + ); + assert.equal(mae([{ score: 1, label: 1 }]), 0); + assert.equal(mae([]), 0); +}); + +const altSamples = (n) => Array.from({ length: n }, (_, i) => ({ x: i % 2, y: i % 2 })); + +test("promotionGate: promotes a candidate that measurably beats the baseline", () => { + const res = promotionGate(altSamples(40), { + baseline: () => 0.5, // useless baseline (MAE 0.5) + fit: () => ({}), + predict: (_m, s) => s.x, // candidate == label → MAE 0 + label: (s) => s.y, + margin: 0.1, + minSamples: 20, + }); + assert.equal(res.mode, "candidate"); + assert.ok(res.candidateMetric < res.baselineMetric); + assert.ok(res.model); +}); + +test("promotionGate: keeps the baseline when the candidate does not beat it by the margin", () => { + const res = promotionGate(altSamples(40), { + baseline: (s) => s.x, // already perfect + fit: () => ({}), + predict: () => 0.5, // useless candidate + label: (s) => s.y, + margin: 0.05, + minSamples: 20, + }); + assert.equal(res.mode, "baseline"); + assert.equal(res.model, undefined); +}); + +test("promotionGate: cold-start (too few labels) keeps the baseline", () => { + const res = promotionGate([{ x: 0, y: 0 }], { + baseline: () => 0, + fit: () => ({}), + predict: () => 0, + label: (s) => s.y, + }); + assert.equal(res.mode, "baseline"); + assert.match(res.reason, /cold-start/); +}); + +test("promotionGate: a candidate that throws while fitting falls back to the baseline", () => { + const res = promotionGate(altSamples(40), { + baseline: (s) => s.x, + fit: () => { + throw new Error("boom"); + }, + predict: () => 0, + label: (s) => s.y, + }); + assert.equal(res.mode, "baseline"); + assert.match(res.reason, /failed to fit/); +}); + +test("fitComplexityCalibration recovers a known affine relation", () => { + const train = [ + { x: 0.1, y: 0.3 }, + { x: 0.2, y: 0.5 }, + { x: 0.3, y: 0.7 }, + { x: 0.4, y: 0.9 }, + ]; // y = 2x + 0.1 + const { a, b } = fitComplexityCalibration(train); + assert.ok(Math.abs(a - 2) < 1e-6); + assert.ok(Math.abs(b - 0.1) < 1e-6); + assert.ok(Math.abs(applyCalibration({ a, b }, 0.25) - 0.6) < 1e-6); +}); + +test("calibrateRouting returns a measured verdict over the held-out fixture", () => { + const res = calibrateRouting(); + assert.equal(res.n, CALIBRATION_SAMPLES.length); + assert.ok(res.mode === "baseline" || res.mode === "candidate"); + assert.equal(typeof res.baselineMetric, "number"); + assert.equal(typeof res.candidateMetric, "number"); +}); + +test("calibratedComplexity applies the calibration only when the gate promoted it", () => { + const kept = calibratedComplexity("fix a typo", { mode: "baseline" }); + assert.equal(kept.calibrated, false); + const promoted = calibratedComplexity("fix a typo", { + mode: "candidate", + model: { a: 0, b: 0.9 }, + }); + assert.equal(promoted.calibrated, true); + assert.ok(Math.abs(promoted.score - 0.9) < 1e-9); // a=0,b=0.9 → 0.9 regardless of the base +});