+ {/* No copy CTA on a rejected rewrite (#778): `result.bullets` is the
+ user's own text, so "Use this" would offer them what they already
+ have — and read as a contradiction of the notice above it. */}
+ {!result.reverted && (
+
0) parts.push(`removed ${formatTokens(dropped)}`);
- if (added.length > 0) parts.push(`invented ${formatTokens(added)}`);
- const detail = parts.join(" and ");
- return (
-
- ⚠
- AI altered a metric — {detail}. Review before saving.
-
- );
-}
-
-export function formatTokens(tokens: readonly string[]): string {
- if (tokens.length === 1) return tokens[0]!;
- if (tokens.length === 2) return `${tokens[0]} and ${tokens[1]}`;
- return `${tokens.slice(0, -1).join(", ")}, and ${tokens[tokens.length - 1]}`;
+/**
+ * A reject/revert is a warning, not a success, even though the delivered
+ * bullets preserve every number: the user asked for a rewrite and did not get
+ * one, and the panel has to read that way. `numbersPreserved` alone can't
+ * decide the tone after #778 — it is true on a reverted rewrite by
+ * construction.
+ */
+function resultTone(result: {
+ numbersPreserved: boolean;
+ reverted: boolean;
+}): "success" | "warning" {
+ return numberDriftStatus(result) === "clean" ? "success" : "warning";
}
diff --git a/src/design-system/shared/InlineDiff.test.tsx b/src/design-system/shared/InlineDiff.test.tsx
new file mode 100644
index 00000000..1ac8314a
--- /dev/null
+++ b/src/design-system/shared/InlineDiff.test.tsx
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The offlinecv Authors
+
+// @vitest-environment jsdom
+
+/**
+ * `noChangeLabel` (#778) is the only branch in `InlineDiff` — everything else
+ * is a straight map over segments — and it is the branch that decides whether
+ * a rejected rewrite reads as "nothing happened" or as "this was declined".
+ *
+ * jsdom via the pragma, raw `createRoot`, matching the feature-component tests.
+ */
+
+import { describe, expect, it, afterEach } from "vitest";
+import { createElement } from "react";
+import { createRoot, type Root } from "react-dom/client";
+import { act } from "react";
+
+(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT =
+ true;
+
+import { InlineDiff } from "./InlineDiff.tsx";
+import { computeTextDiff } from "../../lib/diff/text-diff.ts";
+
+let root: Root | null = null;
+let host: HTMLDivElement | null = null;
+
+function render(node: React.ReactElement): HTMLDivElement {
+ host = document.createElement("div");
+ document.body.appendChild(host);
+ root = createRoot(host);
+ act(() => {
+ root!.render(node);
+ });
+ return host;
+}
+
+afterEach(() => {
+ act(() => root?.unmount());
+ host?.remove();
+ root = null;
+ host = null;
+});
+
+describe("InlineDiff", () => {
+ it("renders every segment in order", () => {
+ const el = render(
+ createElement(InlineDiff, {
+ segments: computeTextDiff("Cut latency 40%.", "Cut p99 latency 40%."),
+ }),
+ );
+ expect(el.textContent).toContain("40%");
+ expect(el.querySelectorAll("span").length).toBeGreaterThan(1);
+ });
+
+ it("shows `noChangeLabel` when the two sides are identical", () => {
+ const el = render(
+ createElement(InlineDiff, {
+ segments: computeTextDiff("Cut latency 40%.", "Cut latency 40%."),
+ noChangeLabel: "No changes applied — your original bullets.",
+ }),
+ );
+ expect(el.textContent).toContain("No changes applied");
+ });
+
+ it("hides `noChangeLabel` the moment there is a real change to show", () => {
+ // The redline speaks for itself when there is one; the caption exists only
+ // for the case where there is nothing on screen to explain the outcome.
+ const el = render(
+ createElement(InlineDiff, {
+ segments: computeTextDiff("Cut latency 40%.", "Cut latency."),
+ noChangeLabel: "No changes applied — your original bullets.",
+ }),
+ );
+ expect(el.textContent).not.toContain("No changes applied");
+ });
+
+ it("renders an unchanged diff exactly as before when no label is passed", () => {
+ const el = render(
+ createElement(InlineDiff, {
+ segments: computeTextDiff("Cut latency 40%.", "Cut latency 40%."),
+ }),
+ );
+ expect(el.querySelector("div")).toBeNull();
+ expect(el.textContent).toBe("Cut latency 40%.");
+ });
+});
diff --git a/src/design-system/shared/InlineDiff.tsx b/src/design-system/shared/InlineDiff.tsx
index d7e2aa2a..f8e5f262 100644
--- a/src/design-system/shared/InlineDiff.tsx
+++ b/src/design-system/shared/InlineDiff.tsx
@@ -10,8 +10,19 @@
* two-column "Original | Proposed" grid.
*
* Props:
- * `segments` — output of `computeTextDiff` from `src/lib/diff/text-diff.ts`
- * `className` — extra classes for the outer block (width, margin, etc.)
+ * `segments` — output of `computeTextDiff` from `src/lib/diff/text-diff.ts`
+ * `className` — extra classes for the outer block (width, margin, etc.)
+ * `noChangeLabel` — caption shown ONLY when the diff has no added/removed
+ * run, i.e. when the two sides are identical
+ *
+ * `noChangeLabel` exists because an all-equal diff renders as plain prose that
+ * is indistinguishable from any other block of text: the reader cannot tell
+ * "these two are the same" from "this is just some text". Whenever a caller
+ * produced the two sides by a process that could have changed them and did
+ * not, that outcome is information, and the component that owns the redline is
+ * the one place that can tell there is nothing to draw. Domain-agnostic on
+ * purpose — the caller supplies the sentence (#778's rewrite-rejected notice
+ * is the first one), this only decides when it is shown.
*
* Rendering notes:
* - Outer element is a `` (inline text content, not a structural section).
@@ -35,13 +46,23 @@ interface InlineDiffProps {
segments: DiffSegment[];
/** Extra classes applied to the outer block — width, overflow, etc. */
className?: string;
+ /**
+ * Caption rendered above the text when the diff contains no `added` or
+ * `removed` segment. Omitted → an unchanged diff renders exactly as before.
+ */
+ noChangeLabel?: React.ReactNode;
}
-export function InlineDiff({ segments, className }: InlineDiffProps) {
+export function InlineDiff({
+ segments,
+ className,
+ noChangeLabel,
+}: InlineDiffProps) {
const base =
"whitespace-pre-wrap break-words text-sm leading-snug";
const cls = className ? `${base} ${className}` : base;
- return (
+ const unchanged = segments.every((seg) => seg.type === "equal");
+ const body = (
{segments.map((seg, i) => (
@@ -50,4 +71,15 @@ export function InlineDiff({ segments, className }: InlineDiffProps) {
))}
);
+
+ if (noChangeLabel === undefined || !unchanged) return body;
+
+ return (
+
+
+ {noChangeLabel}
+
+ {body}
+
-)?(?[$€£¥])?(?\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+|\d+)(?:(?[kKmMbBgGtT][bB]?)|(?[xX]))?(?%)?(?\+)?(?!\w)/g;
+
+/**
+ * Dash characters that can join the two endpoints of a numeric range:
+ * ASCII hyphen-minus, the U+2010–U+2015 dash block (hyphen, non-breaking
+ * hyphen, figure dash, en dash, em dash, horizontal bar) and U+2212 MINUS
+ * SIGN. A PDF extractor emits any of these where the author typed one dash,
+ * so recognising only `-` would make range detection depend on the font.
+ */
+const RANGE_DASH = /[\u002D\u2010-\u2015\u2212]/;
/**
* Verbs/phrasing that signal a bare integer is a headcount when they appear
@@ -59,56 +176,135 @@ const PEOPLE_VERB_PREFIX =
/**
* Nouns that signal a bare integer is a headcount when they appear just
- * after the digit. Anchored to `^\s*` so the noun has to be the first token
- * of the suffix slice.
+ * after the digit. Anchored to the start of the suffix slice, so the noun has
+ * to be its first token — separated by whitespace OR by one of `RANGE_DASH`'s
+ * dashes, because `12-person team` is the same claim as `12 people`, only
+ * attributively attached. Reading the hyphenated form as no claim at all is
+ * what made the legitimate reverse rewording (`a 12-person team` →
+ * `12 people`) look like an invented headcount under rule 3.
*/
-const PEOPLE_NOUN_FOLLOW =
- /^\s*(?:engineers?|developers?|designers?|analysts?|interns?|reports?|people|persons?|members?|employees?|contractors?|consultants?|staff|hires?|recruits?)\b/i;
+const PEOPLE_NOUN_FOLLOW = new RegExp(
+ `^(?:\\s|${RANGE_DASH.source})*(?:engineers?|developers?|designers?|analysts?|interns?|reports?|people|persons?|members?|employees?|contractors?|consultants?|staff|hires?|recruits?)\\b`,
+ "i",
+);
+
+/**
+ * What a `PEOPLE_VERB_PREFIX` match is allowed to run into when there is no
+ * people noun to confirm it: end-of-string, punctuation, or a closed list of
+ * prepositions/conjunctions. Anchored to the start of the suffix slice, same
+ * as `PEOPLE_NOUN_FOLLOW`.
+ *
+ * This is what stops the verb alone from claiming a headcount when a
+ * NON-people noun follows it — `Managed 5 projects` — where the verb is
+ * pointing at that noun, not at a headcount. `Managed 8 across the data
+ * platform` (noun elided) still qualifies, because `across` is in the list.
+ */
+const FUNCTION_WORD_FOLLOWS = new RegExp(
+ "^(?:\\s*$|\\s*[.,;:!?)]|\\s+(?:across|in|on|at|for|to|from|over|within|during|and|with|through|by|per)\\b)",
+ "i",
+);
/** How many characters of context to inspect on each side of a bare integer. */
const PEOPLE_CONTEXT_WINDOW = 30;
+/**
+ * WHY an atom is a numeric fact the guardrail defends — `"none"` when it is
+ * just a digit that happened to be in the text (`phase 2`, `1 of 5
+ * applicants`).
+ *
+ * The reason, not only the boolean, is load-bearing on the add side (rule 3):
+ * a value the OUTPUT reads as a headcount while the INPUT read it as nothing
+ * is a claim the model minted, whereas the same value merely re-grouped
+ * (`"form"`: `1200` → `1,200`) or re-spelled as a range (`"range"`: `50 to 100`
+ * → `50-100`) is the claim the user already made. `"form"` is the number's own
+ * written shape ($, %, x, ~, +, a magnitude suffix, grouping commas, a decimal
+ * point); the other three are the prose around a bare integer.
+ */
+type NumericClaim = "none" | "form" | "headcount" | "year" | "range";
+
interface ClassifiedAtom {
/** Match key used for set equality (lowercased so `$5K` ≡ `$5k`). */
key: string;
/** Human-readable form used in the UI warning (preserves original case). */
display: string;
+ /**
+ * Why this counts as a fact worth defending, or `"none"`. Unclaimed atoms are
+ * never reported as dropped or added, but they still count as PRESENT for the
+ * other side's lookup — see rules 1 and 3 in the module docblock.
+ */
+ claim: NumericClaim;
}
-function classifyAtom(
+function isClaimed(atom: ClassifiedAtom): boolean {
+ return atom.claim !== "none";
+}
+
+/**
+ * Is this match one endpoint of a tight numeric range (`50-100`)?
+ *
+ * Both directions have to be checked, because a range contributes two separate
+ * atoms and each has to be recognised from its own position: the left endpoint
+ * is followed by `dash digit`, the right endpoint is preceded by `digit dash`.
+ *
+ * "Tight" is load-bearing — `digit dash digit` with no whitespace. It is what
+ * separates a range from the two things that look like one:
+ * - `6-month window` / `day-7 retention` — the dash is followed by a letter
+ * that is not a people noun, so neither is a range and both bare integers
+ * stay unclaimed, as before.
+ * - `50 -100` — the ATOM already reads `-100` as a signed number, and it
+ * never reaches this check because a sign claims the atom verbatim.
+ */
+function isRangeEndpoint(match: RegExpExecArray, bullet: string): boolean {
+ const start = match.index;
+ const end = start + match[0].length;
+ const leftEndpoint =
+ RANGE_DASH.test(bullet[end] ?? "") && /\d/.test(bullet[end + 1] ?? "");
+ const rightEndpoint =
+ RANGE_DASH.test(bullet[start - 1] ?? "") &&
+ /\d/.test(bullet[start - 2] ?? "");
+ return leftEndpoint || rightEndpoint;
+}
+
+/**
+ * Does this atom carry a decoration that makes it a numeric fact on its own?
+ * Approximation, sign, currency, magnitude, multiplier, `%` and `+` all change
+ * what the number claims, so any of them is enough — no context needed.
+ */
+function isDecorated(groups: Record): boolean {
+ return (
+ groups.approx !== undefined ||
+ groups.sign !== undefined ||
+ groups.currency !== undefined ||
+ groups.magnitude !== undefined ||
+ groups.multiplier !== undefined ||
+ groups.percent !== undefined ||
+ groups.plus !== undefined
+ );
+}
+
+/**
+ * What, if anything, makes this bare integer worth defending, given the words
+ * around it?
+ *
+ * Three ways to qualify — a headcount, a year, or one endpoint of a tight
+ * range. Everything else ("the 3 of us", "phase 2", "section 4") is noise: it
+ * still produces an atom, so the other side can match against it, but it is
+ * never itself reported as dropped or added.
+ *
+ * The match index IS the digit index on this branch: every prefix decoration
+ * (approximation, sign, currency) implies `isDecorated`, so a caller that
+ * reaches here has nothing between `match.index` and the first digit.
+ *
+ * Years are checked before ranges, which is what keeps `2019-2021` scoring as a
+ * pair of years rather than a range — the two classifications now share the
+ * `num:` key namespace, but a 4-digit year qualifies without needing a dash, so
+ * `2019` in "between 2019 and 2021" stays claimed.
+ */
+function bareIntegerClaim(
match: RegExpExecArray,
bullet: string,
-): ClassifiedAtom | null {
- const [, sign, currency, digits, magnitude, percent] = match;
- const hasSign = Boolean(sign);
- const hasCurrency = Boolean(currency);
- const hasMagnitude = Boolean(magnitude);
- const hasPercent = Boolean(percent);
-
- const display =
- (hasSign ? "-" : "") +
- (hasCurrency ? currency! : "") +
- digits +
- (hasMagnitude ? magnitude! : "") +
- (hasPercent ? "%" : "");
- const key = display.toLowerCase();
-
- // Decorated tokens (currency / magnitude / % / comma groups / decimals
- // / explicit sign) are always tracked verbatim — they are unambiguously
- // meaningful numeric facts.
- if (
- hasCurrency ||
- hasMagnitude ||
- hasPercent ||
- hasSign ||
- digits.includes(",") ||
- digits.includes(".")
- ) {
- return { key, display };
- }
-
- // Bare integer. Inspect surrounding context to decide whether it's a
- // headcount, a year, or noise we should ignore.
+ digits: string,
+): NumericClaim {
const matchStart = match.index;
const before = bullet.slice(
Math.max(0, matchStart - PEOPLE_CONTEXT_WINDOW),
@@ -119,20 +315,73 @@ function classifyAtom(
matchStart + digits.length + PEOPLE_CONTEXT_WINDOW,
);
- if (PEOPLE_VERB_PREFIX.test(before) || PEOPLE_NOUN_FOLLOW.test(after)) {
- return { key: `headcount:${digits}`, display: digits };
+ // A people noun after the digit is decisive on its own.
+ if (PEOPLE_NOUN_FOLLOW.test(after)) {
+ return "headcount";
+ }
+ // The verb alone qualifies only when NO noun follows — the digit runs into
+ // a preposition, punctuation, or the end of the bullet. A non-people noun
+ // right after the digit (`Managed 5 projects`) means the verb is modifying
+ // that noun, not asserting a headcount.
+ if (PEOPLE_VERB_PREFIX.test(before) && FUNCTION_WORD_FOLLOWS.test(after)) {
+ return "headcount";
}
if (digits.length === 4) {
const year = Number(digits);
- if (year >= 1900 && year <= 2099) {
- return { key: `year:${digits}`, display: digits };
- }
+ if (year >= 1900 && year <= 2099) return "year";
}
- // Plain integer without people context or a year shape — too noisy to
- // track. Examples: "the 3 of us", "phase 2", "section 4".
- return null;
+ // A range endpoint (#778). `50-100 tickets` used to track NEITHER number:
+ // both are bare integers with no people context and no year shape, so a
+ // rewrite could drop the whole range and the guardrail scored it clean. The
+ // two endpoints are two independent atoms rather than one `50-100` atom, so a
+ // rewrite that re-spells the dash (`50–100`) or the whole range (`50 to 100`)
+ // still matches — the dash is detection context, never part of the key.
+ return isRangeEndpoint(match, bullet) ? "range" : "none";
+}
+
+function classifyAtom(match: RegExpExecArray, bullet: string): ClassifiedAtom {
+ const g = match.groups!;
+ const digits = g.digits!;
+
+ const prefix = (g.approx ?? "") + (g.sign ?? "") + (g.currency ?? "");
+ const suffix =
+ (g.magnitude ?? g.multiplier ?? "") + (g.percent ?? "") + (g.plus ?? "");
+ const display = prefix + digits + suffix;
+
+ // Grouping commas are presentation, not value: `1,200` and `1200` are the
+ // same figure spelled two ways. The KEY drops them so the two spellings
+ // match; `display` keeps whichever the author (or the model) wrote, because
+ // that is what the warning copy quotes back. Keying on the literal spelling
+ // is what made "Processed 1,200 orders" → "Processed 1200 orders" report a
+ // dropped `1,200` and revert a rewrite that changed nothing but the comma.
+ const value = digits.replace(/,/g, "");
+
+ if (isDecorated(g)) {
+ return {
+ key: (prefix + value + suffix).toLowerCase(),
+ display,
+ claim: "form",
+ };
+ }
+
+ // Undecorated. ONE key namespace regardless of what (if anything) qualified
+ // it — a headcount `12`, a year `2019`, a range endpoint `50` and a grouped
+ // `1,200` are the same value as the same digits written with no context and
+ // no commas, and splitting them by namespace is what made `Managed 12 people`
+ // → `Managed a 12-person team` report a dropped `12`.
+ return {
+ key: `num:${value}`,
+ display,
+ // Grouping and a decimal point are the figure's own written shape, so they
+ // claim it without needing context: `1,200` and `3.14` were written as
+ // figures on purpose.
+ claim:
+ digits.includes(",") || digits.includes(".")
+ ? "form"
+ : bareIntegerClaim(match, bullet, digits),
+ };
}
function extractNumbers(bullets: readonly string[]): ClassifiedAtom[] {
@@ -141,8 +390,7 @@ function extractNumbers(bullets: readonly string[]): ClassifiedAtom[] {
ATOM.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = ATOM.exec(bullet)) !== null) {
- const token = classifyAtom(match, bullet);
- if (token !== null) tokens.push(token);
+ tokens.push(classifyAtom(match, bullet));
}
}
return tokens;
@@ -156,49 +404,106 @@ export interface PreservationResult {
added: string[];
}
+/**
+ * Every claimed atom in `atoms` that `isPresent` cannot find on the other side,
+ * reported by its display form in first-encounter order, at most once per key.
+ *
+ * The presence test is a parameter rather than a set lookup because the two
+ * directions ask different questions of the other side (rule 3): a drop asks
+ * only whether the value survived anywhere, an invention asks whether the claim
+ * was there to begin with. Membership is set-shaped, not counted (rule 2).
+ */
+function missingFrom(
+ atoms: readonly ClassifiedAtom[],
+ isPresent: (atom: ClassifiedAtom) => boolean,
+): string[] {
+ const reported = new Set();
+ const missing: string[] = [];
+ for (const atom of atoms) {
+ if (!isClaimed(atom)) continue;
+ if (reported.has(atom.key) || isPresent(atom)) continue;
+ reported.add(atom.key);
+ missing.push(atom.display);
+ }
+ return missing;
+}
+
+/** How many UNCLAIMED atoms share each key — the drop side's masking guard. */
+function countUnclaimedByKey(
+ atoms: readonly ClassifiedAtom[],
+): Map {
+ const counts = new Map();
+ for (const atom of atoms) {
+ if (isClaimed(atom)) continue;
+ counts.set(atom.key, (counts.get(atom.key) ?? 0) + 1);
+ }
+ return counts;
+}
+
/**
* Check that every numeric fact from the input bullets survives into the
* rewritten bullets, and that no new numeric fact was invented.
*
- * Multiset semantics: two `5%`s in the input must both appear in the output.
- * Diff lists preserve the order tokens were encountered, so the UI can quote
- * them back to the user without sorting noise. Tokens are returned in their
- * original casing (`$5K`, not `$5k`).
+ * Set semantics: a number counts as preserved while it appears at all, so a
+ * licensed merge that de-duplicates a repeated metric is not a drop. Diff lists
+ * preserve the order tokens were encountered, so the UI can quote them back to
+ * the user without sorting noise. Tokens are returned in their original casing
+ * (`$5K`, not `$5k`).
*/
export function checkNumbersPreserved(
input: readonly string[],
output: readonly string[],
): PreservationResult {
- const inputTokens = extractNumbers(input);
- const outputTokens = extractNumbers(output);
+ const inputAtoms = extractNumbers(input);
+ const outputAtoms = extractNumbers(output);
- const outputCounts = new Map();
- for (const t of outputTokens) {
- outputCounts.set(t.key, (outputCounts.get(t.key) ?? 0) + 1);
- }
- const dropped: string[] = [];
- for (const t of inputTokens) {
- const remaining = outputCounts.get(t.key) ?? 0;
- if (remaining === 0) {
- dropped.push(t.display);
- } else {
- outputCounts.set(t.key, remaining - 1);
- }
- }
+ const inputKeys = new Set(inputAtoms.map((a) => a.key));
+ const outputKeys = new Set(outputAtoms.map((a) => a.key));
+ // Only the values the INPUT itself asserted as facts. A headcount the output
+ // asserts has to land in here, not merely in `inputKeys` — see rule 3.
+ const inputClaimedKeys = new Set(
+ inputAtoms.filter(isClaimed).map((a) => a.key),
+ );
+ const outputClaimedKeys = new Set(
+ outputAtoms.filter(isClaimed).map((a) => a.key),
+ );
+ // Per-key counts of UNCLAIMED occurrences on each side — the baseline of
+ // "this digit shows up elsewhere for unrelated reasons" that already
+ // existed before the rewrite touched anything. Only headcount needs this:
+ // it is the one claim kind with a real unclaimed state on the same key
+ // (`num:12` from "team of 12" vs `num:12` from "module 12" of a curriculum).
+ const inputUnclaimedCounts = countUnclaimedByKey(inputAtoms);
+ const outputUnclaimedCounts = countUnclaimedByKey(outputAtoms);
- const inputCounts = new Map();
- for (const t of inputTokens) {
- inputCounts.set(t.key, (inputCounts.get(t.key) ?? 0) + 1);
- }
- const added: string[] = [];
- for (const t of outputTokens) {
- const remaining = inputCounts.get(t.key) ?? 0;
- if (remaining === 0) {
- added.push(t.display);
- } else {
- inputCounts.set(t.key, remaining - 1);
- }
- }
+ // Drop: the value is gone from the output in every spelling (rule 1) —
+ // except for a headcount, which counts as present only if the output
+ // claims it too, OR a NEW unclaimed occurrence of the key appears that
+ // wasn't already there before the rewrite (the "phase 5" masking case
+ // below, which rule 1 is right to treat as a reword, not a loss). Without
+ // the "new" qualifier, an unrelated digit the input ALREADY carried
+ // unclaimed (e.g. "module 12" sitting next to a genuinely-dropped
+ // "12 engineers") would mask the drop just by surviving unchanged — the
+ // masking bug rule 1's blanket `outputKeys.has` used to have. `form`/
+ // `range`/`year` keep the fully lenient lookup: those claims genuinely
+ // depend on prose a rewrite is licensed to move (or, for `year`, have no
+ // unclaimed state to compare against at all), so tightening them risks a
+ // false revert on a legitimate reword rule 1 exists to allow.
+ const dropped = missingFrom(inputAtoms, (atom) => {
+ if (atom.claim !== "headcount") return outputKeys.has(atom.key);
+ if (outputClaimedKeys.has(atom.key)) return true;
+ return (
+ (outputUnclaimedCounts.get(atom.key) ?? 0) >
+ (inputUnclaimedCounts.get(atom.key) ?? 0)
+ );
+ });
+ // Invention: the same lookup, except that a headcount the output asserts is
+ // "present" only if the input asserted that value too (rule 3). `phase 5` →
+ // `5 engineers` reuses the digit while making a claim the résumé never made.
+ const added = missingFrom(outputAtoms, (atom) =>
+ atom.claim === "headcount"
+ ? inputClaimedKeys.has(atom.key)
+ : inputKeys.has(atom.key),
+ );
return {
ok: dropped.length === 0 && added.length === 0,
diff --git a/src/lib/webllm/rewrite-guardrails.ts b/src/lib/webllm/rewrite-guardrails.ts
index cebb6ae5..f7263e44 100644
--- a/src/lib/webllm/rewrite-guardrails.ts
+++ b/src/lib/webllm/rewrite-guardrails.ts
@@ -33,7 +33,7 @@
* The single most important rule in the tree, and the one #609 required to
* have exactly one definition.
*
- * The deterministic `checkNumbersPreserved` multiset diff is what actually
+ * The deterministic `checkNumbersPreserved` set diff is what actually
* catches a violation after the fact; this sentence is what stops the model
* producing one. Both halves are needed — the check can only warn, it cannot
* repair — so neither is redundant with the other.
diff --git a/src/lib/webllm/rewrite-resume.test.ts b/src/lib/webllm/rewrite-resume.test.ts
index d5d5425d..0beba8f7 100644
--- a/src/lib/webllm/rewrite-resume.test.ts
+++ b/src/lib/webllm/rewrite-resume.test.ts
@@ -113,6 +113,7 @@ describe("buildResumeContext", () => {
data: {
bullets: ["Shipped Foo to 10M users."],
numbersPreserved: true,
+ reverted: false,
droppedNumbers: [],
addedNumbers: [],
},
@@ -132,6 +133,7 @@ describe("buildResumeContext", () => {
data: {
bullets: [long],
numbersPreserved: true,
+ reverted: false,
droppedNumbers: [],
addedNumbers: [],
},
@@ -249,7 +251,7 @@ describe("rewriteResumeWithLlm", () => {
);
});
- it("aggregates allNumbersPreserved across sections", async () => {
+ it("reverts the one section that dropped a metric, leaving the others rewritten (#778)", async () => {
const { engine } = makeEngine(async (req: ChatCompletionRequest) => {
// Drop a metric on the second call only.
const ord = req.messages[1]!.content;
@@ -266,13 +268,45 @@ describe("rewriteResumeWithLlm", () => {
TEST_MODEL,
() => {},
);
- expect(result.allNumbersPreserved).toBe(false);
- expect(result.sections[0]!.kind === "experience" && result.sections[0]!.data.numbersPreserved).toBe(
- true,
+ const [first, second] = result.sections;
+ expect(first!.kind === "experience" && first!.data.reverted).toBe(false);
+ expect(first!.kind === "experience" && first!.data.bullets).toEqual([
+ "Drove $1.2M ARR.",
+ ]);
+ // The dropping section keeps the user's own bullet — one bad section does
+ // not cost them the rest of the run.
+ expect(second!.kind === "experience" && second!.data.reverted).toBe(true);
+ expect(second!.kind === "experience" && second!.data.bullets).toEqual([
+ "Saved $5K per quarter",
+ ]);
+ // Every number reached the user, so the aggregate is true — which is why
+ // the UI reads `reverted` and not this flag alone.
+ expect(result.allNumbersPreserved).toBe(true);
+ });
+
+ it("reverts a section that invents a metric, and still reports the token", async () => {
+ // Invention is gated too since the #778 widening, so the aggregate reads
+ // true (the original invents nothing) and `reverted` is what carries the
+ // story — the same pairing the drop case already had.
+ const { engine } = makeEngine(async () =>
+ reply("Drove revenue with 99.9% availability."),
);
- expect(result.sections[1]!.kind === "experience" && result.sections[1]!.data.numbersPreserved).toBe(
- false,
+ const sections: SectionInput[] = [
+ experienceSection("experience:0", "Acme", ["Drove revenue"]),
+ ];
+ const result = await rewriteResumeWithLlm(
+ sections,
+ engine,
+ TEST_MODEL,
+ () => {},
);
+ expect(result.sections[0]!.data.reverted).toBe(true);
+ const section = result.sections[0]!;
+ expect(section.kind === "experience" && section.data.bullets).toEqual([
+ "Drove revenue",
+ ]);
+ expect(result.sections[0]!.data.addedNumbers).toEqual(["99.9%"]);
+ expect(result.allNumbersPreserved).toBe(true);
});
it("fires webllm_resume_rewrite_started and _completed exactly once per run", async () => {
@@ -291,6 +325,7 @@ describe("rewriteResumeWithLlm", () => {
model: TEST_MODEL,
sectionCount: 1,
allNumbersPreserved: true,
+ anyReverted: false,
});
});
@@ -309,6 +344,7 @@ describe("rewriteResumeWithLlm", () => {
inputUnitCount: 1,
outputUnitCount: 1,
numbersPreserved: true,
+ reverted: false,
});
expect(sectionCompletedMock).toHaveBeenNthCalledWith(2, {
model: TEST_MODEL,
@@ -317,9 +353,40 @@ describe("rewriteResumeWithLlm", () => {
inputUnitCount: 1,
outputUnitCount: 1,
numbersPreserved: true,
+ reverted: false,
});
});
+ it("reports the MODEL's number preservation, not the delivered outcome, on a revert", async () => {
+ // The delivered bullets are the user's own after a revert, so
+ // `data.numbersPreserved` is true by construction. Passing that through
+ // would flip what `numbers_preserved` measures mid-release; the series has
+ // to keep describing the model. `reverted` is the new dimension that says
+ // the gate fired. Mirrors `rewrite-section.ts`.
+ const { engine } = makeEngine(async () => reply("Drove revenue."));
+ const sections: SectionInput[] = [
+ experienceSection("experience:0", "Acme", ["Drove $1.2M in ARR"]),
+ ];
+ const result = await rewriteResumeWithLlm(
+ sections,
+ engine,
+ TEST_MODEL,
+ () => {},
+ );
+ expect(result.sections[0]!.data.reverted).toBe(true);
+ expect(sectionCompletedMock).toHaveBeenCalledWith(
+ expect.objectContaining({ numbersPreserved: false, reverted: true }),
+ );
+ expect(completedMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ allNumbersPreserved: false,
+ anyReverted: true,
+ }),
+ );
+ // The value the UI reads is unchanged — it is a property of what shipped.
+ expect(result.allNumbersPreserved).toBe(true);
+ });
+
it("fires webllm_first_resume_rewrite exactly once per model", async () => {
const { engine } = makeEngine(async () => reply("Built X."));
const sections: SectionInput[] = [
@@ -490,8 +557,8 @@ describe("app findings reach the rewrite prompt (#608)", () => {
it("still catches a fabricated number when a finding names one", async () => {
// A finding is allowed to mention a number (a critique suggestion routinely
- // does). If the model copies it into the résumé, `allNumbersPreserved` must
- // still report the invention — a finding must not become a fabrication
+ // does). If the model copies it into the résumé, the gate must still see
+ // and reject the invention — a finding must not become a fabrication
// vector that the deterministic checker stops seeing.
const numeric = new Map([
[
@@ -510,10 +577,15 @@ describe("app findings reach the rewrite prompt (#608)", () => {
() => {},
{ findings: numeric },
);
- expect(result.allNumbersPreserved).toBe(false);
+ expect(result.sections[0]!.data.reverted).toBe(true);
// `checkNumbersPreserved` tokenizes the percent with its unit, so the
// invented token is "40%" — the point is that it is reported as ADDED.
expect(result.sections[0]!.data.addedNumbers).toContain("40%");
+ // And the fabrication never reaches the résumé.
+ const section = result.sections[0]!;
+ expect(section.kind === "experience" && section.data.bullets).toEqual([
+ "Worked on the payments API",
+ ]);
});
});
diff --git a/src/lib/webllm/rewrite-resume.ts b/src/lib/webllm/rewrite-resume.ts
index 27f46d23..004c84f3 100644
--- a/src/lib/webllm/rewrite-resume.ts
+++ b/src/lib/webllm/rewrite-resume.ts
@@ -108,6 +108,14 @@ export interface ResumeRewriteResult {
* True iff every section's `numbersPreserved` was true. The UI aggregates
* each section's `dropped` / `added` tokens for the warning copy, so the
* orchestrator only needs the boolean.
+ *
+ * Since #778 this is a property of what the user RECEIVED, so a section
+ * whose rewrite was rejected counts as preserved — the original preserves
+ * itself. It is therefore no longer sufficient to drive the panel's tone:
+ * read `sections[].data.reverted` alongside it, or a run where every section
+ * was rejected renders as a clean success. Now that invention reverts too,
+ * the only way this reads false is a section whose generation came back
+ * empty — there is no rewrite there to reject.
*/
allNumbersPreserved: boolean;
}
@@ -239,13 +247,21 @@ export async function rewriteResumeWithLlm(
completed.push(outcome);
accumulateOutcomeSignals(outcome, usedVerbs, usedPhrases);
+ // Telemetry measures the MODEL, not the delivered outcome — the same rule
+ // `rewrite-section.ts` follows. `outcome.data.numbersPreserved` is a
+ // property of what the user RECEIVED and is therefore true on every revert,
+ // so passing it through would flip `numbers_preserved` from "the model kept
+ // the numbers" to "the user got clean numbers" with no rename and no way to
+ // compare against anything logged before #778. Re-derive it from the raw
+ // diff lists and report the gate separately as `reverted`.
trackWebllmResumeRewriteSectionCompleted({
model: modelId,
sectionIndex: i,
sectionKind: outcome.kind,
inputUnitCount: inputUnitCountOf(section),
outputUnitCount: outputUnitCountOf(outcome),
- numbersPreserved: outcome.data.numbersPreserved,
+ numbersPreserved: modelPreservedNumbers(outcome),
+ reverted: outcome.data.reverted,
});
}
@@ -254,7 +270,10 @@ export async function rewriteResumeWithLlm(
trackWebllmResumeRewriteCompleted({
model: modelId,
sectionCount: totalSections,
- allNumbersPreserved,
+ // Aggregate of the same MODEL-facing property, not of the delivered
+ // `allNumbersPreserved` the function returns to the UI.
+ allNumbersPreserved: completed.every(modelPreservedNumbers),
+ anyReverted: completed.some((o) => o.data.reverted),
});
// Per-model one-shot — only counts a run that produced at least one
@@ -278,6 +297,19 @@ export async function rewriteResumeWithLlm(
return { sections: completed, allNumbersPreserved };
}
+/**
+ * Did the MODEL'S raw output preserve the section's numbers? Re-derived from
+ * the diff lists rather than read off `data.numbersPreserved`, which since #778
+ * describes the delivered units and is true on every revert by construction.
+ * This is the property the telemetry series has always carried.
+ */
+function modelPreservedNumbers(outcome: SectionOutcome): boolean {
+ return (
+ outcome.data.droppedNumbers.length === 0 &&
+ outcome.data.addedNumbers.length === 0
+ );
+}
+
function isNonEmptySection(section: SectionInput): boolean {
if (section.kind === "summary") return section.text.trim().length > 0;
return section.bullets.some((b) => b.trim().length > 0);
diff --git a/src/lib/webllm/rewrite-section.test.ts b/src/lib/webllm/rewrite-section.test.ts
index b50c1c47..52106cfc 100644
--- a/src/lib/webllm/rewrite-section.test.ts
+++ b/src/lib/webllm/rewrite-section.test.ts
@@ -166,8 +166,17 @@ describe("rewriteSectionWithLlm", () => {
);
// Three input bullets, three output bullets — but the model could
// legitimately return 2 or 4, and the function must handle it.
+ //
+ // The inputs carry the same figures the stub returns so the number gate is
+ // a no-op here: this test is about line splitting, and a stub that invents
+ // `10M` against a numberless input would revert and measure the gate
+ // instead.
const out = await rewriteSectionWithLlm(
- ["worked on Foo", "led the team", "sold things"],
+ [
+ "worked on Foo for 10M users",
+ "led 5 engineers, p99 latency 40%",
+ "sold things, $1.2M ARR",
+ ],
engine,
TEST_MODEL,
);
@@ -208,7 +217,7 @@ describe("rewriteSectionWithLlm", () => {
expect(out.addedNumbers).toEqual([]);
});
- it("flags numbersPreserved=false with the specific dropped token", async () => {
+ it("rejects a rewrite that drops a number and returns the input bullets (#778)", async () => {
const { engine } = makeEngine(async () =>
reply("Saved the team some money each quarter."),
);
@@ -217,21 +226,72 @@ describe("rewriteSectionWithLlm", () => {
engine,
TEST_MODEL,
);
- expect(out.numbersPreserved).toBe(false);
+ // The user gets their own bullet back, verbatim — the model's fluent but
+ // figure-less rewrite never reaches them.
+ expect(out.bullets).toEqual(["Saved the team $5K per quarter."]);
+ expect(out.reverted).toBe(true);
+ // `numbersPreserved` describes what was DELIVERED, so it is true here:
+ // nothing reached the user missing. `droppedNumbers` still names what the
+ // model lost, which is what the revert notice quotes.
+ expect(out.numbersPreserved).toBe(true);
expect(out.droppedNumbers).toEqual(["$5K"]);
});
- it("flags an invented number in added", async () => {
+ it("keeps a rewrite that carries every number through (#778)", async () => {
const { engine } = makeEngine(async () =>
- reply("Improved availability to 99.9%."),
+ reply("Cut $5K per quarter from the team's cloud spend."),
+ );
+ const out = await rewriteSectionWithLlm(
+ ["Saved the team $5K per quarter."],
+ engine,
+ TEST_MODEL,
+ );
+ expect(out.bullets).toEqual([
+ "Cut $5K per quarter from the team's cloud spend.",
+ ]);
+ expect(out.reverted).toBe(false);
+ expect(out.numbersPreserved).toBe(true);
+ });
+
+ it("does NOT revert a merge that moves a number to another bullet (#778)", async () => {
+ // Section rewrite is allowed to merge; the gate is a whole-section
+ // set diff precisely so a number changing bullets is not a drop.
+ const { engine } = makeEngine(async () =>
+ reply("Cut cloud spend $5K per quarter while lifting uptime 40%."),
+ );
+ const out = await rewriteSectionWithLlm(
+ ["Saved the team $5K per quarter.", "Lifted uptime 40%."],
+ engine,
+ TEST_MODEL,
);
+ expect(out.reverted).toBe(false);
+ expect(out.bullets).toHaveLength(1);
+ });
+
+ it("does NOT revert an empty generation — that is a failure, not a drop (#778)", async () => {
+ const { engine } = makeEngine(async () => reply(" "));
const out = await rewriteSectionWithLlm(
- ["Improved availability."],
+ ["Saved the team $5K per quarter."],
engine,
TEST_MODEL,
);
- expect(out.numbersPreserved).toBe(false);
+ expect(out.bullets).toEqual([]);
+ expect(out.reverted).toBe(false);
+ });
+
+ it("reverts an invented number and reports it in added", async () => {
+ // The widened #778 gate: invention reverts exactly like a drop, so the
+ // user's own bullet comes back and `addedNumbers` carries the evidence.
+ const original = ["Improved availability."];
+ const { engine } = makeEngine(async () =>
+ reply("Improved availability to 99.9%."),
+ );
+ const out = await rewriteSectionWithLlm(original, engine, TEST_MODEL);
+ expect(out.bullets).toEqual(original);
+ expect(out.reverted).toBe(true);
expect(out.addedNumbers).toEqual(["99.9%"]);
+ // Property of what was DELIVERED — the original invents nothing.
+ expect(out.numbersPreserved).toBe(true);
});
it("fires webllm_section_rewrite_started and _completed with counts and preservation flag", async () => {
@@ -248,6 +308,27 @@ describe("rewriteSectionWithLlm", () => {
inputBulletCount: 3,
outputBulletCount: 2,
numbersPreserved: true,
+ reverted: false,
+ });
+ });
+
+ it("reports the MODEL's preservation plus a reverted flag in telemetry (#778)", async () => {
+ const { engine } = makeEngine(async () =>
+ reply("Saved the team some money each quarter."),
+ );
+ await rewriteSectionWithLlm(
+ ["Saved the team $5K per quarter."],
+ engine,
+ TEST_MODEL,
+ );
+ // `numbers_preserved` keeps measuring the raw generation so the series is
+ // comparable across the release; `reverted` is what says the gate fired.
+ expect(trackCompletedMock).toHaveBeenCalledWith({
+ model: TEST_MODEL,
+ inputBulletCount: 1,
+ outputBulletCount: 1,
+ numbersPreserved: false,
+ reverted: true,
});
});
diff --git a/src/lib/webllm/rewrite-section.ts b/src/lib/webllm/rewrite-section.ts
index 2e413e8f..73224f5e 100644
--- a/src/lib/webllm/rewrite-section.ts
+++ b/src/lib/webllm/rewrite-section.ts
@@ -6,8 +6,10 @@ import {
trackWebllmSectionRewriteCompleted,
trackWebllmSectionRewriteStarted,
} from "../analytics.ts";
-import { cleanRewriteLine } from "./post-process.ts";
-import { checkNumbersPreserved } from "./preserve-numbers.ts";
+import {
+ applyNumberPreservation,
+ cleanRewriteLine,
+} from "./post-process.ts";
import {
composeRulesPrompt,
keepIfAlreadyStrongRule,
@@ -143,11 +145,19 @@ export interface SectionRewriteOptions {
}
export interface SectionRewriteResult {
- /** The rewritten bullets (M may differ from N). */
+ /**
+ * What the user gets (M may differ from N) — the model's rewrite, or the
+ * input bullets unchanged when the #778 gate rejected it.
+ */
bullets: string[];
- /** True iff every input number survived and none were invented. */
+ /** True iff `bullets` carries every input number and invents none. */
numbersPreserved: boolean;
- /** Numeric tokens that did not survive (UI surfaces these inline). */
+ /**
+ * The rewrite dropped a number and was rejected; `bullets` is the input
+ * (#778). The panel must say so — see `NumberPreservationWarning`.
+ */
+ reverted: boolean;
+ /** Numeric tokens the model dropped (UI surfaces these inline). */
droppedNumbers: string[];
/** Numeric tokens that appeared from nowhere (UI surfaces these inline). */
addedNumbers: string[];
@@ -220,13 +230,24 @@ export async function rewriteSectionWithLlm(
.map((line) => cleanRewriteLine(line))
.filter((line) => line.length > 0);
- const preservation = checkNumbersPreserved(bullets, rewrittenBullets);
+ // #778: a rewrite that drops OR invents a number is rejected here and the
+ // input bullets are returned instead.
+ const outcome = applyNumberPreservation(bullets, rewrittenBullets);
+ // Telemetry measures the MODEL, not the delivered outcome: every property
+ // below describes `rewrittenBullets`, so `numbers_preserved` means the
+ // same thing after #778 as before it and the series stays comparable
+ // across the release. `reverted` is what says whether the gate fired: the
+ // gate now covers both halves, so `numbers_preserved: false` with
+ // `reverted: false` narrows to a generation that came back empty.
trackWebllmSectionRewriteCompleted({
model: modelId,
inputBulletCount: bullets.length,
outputBulletCount: rewrittenBullets.length,
- numbersPreserved: preservation.ok,
+ numbersPreserved:
+ outcome.droppedNumbers.length === 0 &&
+ outcome.addedNumbers.length === 0,
+ reverted: outcome.reverted,
});
// Same gating as the per-bullet path: only count the first *successful*
@@ -241,10 +262,11 @@ export async function rewriteSectionWithLlm(
}
return {
- bullets: rewrittenBullets,
- numbersPreserved: preservation.ok,
- droppedNumbers: preservation.dropped,
- addedNumbers: preservation.added,
+ bullets: outcome.bullets,
+ numbersPreserved: outcome.numbersPreserved,
+ reverted: outcome.reverted,
+ droppedNumbers: outcome.droppedNumbers,
+ addedNumbers: outcome.addedNumbers,
};
} finally {
releaseInference(modelId);
diff --git a/src/lib/webllm/rewrite-summary.test.ts b/src/lib/webllm/rewrite-summary.test.ts
index 65356056..0aad955b 100644
--- a/src/lib/webllm/rewrite-summary.test.ts
+++ b/src/lib/webllm/rewrite-summary.test.ts
@@ -98,7 +98,13 @@ describe("rewriteSummaryWithLlm", () => {
const { engine } = makeEngine(async () =>
reply("Senior engineer.\nLed a team of 5.\nShipped 3 products."),
);
- const out = await rewriteSummaryWithLlm("Engineer.", engine, TEST_MODEL);
+ // Input carries the headcount the stub returns so the number gate stays a
+ // no-op — this test is about line collapsing, not about the gate.
+ const out = await rewriteSummaryWithLlm(
+ "Engineer with a team of 5.",
+ engine,
+ TEST_MODEL,
+ );
expect(out.text).toBe("Senior engineer. Led a team of 5. Shipped 3 products.");
});
@@ -122,7 +128,7 @@ describe("rewriteSummaryWithLlm", () => {
expect(out.numbersPreserved).toBe(true);
});
- it("flags numbersPreserved=false when a metric is dropped", async () => {
+ it("rejects a rewrite that drops a metric and returns the original paragraph (#778)", async () => {
const { engine } = makeEngine(async () =>
reply("Senior engineer with a decade of experience."),
);
@@ -131,10 +137,25 @@ describe("rewriteSummaryWithLlm", () => {
engine,
TEST_MODEL,
);
- expect(out.numbersPreserved).toBe(false);
+ expect(out.text).toBe("I drove $5K in revenue per quarter.");
+ expect(out.reverted).toBe(true);
+ expect(out.numbersPreserved).toBe(true);
expect(out.droppedNumbers).toEqual(["$5K"]);
});
+ it("leaves an empty generation empty rather than reverting it (#778)", async () => {
+ // A blank response is a failed generation the caller already handles; the
+ // gate must not dress it up as "kept your original".
+ const { engine } = makeEngine(async () => reply(" "));
+ const out = await rewriteSummaryWithLlm(
+ "I drove $5K in revenue per quarter.",
+ engine,
+ TEST_MODEL,
+ );
+ expect(out.text).toBe("");
+ expect(out.reverted).toBe(false);
+ });
+
it("returns an empty text on null model content without throwing", async () => {
const { engine } = makeEngine(async () => reply(null));
const out = await rewriteSummaryWithLlm("Engineer.", engine, TEST_MODEL);
diff --git a/src/lib/webllm/rewrite-summary.ts b/src/lib/webllm/rewrite-summary.ts
index 6d24b379..d7dfbd77 100644
--- a/src/lib/webllm/rewrite-summary.ts
+++ b/src/lib/webllm/rewrite-summary.ts
@@ -1,8 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2026 The offlinecv Authors
-import { cleanRewriteLine } from "./post-process.ts";
-import { checkNumbersPreserved } from "./preserve-numbers.ts";
+import {
+ applyNumberPreservation,
+ cleanRewriteLine,
+} from "./post-process.ts";
import {
composeRulesPrompt,
DROP_FILLER_RULE,
@@ -29,10 +31,11 @@ import { acquireInference, releaseInference } from "./web-llm.ts";
* the cross-model `acquireInference` lock can defer `.unload()` while
* this call is in flight.
* - Runs the deterministic number-preservation check after the model
- * responds (same `checkNumbersPreserved` used by the section path —
- * the multiset diff is shape-agnostic).
- * - Returns `numbersPreserved` + `dropped/added` so the UI can surface
- * the same inline warning shape as the section path.
+ * responds (same `applyNumberPreservation` used by the section path —
+ * the set diff is shape-agnostic), including the #778 reject gate:
+ * a rewrite that drops or invents a number returns the ORIGINAL paragraph.
+ * - Returns `numbersPreserved` + `reverted` + `dropped/added` so the UI can
+ * surface the same inline warning shape as the section path.
*
* Output handling: paragraphs sometimes come back across multiple lines if
* the model adds a wrap (or hallucinates a `Rewritten:` echo). We run each
@@ -112,11 +115,20 @@ export interface SummaryRewriteOptions {
}
export interface SummaryRewriteResult {
- /** The rewritten summary paragraph. Empty string when the model returned nothing. */
+ /**
+ * What the user gets. Empty string when the model returned nothing; the
+ * ORIGINAL paragraph when the #778 gate rejected the rewrite.
+ */
text: string;
- /** True iff every input number survived and none were invented. */
+ /** True iff `text` carries every input number and invents none. */
numbersPreserved: boolean;
- /** Numeric tokens that did not survive (UI surfaces these inline). */
+ /**
+ * The rewrite dropped or invented a number and was rejected; `text` is the
+ * original paragraph (#778). The panel must say so rather than showing an
+ * empty diff.
+ */
+ reverted: boolean;
+ /** Numeric tokens the model dropped (UI surfaces these inline). */
droppedNumbers: string[];
/** Numeric tokens that appeared from nowhere (UI surfaces these inline). */
addedNumbers: string[];
@@ -166,13 +178,21 @@ export async function rewriteSummaryWithLlm(
.join(" ")
.trim();
- const preservation = checkNumbersPreserved([summary], text ? [text] : []);
+ // #778, applied to the one-unit paragraph shape: `applyNumberPreservation`
+ // works on arrays, so wrap and unwrap. A blank generation stays blank —
+ // the gate deliberately leaves an empty rewrite to the caller's
+ // failed-generation handling rather than dressing it up as "kept yours".
+ const outcome = applyNumberPreservation(
+ [summary],
+ text ? [text] : [],
+ );
return {
- text,
- numbersPreserved: preservation.ok,
- droppedNumbers: preservation.dropped,
- addedNumbers: preservation.added,
+ text: outcome.bullets[0] ?? "",
+ numbersPreserved: outcome.numbersPreserved,
+ reverted: outcome.reverted,
+ droppedNumbers: outcome.droppedNumbers,
+ addedNumbers: outcome.addedNumbers,
};
} finally {
releaseInference(modelId);
diff --git a/tests/fixtures/rewrite/reports/README.md b/tests/fixtures/rewrite/reports/README.md
index 0eacf089..d99f1723 100644
--- a/tests/fixtures/rewrite/reports/README.md
+++ b/tests/fixtures/rewrite/reports/README.md
@@ -97,6 +97,74 @@ runs show `Numbers` between 0% and 80% (#778) and `Dedup` at 0% in eight of
nine cells — unrelated to #608, but visible in these files and worth its own
look rather than being read as noise.
+> **The `Numbers` reading above describes the 2026-08-07 runs and the code as
+> it stood then.** #778 has since changed what the criterion measures — see
+> "Reading `Numbers` after #778" below before comparing a new run to these.
+
+## Reading `Numbers` after #778
+
+#778 made the number-preservation guardrail binding rather than advisory: a
+rewrite that would drop a concrete number **or invent one that was not in the
+input** is **rejected**, and the user keeps their original bullets
+(`applyNumberPreservation` in `src/lib/webllm/post-process.ts`). The eval
+harness runs the same gate before scoring, so `Numbers` is now a rate over what
+a user would have *received*.
+
+Three consequences for anyone reading a report:
+
+1. **A high `Numbers` no longer means the model kept the figures.** Read it
+ with the new **`Reverted`** column, which is the share of cells the gate
+ rejected. `100% / 0%` is a model that got it right; `100% / 60%` is the gate
+ carrying it. `Reverted` is deliberately excluded from `Aggregate` — a revert
+ is the guardrail working, and scoring it as a criterion in either direction
+ would corrupt the number a default-model choice is made from.
+2. **`Numbers` is now ~100% by construction, so it carries almost no signal
+ on its own.** The gate covers both halves the criterion measures — dropping
+ and inventing — so every cell that produced any output at all is either a
+ clean rewrite or a revert, and both score `PASS`. The only route to a `fail`
+ left is a generation that came back empty, which the gate deliberately does
+ not touch. `Reverted` is the column that carries the model's actual
+ behaviour; the earlier drop-only gate is why the invention-only cells (the
+ `weak-marketing-generalist` cells, most obviously) used to score `fail`
+ instead.
+3. **Every other criterion is scored on the reverted output too.** That is
+ intentional, not a bug: a reverted `redundant` fixture honestly scores
+ `Dedup: fail`, because the user's bullets were not deduped.
+
+The per-cell `Reverted` column carries the tokens that triggered each rejection
+(`REVERTED: $4.2M, 14%`), because the rubric can no longer re-derive them — the
+scored bullets *are* the input once a cell reverts. Dropped tokens are listed
+first and invented ones after, undifferentiated: the cell's verdict is the same
+either way, and if anything the invented half is the worse one — a dropped
+figure costs the user a true claim they can put back, an invented one would
+have put a false claim in the document they hand an employer.
+
+**The 2026-08-07 reports predate all of this.** Their `Numbers` column is the
+old measurement (did the raw generation keep every figure) and they carry no
+`Reverted` column at all, exactly as the 2026-06-23 pair carries no `Steering`
+column. Do not read the two generations of the column as one series.
+
+**No post-#778 run is committed yet.** The gate and the extraction fixes it
+rests on are unit-tested (`post-process.test.ts`, `preserve-numbers.test.ts`),
+but a fresh `npm run eval:rewrite` on a WebGPU machine is still owed here
+before any claim about the shipped model's post-fix rate is made from this
+directory. #778's ~100% `Numbers` target is now reachable *by construction* —
+both halves of the criterion revert — but "by construction" is an argument, not
+a measurement, and the run is what would show whether `Reverted` lands at 10% or
+at 90%. That share, not `Numbers`, is the number worth waiting for.
+
+### Why prompt tuning was not tried first
+
+Recorded because it is the obvious question a reader of a future run will ask.
+`PRESERVE_NUMBERS_RULE` has been in every rewrite prompt since #609 and the
+2026-08-07 runs measured all three registry models breaking it, including under
+the `terse` variant that strips every competing instruction. The models in
+`MODEL_REGISTRY` (1.5B–3B) are not reliable enough at "carry these tokens
+through verbatim" for a wording change to be expected to move the number, so
+#778 built the deterministic backstop instead. Prompt tuning is worth
+revisiting when a future model generation makes small-model instruction
+adherence trustworthy — not before.
+
A third anomaly is in these files and no criterion catches it: **19 of the
24 Gemma 2 `terse` bullets ship literal markdown bold** — `"**Led** the
migration of the billing platform…"`. Those are `perBullet[].text`, i.e.