From 789f35b5064ddef6bc30d406081780ba365fde3c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 08:57:56 +0000 Subject: [PATCH] refactor(ml): V51 a store that fits on a screen No behaviour change: the 906 existing unit tests and 97 Chromium e2e tests pass unchanged. - lab-store.ts 1050 -> 530 lines; the onmessage if/else chain becomes a typed dispatch table (store/responses.ts), one entry per response kind - state, V47 fence and threshold artifact moved to store/ - v35..v40 test files renamed after what they pin - numbersOf / missingRatio / sharesOverEdges merged into quality/distribution.ts (quantile functions kept apart: 0 vs NaN on []) - responses.test.ts pins each state-only handler; it also restores the function-coverage floor the branch-to-function move had dipped - PLAN/CHANGELOG: V51 delivered, version 1.51.0 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELAjHYDyxa5zbbdcke1bMA --- CHANGELOG.md | 6 + PLAN.md | 2 +- README.md | 2 +- .../plans/2026-10-09-v51-store-refactor.md | 34 + package-lock.json | 4 +- package.json | 2 +- .../{v39.test.ts => column-recipe.test.ts} | 0 src/features/data/quality/distribution.ts | 38 ++ src/features/data/quality/drift.ts | 33 +- .../{v40.test.ts => quality-score.test.ts} | 0 src/features/data/quality/reference.ts | 30 +- .../{v38.test.ts => french-numbers.test.ts} | 0 src/features/ml/lab-store.ts | 581 +----------------- src/features/ml/store/artifacts.ts | 25 + src/features/ml/store/fence.ts | 74 +++ src/features/ml/store/responses.test.ts | 122 ++++ src/features/ml/store/responses.ts | 305 +++++++++ src/features/ml/store/state.ts | 223 +++++++ ...{v35.test.ts => honest-evaluation.test.ts} | 0 ...v36.test.ts => imbalance-ensemble.test.ts} | 0 .../{v37.test.ts => knn-and-helpers.test.ts} | 0 21 files changed, 869 insertions(+), 612 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-09-v51-store-refactor.md rename src/features/data/quality/{v39.test.ts => column-recipe.test.ts} (100%) create mode 100644 src/features/data/quality/distribution.ts rename src/features/data/quality/{v40.test.ts => quality-score.test.ts} (100%) rename src/features/ml/data/{v38.test.ts => french-numbers.test.ts} (100%) create mode 100644 src/features/ml/store/artifacts.ts create mode 100644 src/features/ml/store/fence.ts create mode 100644 src/features/ml/store/responses.test.ts create mode 100644 src/features/ml/store/responses.ts create mode 100644 src/features/ml/store/state.ts rename src/features/ml/train/{v35.test.ts => honest-evaluation.test.ts} (100%) rename src/features/ml/train/{v36.test.ts => imbalance-ensemble.test.ts} (100%) rename src/features/ml/train/{v37.test.ts => knn-and-helpers.test.ts} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index cedd2a1..55a97d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ entry per wave, newest first; the first six waves (V1–V6, the MVP) predate the are recorded in PLAN.md §B and §J. Each entry carries the opening sentence of its row and the reason the wave was built. +## V51 — A store that fits on a screen + +No behaviour change, and the proof is the suite: the 906 unit tests and the 97 Chromium e2e tests that passed before pass unchanged after. + +_Why:_ Approved 09/10/2026 after V50. Only after V47 to V50, as the plan required: the store's concurrency rules and the metrics were pinned by tests before their code moved, so a regression in the move would have failed loudly. + ## V50 — Metrics that agree with the reference on every edge Measured before anything changed, then checked against scikit-learn rather than against our own reading of it. diff --git a/PLAN.md b/PLAN.md index 4ae0fca..4e3e34b 100644 --- a/PLAN.md +++ b/PLAN.md @@ -616,7 +616,7 @@ before features, and the one that produces wrong data comes first of all. | **V48 — delivered** | **A damaged share link ends on a refusal, never on a crash.** The plan for this row was a deeper validator: check `insights.importance`, numeric metrics, `classes` and `confusion`, the fields the audit had named. The wave opened with a measurement instead, and the measurement changed the plan. Three real links were captured from the running app (titanic with uncertainty, threshold and segments; iris with k-NN tuning and groups; mpg as a regression) and every field of each was deleted, nulled, or retyped as a string, a number, an object or an array: **3 948 damaged links, 3 660 accepted by the decoder, 1 020 of those crashed the page, on 99 distinct paths**, from `insights.importance` to `artifacts.exploration.clusters[].traits[].clusterMean`. A validator for 99 paths would be the largest file in the share module and would drift with every analysis a run learns to carry. So the decoder keeps checking what every view reads first, and `SharedRunPage` wraps the run in a render guard: any failure while drawing it shows the same refusal page, with its own explanation (the link decoded, but part of the run does not have the shape LabML writes) and nothing from the run, so no half-read figure can pass for a real one. The three links are committed under `tests/share-links/`, and the measurement is now a test: every field shape is damaged six ways and the page must end on the run or on the refusal; it fails on the code before this wave and adds about 25 s to the unit suite, the price of covering the whole format rather than a sample. The five copies of the download helper become one, `src/lib/download.ts`, which revokes its object URL after the click has been handled instead of in the same tick, where some engines cancel the download silently. **What this wave deliberately does not do**: write a field-by-field schema for every artifact, show a partially readable run, or guard event handlers (a click on a damaged run is not reachable: the refusal replaces it first). 889 unit tests, 123 e2e. | Approved 09/10/2026 after V47. A link is input from a stranger, and the most common stranger is a chat client that cut it. The decoder already refused the shapes the V35 audit had found; the measurement showed it could not keep up with the format, and that the robust answer was at the point of failure rather than at the gate. | | **V49 — delivered** | **CI that cannot be steered by a branch name.** The wave opened with zizmor 1.30.1 on the three workflows: **28 findings** (1 template injection, 21 actions pinned by a movable tag, 6 checkouts leaving the git token on the runner disk). **(1) Injection**: the deploy step expanded `github.head_ref` inside its script, so a branch named `x";…;"` would have run as code; the value now reaches the shell through `env:`. Low severity today (only branches of this repository deploy) and fixed before that stops being true. **(2) Pins**: every action is pinned to a full commit SHA with its release in a trailing comment (`# v7.0.1`), which Dependabot keeps current; a tag can be moved by whoever controls the action's repository, a SHA cannot. **(3) Credentials**: every checkout sets `persist-credentials: false`; no job pushes, so nothing needed the token on disk. After the fixes the offline audit reports **0 findings**, and a new `Workflow audit (zizmor)` job runs it on every pull request, installed from PyPI at a pinned version rather than through a third-party action, with the token the online audits need to check that each pinned SHA really belongs to its repository. Three new tests in `supply-chain.test.ts` hold the same rules per workflow file (SHA plus release comment on every `uses:`, no persisted credentials, no pull-request-controlled `${{ }}` inside a `run:`); they fail on the workflows before this wave. **(4) Coverage**: `npm run test:coverage` (V8 provider) replaces `npm run test` in CI with a floor set from the measured baseline rounded down, not a round number: **64.56 % statements, 53.28 % branches, 54.46 % functions, 65.62 % lines** on 898 unit tests, floors 64 / 53 / 54 / 65. The figures cover unit tests only; the interface is mostly exercised by the e2e suite, which this does not measure. Cost: the quality job grows by about a minute. **What this wave deliberately does not do**: chase a coverage target, add e2e coverage instrumentation, or gate deploys on the audit job. 898 unit tests, 123 e2e. | Approved 09/10/2026 after V48. Continues V44: a security gate is worth its maintenance only when it distinguishes a real exposure, and every rule here is now enforced by a test or a job rather than by a reviewer remembering it. | | **V50 — delivered** | **Metrics that agree with the reference on every edge.** Measured before anything changed, then checked against scikit-learn rather than against our own reading of it. **(1) Macro averages** ran over every class the task knows; scikit-learn averages over the labels present in `y_true` or `y_pred`. A perfect score on a batch holding two of three classes read **0.667** instead of **1.0**. Both implementations (`macroPrf` and the V45 decision policy) now follow scikit-learn, and the V42 generator gained an `absentClass` case (four classes declared, three present, one predicted but never true) that the old code failed. **(2) Rare classes**: the stratified split sent a class's only row to test, and the validation carve did the same to a two-row class, so the models never saw an example of either. Measured on 100 rows plus a rare class: 1 row gave test 1 / train 0, 2 rows gave test 1 / validation 1 / train 0. A single-row class now stays on the training side (1 → train 1; 2 → test 1, train 1; 3 and more unchanged, and the seeded draw of every other class is byte-identical). This is not hypothetical: three targets the demos offer have one-row classes (titanic → parch, cafe-sales → city, cafe-sales → product). On titanic → parch every model's macro F1 rises once the empty classes stop counting (GBDT 0.619 → 0.716, ensemble 0.422 → 0.548), and the decomposition shows the metric change alone moves nothing while every class is present: the gain comes from the split. The validation-picked winner moves from GBDT to the forest because the validation rows changed, not because a metric did. When every class has one row, nothing is left to test on and training refuses by name (`too-few-rows`) instead of reporting metrics over an empty set. **(3) Empty sets**: accuracy, log loss, RMSE, MAE, R² and the macro trio now return NaN on zero rows; before, RMSE 0, MAE 0 and R² 1 described a perfect model measured on nothing. No caller passes an empty set today (the test split keeps a row, batch scoring skips unlabeled files), so this is a contract for the next caller; the one theoretical path left, tuning on fewer than four rows, is noted rather than guarded. **What this wave deliberately does not do**: change any figure on a dataset whose classes all have three rows or more (every demo target used by a test or a tutorial), add a warning banner for rare classes, or guard k-fold tuning on tiny data. 906 unit tests, 123 e2e. | Approved 09/10/2026 after V49. The same rule as V35: a number must not flatter or punish by accident. Two of the three edges were reachable from the demo picker, and the reference that would have caught the first one existed since V42 but held no case with an absent class. | -| **V51 — pending** | **A store that fits on a screen.** `lab-store.ts` (about 1 000 lines, a 250-line `onmessage` chain) split into feature slices with a dispatch table; tests named after waves (`v35.test.ts` to `v40.test.ts`) renamed after the behaviour they pin; the duplicated missing-ratio and histogram code in `drift.ts` and `reference.ts` merged. No behaviour change, proven by the suite staying identical. | Only after V47 to V50: refactoring code whose defects are not yet pinned by tests moves the defects around instead of removing them. | +| **V51 — delivered** | **A store that fits on a screen.** No behaviour change, and the proof is the suite: the 906 unit tests and the 97 Chromium e2e tests that passed before pass unchanged after. **(1) `lab-store.ts`, 1 050 → 530 lines.** The 250-line `onmessage` chain becomes a dispatch table in `store/responses.ts`, one entry per `WorkerResponse` kind, typed as a mapped type so the compiler refuses a kind with no handler and a handler for a kind that does not exist. The chain's last `else` used to catch every unlisted kind as an error silently; that safety net is now a compile error instead. The bodies were moved by script, not retyped. The state shape and starting values go to `store/state.ts`, V47's stale-response fence to `store/fence.ts`, the threshold artifact builder to `store/artifacts.ts`; `lab-store.ts` keeps the actions and re-exports its public names, so no importer changed. Feature slices were considered and not done: the actions share one worker and one `send`, and slicing them would have moved code without separating anything. **(2) Tests named after waves** renamed after what they pin: `v35` → `honest-evaluation`, `v36` → `imbalance-ensemble`, `v37` → `knn-and-helpers`, `v38` → `french-numbers`, `v39` → `column-recipe`, `v40` → `quality-score`. The two mixed files (`v35`, `v36`) were renamed, not split, to keep the change mechanical. **(3) Duplicated helpers**: `numbersOf`, `missingRatio` and the binned shares, copied identically in `drift.ts` and `reference.ts`, now live once in `quality/distribution.ts`. The two quantile functions were left apart on purpose: on an empty list one returns 0 and the other NaN, and merging them would have been a behaviour change. **Measured side effect**: the coverage gate from V49 fired. The 35 branches became 35 functions, so the function denominator rose from 2 142 to 2 174 while covered functions barely moved, and the floor of 54 % failed at 53.72 %. Rather than lowering the floor, a table-driven test (`store/responses.test.ts`, 17 cases) now pins which fields each state-only answer writes, which is also the check a moved body needs; function coverage is back to 54.46 %. **What this wave deliberately does not do**: change a single handler body, split the mixed test files, or unify the quantile functions. 923 unit tests, 123 e2e. | Approved 09/10/2026 after V50. Only after V47 to V50, as the plan required: the store's concurrency rules and the metrics were pinned by tests before their code moved, so a regression in the move would have failed loudly. | | **V52 — pending** | **Major upgrades, one at a time, each measured.** Vitest 5, vite-plugin-pwa 2, TypeScript 7. Each in its own pull request with the before/after numbers that matter for it (suite time, build time, service-worker behaviour offline). | V44's rule: a major update keeps its migration cost visible. | V12 (consented generative chat) remains pending a product decision. No wave diff --git a/README.md b/README.md index 15dc58b..b366f7e 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ Wikimedia Commons), the portrait is NASA, public domain._ a doc page carries its whole article, in English until the app mounts); Lighthouse mobile ≈ 0.99 on `/ml` under real throttling. Heavy dependencies (Dexie, SheetJS, ONNX Runtime) load lazily. -- **Quality bar.** 906 unit tests and 123 Playwright end-to-end tests across five +- **Quality bar.** 923 unit tests and 123 Playwright end-to-end tests across five projects — desktop, a phone viewport in English and in French, dark mode, and Cloudflare Pages' own routing emulated by `wrangler pages dev` (a real 404, the security headers as served) — covering offline PWA, a fake webcam, a horizontal-overflow guard on diff --git a/docs/superpowers/plans/2026-10-09-v51-store-refactor.md b/docs/superpowers/plans/2026-10-09-v51-store-refactor.md new file mode 100644 index 0000000..1ab8efb --- /dev/null +++ b/docs/superpowers/plans/2026-10-09-v51-store-refactor.md @@ -0,0 +1,34 @@ +# V51: a store that fits on a screen + +## Rule + +No behaviour change. The proof is the suite: 906 unit tests and 97 Chromium +e2e tests pass unchanged before and after. + +## Changes + +| Before | After | +| -------------------------------------- | ---------------------------------------------------------- | +| `lab-store.ts`, 1 050 lines | 530 lines: actions only, public names re-exported | +| 250-line `onmessage` `if / else` chain | `store/responses.ts`, one typed entry per `WorkerResponse` | +| State shape and initial values inline | `store/state.ts` | +| V47 fence inline | `store/fence.ts` | +| `thresholdArtifact` inside the closure | `store/artifacts.ts` (pure) | +| `v35` … `v40` test files | named after what they pin | +| Helpers copied in drift and reference | `quality/distribution.ts` | + +The dispatch table is a mapped type over `WorkerResponse['kind']`: a missing +or extra handler is a compile error. The old chain's final `else` handled any +unlisted kind as `error` without saying so. + +## Coverage gate + +Branches became functions: 2 142 → 2 174 functions, floor 54 % failed at +53.72 %. Answered with `store/responses.test.ts` (17 cases pinning the fields +each state-only answer writes), not by lowering the floor. Result: 54.46 %. + +## Not done + +Feature slices (the actions share one worker and one `send`), splitting the +mixed `v35`/`v36` test files, unifying the two quantile functions (they differ +on an empty list: 0 vs NaN). diff --git a/package-lock.json b/package-lock.json index 62c2ce6..9363ef3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "labml", - "version": "1.50.0", + "version": "1.51.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "labml", - "version": "1.50.0", + "version": "1.51.0", "license": "MIT", "dependencies": { "@duckdb/duckdb-wasm": "1.28.0", diff --git a/package.json b/package.json index ed4e39f..50d26f7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "labml", "private": true, - "version": "1.50.0", + "version": "1.51.0", "license": "MIT", "type": "module", "engines": { diff --git a/src/features/data/quality/v39.test.ts b/src/features/data/quality/column-recipe.test.ts similarity index 100% rename from src/features/data/quality/v39.test.ts rename to src/features/data/quality/column-recipe.test.ts diff --git a/src/features/data/quality/distribution.ts b/src/features/data/quality/distribution.ts new file mode 100644 index 0000000..703e25a --- /dev/null +++ b/src/features/data/quality/distribution.ts @@ -0,0 +1,38 @@ +/** + * V51 — the three helpers drift detection (V17) and the reference profile + * (V40) both need, which each file used to carry a copy of. The copies were + * identical; keeping one means a fix to how a number is read, or how a + * missing cell is counted, reaches both features at once. + */ +import { isMissing, parseNumber } from '@/features/ml/data/infer'; +import type { Cell } from '@/features/ml/data/types'; + +/** Every cell that reads as a number, in order; missing and non-numeric cells skipped. */ +export function numbersOf(values: Cell[]): number[] { + const numbers: number[] = []; + for (const value of values) { + if (isMissing(value)) continue; + const parsed = parseNumber((value as string).trim()); + if (parsed !== null) numbers.push(parsed); + } + return numbers; +} + +/** Share of missing cells; 0 for an empty column. */ +export function missingRatio(values: Cell[]): number { + if (values.length === 0) return 0; + let missing = 0; + for (const value of values) if (isMissing(value)) missing += 1; + return missing / values.length; +} + +/** Shares over bins whose edges are given (open-ended tails); all zero for no numbers. */ +export function sharesOverEdges(edges: number[], numbers: number[]): number[] { + const counts = new Array(edges.length + 1).fill(0); + for (const value of numbers) { + let bin = 0; + while (bin < edges.length && value > edges[bin]) bin += 1; + counts[bin] += 1; + } + return counts.map((count) => (numbers.length > 0 ? count / numbers.length : 0)); +} diff --git a/src/features/data/quality/drift.ts b/src/features/data/quality/drift.ts index a95875d..1f56f74 100644 --- a/src/features/data/quality/drift.ts +++ b/src/features/data/quality/drift.ts @@ -5,8 +5,9 @@ * the reference), new/vanished categories, missing-rate shifts. Pure and * deterministic; the conventional PSI thresholds 0.1 / 0.25 grade severity. */ -import { inferColumnType, isMissing, parseNumber } from '@/features/ml/data/infer'; +import { inferColumnType, isMissing } from '@/features/ml/data/infer'; import { quantileSorted } from '@/features/data/quality/checks'; +import { missingRatio, numbersOf, sharesOverEdges } from '@/features/data/quality/distribution'; import type { Cell, ColumnType } from '@/features/ml/data/types'; const NUMERIC_BINS = 10; @@ -62,34 +63,6 @@ export function psi(reference: number[], current: number[]): number { return total; } -function numbersOf(values: Cell[]): number[] { - const numbers: number[] = []; - for (const value of values) { - if (isMissing(value)) continue; - const parsed = parseNumber((value as string).trim()); - if (parsed !== null) numbers.push(parsed); - } - return numbers; -} - -function missingRatio(values: Cell[]): number { - if (values.length === 0) return 0; - let missing = 0; - for (const value of values) if (isMissing(value)) missing += 1; - return missing / values.length; -} - -/** Shares over bins whose edges are the reference's quantiles (open-ended tails). */ -function numericShares(edges: number[], numbers: number[]): number[] { - const counts = new Array(edges.length + 1).fill(0); - for (const value of numbers) { - let bin = 0; - while (bin < edges.length && value > edges[bin]) bin += 1; - counts[bin] += 1; - } - return counts.map((count) => (numbers.length > 0 ? count / numbers.length : 0)); -} - function numericDrift(column: string, ref: Cell[], next: Cell[]): ColumnDrift { const refNumbers = numbersOf(ref).sort((a, b) => a - b); const newNumbers = numbersOf(next); @@ -101,7 +74,7 @@ function numericDrift(column: string, ref: Cell[], next: Cell[]): ColumnDrift { const value = refNumbers.length === 0 || newNumbers.length === 0 ? 0 - : psi(numericShares(edges, refNumbers), numericShares(edges, newNumbers)); + : psi(sharesOverEdges(edges, refNumbers), sharesOverEdges(edges, newNumbers)); const mean = (numbers: number[]) => numbers.length > 0 ? numbers.reduce((a, v) => a + v, 0) / numbers.length : 0; return { diff --git a/src/features/data/quality/v40.test.ts b/src/features/data/quality/quality-score.test.ts similarity index 100% rename from src/features/data/quality/v40.test.ts rename to src/features/data/quality/quality-score.test.ts diff --git a/src/features/data/quality/reference.ts b/src/features/data/quality/reference.ts index 1f0df88..e5f492e 100644 --- a/src/features/data/quality/reference.ts +++ b/src/features/data/quality/reference.ts @@ -13,7 +13,7 @@ * salary distribution, not anybody's salary. That is what makes it safe to * commit next to the code it describes, which is the whole point of having it. */ -import { isMissing, inferColumnType, parseNumber } from '@/features/ml/data/infer'; +import { isMissing, inferColumnType } from '@/features/ml/data/infer'; import { psi, severityOf, @@ -21,6 +21,7 @@ import { type DriftSeverity, } from '@/features/data/quality/drift'; import type { Cell } from '@/features/ml/data/types'; +import { missingRatio, numbersOf, sharesOverEdges } from '@/features/data/quality/distribution'; /** Same bin count as V11, so a profile and a live comparison agree. */ const NUMERIC_BINS = 10; @@ -59,23 +60,6 @@ export interface DataProfile { const OTHER = '__other__'; -function numbersOf(values: Cell[]): number[] { - const numbers: number[] = []; - for (const value of values) { - if (isMissing(value)) continue; - const parsed = parseNumber((value as string).trim()); - if (parsed !== null) numbers.push(parsed); - } - return numbers; -} - -function missingRatio(values: Cell[]): number { - if (values.length === 0) return 0; - let missing = 0; - for (const value of values) if (isMissing(value)) missing += 1; - return missing / values.length; -} - function quantile(sorted: number[], q: number): number { if (sorted.length === 0) return 0; const position = (sorted.length - 1) * q; @@ -84,16 +68,6 @@ function quantile(sorted: number[], q: number): number { return low === high ? sorted[low] : sorted[low] + (position - low) * (sorted[high] - sorted[low]); } -function sharesOverEdges(edges: number[], numbers: number[]): number[] { - const counts = new Array(edges.length + 1).fill(0); - for (const value of numbers) { - let bin = 0; - while (bin < edges.length && value > edges[bin]) bin += 1; - counts[bin] += 1; - } - return counts.map((count) => (numbers.length > 0 ? count / numbers.length : 0)); -} - function categoryShares(values: Cell[]): { shares: Record; names: string[] } { const counts = new Map(); let total = 0; diff --git a/src/features/ml/data/v38.test.ts b/src/features/ml/data/french-numbers.test.ts similarity index 100% rename from src/features/ml/data/v38.test.ts rename to src/features/ml/data/french-numbers.test.ts diff --git a/src/features/ml/lab-store.ts b/src/features/ml/lab-store.ts index c77c004..fe35c8d 100644 --- a/src/features/ml/lab-store.ts +++ b/src/features/ml/lab-store.ts @@ -1,313 +1,38 @@ import { create } from 'zustand'; -import type { ReadFormat } from '@/features/ml/data/locale'; -import type { RunArtifacts, RunRecord } from '@/features/ml/projects/types'; -import type { - ColumnSuggestion, - DatasetMeta, - ColumnProfile, - ExclusionReason, - TaskInfo, -} from '@/features/ml/data/types'; -import { thresholdMetrics } from '@/features/ml/train/threshold'; -import type { BatchScore } from '@/features/ml/train/score'; -import type { ImportedManifest } from '@/features/ml/train/deserialize'; -import type { SegmentAnalysis } from '@/features/ml/train/segments'; -import type { ThresholdAnalysis } from '@/features/ml/train/threshold-analysis'; -import type { UncertaintyAnalysis } from '@/features/ml/train/uncertainty'; -import type { TunableKey, TuneOutcome } from '@/features/ml/train/search'; -import type { LearningCurveOutcome } from '@/features/ml/train/learning-curve'; -import type { RobustRankResult } from '@/features/ml/train/robust'; -import type { RankingMetric, SplitChoice } from '@/features/ml/train/types'; -import type { ExplorationPayload } from '@/features/ml/unsupervised/explore'; -import type { ForecastPayload } from '@/features/ml/timeseries/run'; -import type { ShapleyExplanation } from '@/features/ml/train/shapley'; -import type { - InsightsPayload, - ModelKey, - ModelResult, - TrainSummary, - WhatIfResult, -} from '@/features/ml/train/types'; +import type { RunArtifacts } from '@/features/ml/projects/types'; +import type { ExclusionReason } from '@/features/ml/data/types'; import type { WorkerRequest, WorkerResponse } from '@/features/ml/worker-protocol'; -import { bestResult } from '@/features/ml/train/ranking'; import { evaluateMulticlassPolicy } from '@/features/ml/train/multiclass-decision'; -import type { MulticlassDecisionAnalysis } from '@/features/ml/train/multiclass-decision-analysis'; import { editMulticlassThreshold, isExploratoryTestRequest, - receiveMulticlassDecisionTest, - type MulticlassDecisionEditorState, } from '@/features/ml/train/multiclass-decision-state'; - -export type LabStatus = 'idle' | 'parsing' | 'ready' | 'error'; -export type TrainStatus = 'idle' | 'training' | 'done'; - -export const TRAIN_SEED = 42; -export const TEST_RATIO = 0.2; - -interface LabState { - status: LabStatus; - error: string | null; - rowsParsed: number; - meta: DatasetMeta | null; - profiles: ColumnProfile[]; - preview: Record[]; - /** V38: how the file was read — null until a file has been parsed. */ - readFormat: ReadFormat | null; - /** Target-independent suggestions computed at parse time. */ - baseline: ColumnSuggestion[]; - target: string | null; - task: TaskInfo | null; - targetUnsupported: 'type' | 'tooManyClasses' | 'empty' | null; - /** Target-dependent leak suggestions. */ - leaks: ColumnSuggestion[]; - /** Manual include/exclude decisions that override the suggestions. */ - overrides: Record; - /** V35: announced non-random split, chosen in the UI. Null = seeded random. */ - splitChoice: SplitChoice | null; - /** V36: class weighting, off by default — a knob that does nothing is worse than none. */ - classWeighting: boolean; - /** V36: which metric the leaderboard ranks on. Null = the task's default. */ - rankMetric: RankingMetric | null; - /** V36: which class the threshold panel reads one-vs-rest (multiclass only). */ - thresholdClass: number; - trainStatus: TrainStatus; - modelProgress: { key: ModelKey; index: number; total: number } | null; - results: ModelResult[]; - summary: TrainSummary | null; - /** Insights bundle for the currently inspected model (defaults to the best). */ - insights: InsightsPayload | null; - whatIf: WhatIfResult | null; - /** Shapley explanation of the latest what-if row (cleared with it). */ - explanation: ShapleyExplanation | null; - tuneStatus: 'idle' | 'running' | 'done'; - tuneProgress: { done: number; total: number; bestCv: number | null } | null; - tuneOutcome: TuneOutcome | null; - curveStatus: 'idle' | 'running' | 'done'; - curveProgress: { done: number; total: number } | null; - robustStatus: 'idle' | 'running' | 'done'; - robustProgress: { done: number; total: number } | null; - robustOutcome: RobustRankResult | null; - /** null after a run = the worker refused (named): no curve theater. */ - curveOutcome: LearningCurveOutcome | null; - exploreStatus: 'idle' | 'running' | 'done'; - exploration: ExplorationPayload | null; - forecastStatus: 'idle' | 'running' | 'done' | 'error'; - forecastPayload: ForecastPayload | null; - batchStatus: 'idle' | 'scoring' | 'done' | 'error'; - batchResult: BatchScore | null; - batchError: string | null; - /** Binary + probabilistic models only — null otherwise. */ - thresholdAnalysis: ThresholdAnalysis | null; - thresholdChoice: { threshold: number; costFp: number; costFn: number }; - /** Validation-only editor for abstaining multiclass decisions. */ - multiclassDecision: { - analysis: MulticlassDecisionAnalysis; - editor: MulticlassDecisionEditorState; - } | null; - multiclassDecisionTestPending: boolean; - multiclassDecisionUnavailable: boolean; - /** Per-segment metrics of the inspected model — null when nothing sliceable. */ - segmentAnalysis: SegmentAnalysis | null; - /** Leaderboard-wide 95% intervals — belongs to the run, not the inspected model. */ - uncertaintyAnalysis: UncertaintyAnalysis | null; - /** The auto-saved record of the current run (id set once stored). */ - currentRun: RunRecord | null; - /** Local id of the stored copy of the CURRENT dataset — null if not kept. */ - savedDatasetId: number | null; - datasetSaving: boolean; - /** Refusal detail when a save does not fit the quota — named, never silent. */ - datasetQuotaError: { usedBytes: number; neededBytes: number; quotaBytes: number } | null; - /** v22: the manifest of a re-imported exported model — null when none. */ - importedManifest: ImportedManifest | null; - importedStatus: 'idle' | 'loading' | 'scoring'; - importedResult: BatchScore | null; - importedError: string | null; - /** File produced by an export action, consumed once by the UI download effect. */ - exportedFile: { name: string; mime: string; content: string } | null; - loadFile: (file: File) => void; - loadDemo: (fileName: string) => void; - saveDataset: () => void; - openDataset: (id: number) => void; - forgetDataset: (id: number) => void; - importModelFile: (file: File) => void; - importedScoreFile: (file: File) => void; - importedScoreDemo: (fileName: string) => void; - clearImported: () => void; - setTarget: (column: string | null) => void; - toggleColumn: (column: string) => void; - train: () => void; - cancelTrain: () => void; - selectInsightModel: (model: ModelKey) => void; - requestWhatIf: (values: Record) => void; - requestExplanation: (values: Record) => void; - tune: (model: TunableKey) => void; - cancelTune: () => void; - learningCurve: (model: ModelKey) => void; - robustRank: () => void; - cancelRobust: () => void; - setSplitChoice: (choice: SplitChoice | null) => void; - setClassWeighting: (on: boolean) => void; - setRankMetric: (metric: RankingMetric | null) => void; - setThresholdClass: (index: number) => void; - cancelCurve: () => void; - explore: () => void; - forecast: (dateColumn: string, valueColumn: string) => void; - scoreBatch: (file: File) => void; - scoreBatchDemo: (fileName: string) => void; - chooseThreshold: ( - partial: Partial<{ threshold: number; costFp: number; costFn: number }>, - ) => void; - setMulticlassThreshold: (classIndex: number, threshold: number) => void; - testMulticlassDecision: () => void; - exportModel: () => void; - exportPredictions: () => void; - clearExportedFile: () => void; - reset: () => void; -} +import { + initialData, + initialTraining, + TEST_RATIO, + TRAIN_SEED, + type LabState, +} from '@/features/ml/store/state'; +import { isStaleResponse, isTrainingBusy } from '@/features/ml/store/fence'; +import { thresholdArtifact } from '@/features/ml/store/artifacts'; +import { createResponseHandlers } from '@/features/ml/store/responses'; + +export { + TEST_RATIO, + TRAIN_SEED, + type LabStatus, + type TrainStatus, +} from '@/features/ml/store/state'; +export { isStaleResponse, isTrainingBusy } from '@/features/ml/store/fence'; let worker: Worker | null = null; -/** V47: the worker answers that belong to one training stream. */ -const TRAINING_STREAM = new Set([ - 'model-start', - 'model-result', - 'train-complete', - 'train-cancelled', -]); - -/** - * V47: the answers computed FROM a finished run — insights, analyses, tuning, - * exports. They are only meaningful while that run is the one on screen. - */ -const RUN_DERIVED = new Set([ - 'insights', - 'what-if-result', - 'explanation', - 'tune-progress', - 'tune-complete', - 'tune-cancelled', - 'curve-progress', - 'curve-complete', - 'curve-cancelled', - 'robust-progress', - 'robust-complete', - 'robust-cancelled', - 'batch-scored', - 'batch-error', - 'threshold-result', - 'multiclass-decision-result', - 'multiclass-decision-tested', - 'segments-result', - 'uncertainty-result', - 'model-json', - 'predictions-csv', -]); - -/** - * V47: a worker answer that no longer matches what the page shows. Changing - * the target or the feature set resets the training state, but the worker - * keeps answering requests sent before the reset; without this fence a late - * `model-result` joined the new leaderboard and the auto-save recorded the - * NEW target beside the OLD scores. Messages arrive in the order the worker - * sends them, so the training status alone is enough to tell them apart: a - * training message outside a run, or a run-derived answer when no run is - * done, belongs to a state that was already thrown away. - */ -export function isStaleResponse(kind: WorkerResponse['kind'], trainStatus: TrainStatus): boolean { - if (TRAINING_STREAM.has(kind)) return trainStatus !== 'training'; - if (RUN_DERIVED.has(kind)) return trainStatus !== 'done'; - return false; -} - -/** - * V47: true while the worker is computing something tied to the current - * target and feature set. Changing either is refused until it ends or is - * cancelled, so a result can never be filed under a question it did not answer. - */ -export function isTrainingBusy( - state: Pick< - LabState, - 'trainStatus' | 'tuneStatus' | 'curveStatus' | 'robustStatus' | 'batchStatus' - >, -): boolean { - return ( - state.trainStatus === 'training' || - state.tuneStatus === 'running' || - state.curveStatus === 'running' || - state.robustStatus === 'running' || - state.batchStatus === 'scoring' - ); -} - function terminateWorker() { worker?.terminate(); worker = null; } -const initialTraining = { - trainStatus: 'idle' as TrainStatus, - modelProgress: null, - results: [], - summary: null, - insights: null, - whatIf: null, - explanation: null, - tuneStatus: 'idle' as const, - tuneProgress: null, - tuneOutcome: null, - curveStatus: 'idle' as const, - curveProgress: null, - curveOutcome: null as LearningCurveOutcome | null, - robustStatus: 'idle' as const, - robustProgress: null, - robustOutcome: null as RobustRankResult | null, - exploreStatus: 'idle' as const, - exploration: null, - forecastStatus: 'idle' as const, - forecastPayload: null, - batchStatus: 'idle' as const, - batchResult: null, - batchError: null, - thresholdAnalysis: null as ThresholdAnalysis | null, - thresholdChoice: { threshold: 0.5, costFp: 1, costFn: 1 }, - multiclassDecision: null as LabState['multiclassDecision'], - multiclassDecisionTestPending: false, - multiclassDecisionUnavailable: false, - segmentAnalysis: null as SegmentAnalysis | null, - uncertaintyAnalysis: null as UncertaintyAnalysis | null, - currentRun: null, - exportedFile: null, -}; - -const initialData = { - status: 'idle' as LabStatus, - splitChoice: null as SplitChoice | null, - classWeighting: false, - rankMetric: null as RankingMetric | null, - thresholdClass: 0, - error: null, - rowsParsed: 0, - meta: null, - profiles: [], - preview: [], - readFormat: null, - baseline: [], - target: null, - task: null, - targetUnsupported: null, - leaks: [], - overrides: {}, - savedDatasetId: null as number | null, - datasetSaving: false, - datasetQuotaError: null as LabState['datasetQuotaError'], - importedManifest: null as ImportedManifest | null, - importedStatus: 'idle' as const, - importedResult: null as BatchScore | null, - importedError: null as string | null, - ...initialTraining, -}; - function activeDecisionPolicy(state: LabState) { const decision = state.currentRun?.artifacts?.multiclassDecision; return decision && decision.model === state.insights?.model ? decision.policy : undefined; @@ -331,29 +56,6 @@ export const useLabStore = create((set, get) => { } } - /** The persisted form of the analysis + the user's current cut. */ - function thresholdArtifact( - analysis: ThresholdAnalysis, - choice: { threshold: number; costFp: number; costFn: number }, - ) { - const y = analysis.pairs.map(([, label]) => label); - const p = analysis.pairs.map(([proba]) => proba); - return { - model: analysis.model, - positiveClass: analysis.positiveClass, - positiveRate: analysis.pr.positiveRate, - averagePrecision: analysis.pr.averagePrecision, - brier: analysis.calibration.brier, - prPoints: analysis.pr.points, - calibrationBins: analysis.calibration.bins, - chosen: { - ...thresholdMetrics(y, p, choice.threshold, choice.costFp, choice.costFn), - costFp: choice.costFp, - costFn: choice.costFn, - }, - }; - } - /** * Store the rebuilt CSV in IndexedDB (v19 opt-in). Everything is dynamic * here — Dexie and lz-string stay out of the initial /ml bundle. Over @@ -407,237 +109,10 @@ export const useLabStore = create((set, get) => { worker.onmessage = (event: MessageEvent) => { const message = event.data; if (isStaleResponse(message.kind, get().trainStatus)) return; - if (message.kind === 'progress') { - set({ rowsParsed: message.rows }); - } else if (message.kind === 'parsed') { - const { meta, profiles, preview, suggestions, readFormat } = message.payload; - set({ - status: 'ready', - meta, - profiles, - preview, - baseline: suggestions, - readFormat: readFormat ?? null, - rowsParsed: meta.rowCount, - }); - } else if (message.kind === 'target-analyzed') { - set({ - task: message.payload.task, - targetUnsupported: message.payload.unsupportedReason ?? null, - leaks: message.payload.suggestions, - }); - } else if (message.kind === 'model-start') { - set({ - trainStatus: 'training', - modelProgress: { key: message.key, index: message.index, total: message.total }, - }); - } else if (message.kind === 'model-result') { - set({ results: [...get().results, message.result] }); - } else if (message.kind === 'train-complete') { - set({ trainStatus: 'done', modelProgress: null, summary: message.summary }); - // Fetch insights for the winning model right away. - // V35: the SAME ranking rule as the leaderboard — otherwise the - // table crowns one model and the insights panel opens another. - const best = bestResult(get().results, message.summary.taskType); - if (best !== null) { - send({ kind: 'model-insights', model: best.key }); - // Leaderboard-wide intervals ride along with every completed run. - send({ kind: 'uncertainty-analysis' }); - } - } else if (message.kind === 'train-cancelled') { - set({ ...initialTraining }); - } else if (message.kind === 'insights') { - set({ insights: message.payload, whatIf: null }); - // Imbalance tools ride along; the worker answers null when N/A. - send({ - kind: 'threshold-analysis', - model: message.payload.model, - focusClass: get().thresholdClass, - }); - send({ kind: 'multiclass-decision-analysis', model: message.payload.model }); - send({ kind: 'segment-analysis', model: message.payload.model }); - // First insights after a completed run = winning model → auto-save. - const state = get(); - if ( - state.trainStatus === 'done' && - state.currentRun === null && - state.meta && - state.target && - state.task && - state.summary - ) { - const createdAt = Date.now(); - const record: RunRecord = { - name: `${state.meta.name.replace(/\.[a-z]+$/i, '')} · ${state.target}`, - createdAt, - dataset: { - name: state.meta.name, - rowCount: state.meta.rowCount, - columnCount: state.meta.columnCount, - }, - target: state.target, - taskType: state.task.type, - seed: state.summary.seed, - results: state.results, - summary: state.summary, - insights: message.payload, - ...(state.savedDatasetId !== null ? { datasetId: state.savedDatasetId } : {}), - }; - set({ currentRun: record }); - // Dexie is loaded on demand so /ml renders without it. - void import('@/features/ml/projects/db').then(({ db }) => - db.runs.add(record).then((id) => { - // Match by createdAt: an artifact may have replaced the object - // meanwhile — keep it, and persist what it attached. - const current = get().currentRun; - if (current && current.createdAt === record.createdAt) { - set({ currentRun: { ...current, id } }); - if (current.artifacts) { - void db.runs.update(id, { artifacts: current.artifacts }); - } - } - }), - ); - } - } else if (message.kind === 'what-if-result') { - set({ whatIf: message.payload, explanation: null }); - } else if (message.kind === 'explanation') { - set({ explanation: message.payload }); - attachArtifact({ explanation: message.payload }); - } else if (message.kind === 'tune-progress') { - set({ - tuneProgress: { done: message.done, total: message.total, bestCv: message.bestCv }, - }); - } else if (message.kind === 'tune-complete') { - set({ tuneStatus: 'done', tuneProgress: null, tuneOutcome: message.payload }); - attachArtifact({ tuning: message.payload }); - } else if (message.kind === 'tune-cancelled') { - set({ tuneStatus: 'idle', tuneProgress: null }); - } else if (message.kind === 'curve-progress') { - set({ curveProgress: { done: message.done, total: message.total } }); - } else if (message.kind === 'curve-complete') { - set({ curveStatus: 'done', curveProgress: null, curveOutcome: message.payload }); - if (message.payload) attachArtifact({ learningCurve: message.payload }); - } else if (message.kind === 'curve-cancelled') { - set({ curveStatus: 'idle', curveProgress: null }); - } else if (message.kind === 'robust-progress') { - set({ robustProgress: { done: message.done, total: message.total } }); - } else if (message.kind === 'robust-complete') { - set({ robustStatus: 'done', robustProgress: null, robustOutcome: message.payload }); - attachArtifact({ robustRank: message.payload }); - } else if (message.kind === 'robust-cancelled') { - set({ robustStatus: 'idle', robustProgress: null }); - } else if (message.kind === 'explore-result') { - set({ exploreStatus: 'done', exploration: message.payload }); - attachArtifact({ exploration: message.payload }); - } else if (message.kind === 'forecast-result') { - set({ forecastStatus: 'done', forecastPayload: message.payload }); - attachArtifact({ forecast: message.payload }); - } else if (message.kind === 'batch-scored') { - set({ batchStatus: 'done', batchResult: message.payload, batchError: null }); - // The record keeps the numbers, never the row-level CSV. - const artifact = { ...message.payload } as Partial; - delete artifact.csv; - delete artifact.preview; - attachArtifact({ batchScore: artifact as Omit }); - } else if (message.kind === 'batch-error') { - set({ batchStatus: 'error', batchResult: null, batchError: message.message }); - } else if (message.kind === 'model-loaded') { - set({ - importedManifest: message.manifest, - importedStatus: 'idle', - importedResult: null, - importedError: null, - }); - } else if (message.kind === 'imported-scored') { - set({ importedResult: message.payload, importedStatus: 'idle', importedError: null }); - } else if (message.kind === 'import-error') { - set({ importedError: message.message, importedStatus: 'idle' }); - } else if (message.kind === 'threshold-result') { - const choice = { threshold: 0.5, costFp: 1, costFn: 1 }; - set({ thresholdAnalysis: message.payload, thresholdChoice: choice }); - if (message.payload) { - attachArtifact({ threshold: thresholdArtifact(message.payload, choice) }); - } - } else if (message.kind === 'multiclass-decision-result') { - const analysis = message.payload; - set({ - multiclassDecisionTestPending: false, - multiclassDecisionUnavailable: - analysis === null && (get().task?.classes?.length ?? 0) > 2, - multiclassDecision: analysis - ? { - analysis, - editor: { - policy: { thresholds: analysis.classes.map(() => 0) }, - phase: 'draft', - test: null, - }, - } - : null, - }); - } else if (message.kind === 'multiclass-decision-tested') { - const state = get(); - const current = state.multiclassDecision; - if ( - current && - state.multiclassDecisionTestPending && - current.analysis.model === message.model - ) { - const editor = receiveMulticlassDecisionTest(current.editor, message.payload); - const validation = evaluateMulticlassPolicy( - current.analysis.validationLabels, - current.analysis.validationProbabilities, - editor.policy, - ); - set({ - multiclassDecision: { ...current, editor }, - multiclassDecisionTestPending: false, - }); - attachArtifact({ - multiclassDecision: { - model: current.analysis.model, - classes: current.analysis.classes, - policy: editor.policy, - validation, - test: message.payload.metrics, - exploratory: editor.phase === 'exploratory', - }, - }); - } - } else if (message.kind === 'segments-result') { - set({ segmentAnalysis: message.payload }); - if (message.payload) { - attachArtifact({ segments: message.payload }); - } - } else if (message.kind === 'uncertainty-result') { - set({ uncertaintyAnalysis: message.payload }); - if (message.payload) { - attachArtifact({ uncertainty: message.payload }); - } - } else if (message.kind === 'model-json') { - if (message.json !== null) { - set({ - exportedFile: { - name: `labml-${message.model}.json`, - mime: 'application/json', - content: message.json, - }, - }); - } - } else if (message.kind === 'dataset-csv') { - void persistDatasetCsv(message.csv); - } else if (message.kind === 'predictions-csv') { - set({ - exportedFile: { - name: `labml-${message.model}-predictions.csv`, - mime: 'text/csv', - content: message.csv, - }, - }); - } else { - set({ status: 'error', error: message.message, ...initialTraining }); - } + // TypeScript cannot correlate `message.kind` with the handler it + // indexes, so the call is widened once here; the table itself is + // checked kind by kind in store/responses.ts. + (handlers[message.kind] as (response: WorkerResponse) => void)(message); }; // V47: a crashed worker is gone — keeping its reference sent every // later request into the void, and leaving the busy flags up kept the @@ -656,6 +131,14 @@ export const useLabStore = create((set, get) => { worker.postMessage(request); } + const handlers = createResponseHandlers({ + set, + get, + send, + attachArtifact, + persistDatasetCsv, + }); + return { ...initialData, diff --git a/src/features/ml/store/artifacts.ts b/src/features/ml/store/artifacts.ts new file mode 100644 index 0000000..54b6275 --- /dev/null +++ b/src/features/ml/store/artifacts.ts @@ -0,0 +1,25 @@ +import { thresholdMetrics } from '@/features/ml/train/threshold'; +import type { ThresholdAnalysis } from '@/features/ml/train/threshold-analysis'; + +/** The persisted form of the analysis + the user's current cut. */ +export function thresholdArtifact( + analysis: ThresholdAnalysis, + choice: { threshold: number; costFp: number; costFn: number }, +) { + const y = analysis.pairs.map(([, label]) => label); + const p = analysis.pairs.map(([proba]) => proba); + return { + model: analysis.model, + positiveClass: analysis.positiveClass, + positiveRate: analysis.pr.positiveRate, + averagePrecision: analysis.pr.averagePrecision, + brier: analysis.calibration.brier, + prPoints: analysis.pr.points, + calibrationBins: analysis.calibration.bins, + chosen: { + ...thresholdMetrics(y, p, choice.threshold, choice.costFp, choice.costFn), + costFp: choice.costFp, + costFn: choice.costFn, + }, + }; +} diff --git a/src/features/ml/store/fence.ts b/src/features/ml/store/fence.ts new file mode 100644 index 0000000..4eda1cd --- /dev/null +++ b/src/features/ml/store/fence.ts @@ -0,0 +1,74 @@ +import type { WorkerResponse } from '@/features/ml/worker-protocol'; +import type { LabState, TrainStatus } from '@/features/ml/store/state'; + +/** V47: the worker answers that belong to one training stream. */ +const TRAINING_STREAM = new Set([ + 'model-start', + 'model-result', + 'train-complete', + 'train-cancelled', +]); + +/** + * V47: the answers computed FROM a finished run — insights, analyses, tuning, + * exports. They are only meaningful while that run is the one on screen. + */ +const RUN_DERIVED = new Set([ + 'insights', + 'what-if-result', + 'explanation', + 'tune-progress', + 'tune-complete', + 'tune-cancelled', + 'curve-progress', + 'curve-complete', + 'curve-cancelled', + 'robust-progress', + 'robust-complete', + 'robust-cancelled', + 'batch-scored', + 'batch-error', + 'threshold-result', + 'multiclass-decision-result', + 'multiclass-decision-tested', + 'segments-result', + 'uncertainty-result', + 'model-json', + 'predictions-csv', +]); + +/** + * V47: a worker answer that no longer matches what the page shows. Changing + * the target or the feature set resets the training state, but the worker + * keeps answering requests sent before the reset; without this fence a late + * `model-result` joined the new leaderboard and the auto-save recorded the + * NEW target beside the OLD scores. Messages arrive in the order the worker + * sends them, so the training status alone is enough to tell them apart: a + * training message outside a run, or a run-derived answer when no run is + * done, belongs to a state that was already thrown away. + */ +export function isStaleResponse(kind: WorkerResponse['kind'], trainStatus: TrainStatus): boolean { + if (TRAINING_STREAM.has(kind)) return trainStatus !== 'training'; + if (RUN_DERIVED.has(kind)) return trainStatus !== 'done'; + return false; +} + +/** + * V47: true while the worker is computing something tied to the current + * target and feature set. Changing either is refused until it ends or is + * cancelled, so a result can never be filed under a question it did not answer. + */ +export function isTrainingBusy( + state: Pick< + LabState, + 'trainStatus' | 'tuneStatus' | 'curveStatus' | 'robustStatus' | 'batchStatus' + >, +): boolean { + return ( + state.trainStatus === 'training' || + state.tuneStatus === 'running' || + state.curveStatus === 'running' || + state.robustStatus === 'running' || + state.batchStatus === 'scoring' + ); +} diff --git a/src/features/ml/store/responses.test.ts b/src/features/ml/store/responses.test.ts new file mode 100644 index 0000000..19f841b --- /dev/null +++ b/src/features/ml/store/responses.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createResponseHandlers } from '@/features/ml/store/responses'; +import { initialData, type LabState } from '@/features/ml/store/state'; +import type { WorkerResponse } from '@/features/ml/worker-protocol'; + +/** + * V51 — the dispatch table, one entry at a time. The bodies were moved out of + * a 250-line `if / else if` chain unchanged; these cases pin, for every answer + * that only updates state, which fields it writes. A body pasted under the + * wrong kind fails here. + */ +function harness(state: Partial = {}) { + let current = { ...initialData, ...state } as LabState; + const set = vi.fn((patch: Partial | ((s: LabState) => Partial)) => { + current = { ...current, ...(typeof patch === 'function' ? patch(current) : patch) }; + }); + const attachArtifact = vi.fn(); + const send = vi.fn(); + const handlers = createResponseHandlers({ + set: set as never, + get: () => current, + send, + attachArtifact, + persistDatasetCsv: vi.fn(async () => undefined), + }); + const dispatch = (message: WorkerResponse) => + (handlers[message.kind] as (response: WorkerResponse) => void)(message); + return { dispatch, state: () => current, attachArtifact, send }; +} + +const payload = { marker: true } as never; + +const CASES: [string, WorkerResponse, Partial][] = [ + ['progress', { kind: 'progress', rows: 5000 }, { rowsParsed: 5000 }], + [ + 'model-start', + { kind: 'model-start', key: 'tree', index: 2, total: 9 }, + { trainStatus: 'training', modelProgress: { key: 'tree', index: 2, total: 9 } }, + ], + ['what-if-result', { kind: 'what-if-result', payload }, { whatIf: payload, explanation: null }], + [ + 'tune-progress', + { kind: 'tune-progress', done: 3, total: 12, bestCv: 0.8 }, + { tuneProgress: { done: 3, total: 12, bestCv: 0.8 } }, + ], + ['tune-cancelled', { kind: 'tune-cancelled' }, { tuneStatus: 'idle', tuneProgress: null }], + [ + 'curve-progress', + { kind: 'curve-progress', done: 1, total: 5 }, + { curveProgress: { done: 1, total: 5 } }, + ], + ['curve-cancelled', { kind: 'curve-cancelled' }, { curveStatus: 'idle', curveProgress: null }], + [ + 'robust-progress', + { kind: 'robust-progress', done: 4, total: 80 }, + { robustProgress: { done: 4, total: 80 } }, + ], + [ + 'robust-cancelled', + { kind: 'robust-cancelled' }, + { robustStatus: 'idle', robustProgress: null }, + ], + [ + 'explore-result', + { kind: 'explore-result', payload }, + { exploreStatus: 'done', exploration: payload }, + ], + [ + 'forecast-result', + { kind: 'forecast-result', payload }, + { forecastStatus: 'done', forecastPayload: payload }, + ], + [ + 'batch-error', + { kind: 'batch-error', message: 'missing-columns' }, + { batchStatus: 'error', batchResult: null, batchError: 'missing-columns' }, + ], + [ + 'model-loaded', + { kind: 'model-loaded', manifest: payload }, + { + importedManifest: payload, + importedStatus: 'idle', + importedResult: null, + importedError: null, + }, + ], + [ + 'imported-scored', + { kind: 'imported-scored', payload }, + { importedResult: payload, importedStatus: 'idle', importedError: null }, + ], + [ + 'import-error', + { kind: 'import-error', message: 'bad-manifest' }, + { importedError: 'bad-manifest', importedStatus: 'idle' }, + ], + [ + 'error', + { kind: 'error', message: 'no-features' }, + { status: 'error', error: 'no-features', trainStatus: 'idle' }, + ], +]; + +describe('worker response table', () => { + it.each(CASES)('%s writes exactly its own fields', (_kind, message, expected) => { + const { dispatch, state } = harness({ trainStatus: 'done' }); + dispatch(message); + expect(state()).toMatchObject(expected); + }); + + it('attaches the late analyses to the run, and only those', () => { + const { dispatch, attachArtifact } = harness({ trainStatus: 'done' }); + dispatch({ kind: 'explore-result', payload }); + dispatch({ kind: 'forecast-result', payload }); + dispatch({ kind: 'tune-progress', done: 1, total: 2, bestCv: null }); + expect(attachArtifact.mock.calls).toEqual([ + [{ exploration: payload }], + [{ forecast: payload }], + ]); + }); +}); diff --git a/src/features/ml/store/responses.ts b/src/features/ml/store/responses.ts new file mode 100644 index 0000000..931cc60 --- /dev/null +++ b/src/features/ml/store/responses.ts @@ -0,0 +1,305 @@ +/** + * V51 — what the store does with each answer from the ML worker. + * + * This was a 250-line `if / else if` chain inside `onmessage`, whose last + * `else` silently caught every kind nobody had listed. It is now a table with + * one entry per `WorkerResponse` kind, typed so that the compiler refuses a + * kind with no handler and a handler for a kind that does not exist. The + * bodies are moved unchanged. + */ +import type { StoreApi } from 'zustand'; +import type { RunArtifacts, RunRecord } from '@/features/ml/projects/types'; +import type { BatchScore } from '@/features/ml/train/score'; +import type { WorkerRequest, WorkerResponse } from '@/features/ml/worker-protocol'; +import { bestResult } from '@/features/ml/train/ranking'; +import { evaluateMulticlassPolicy } from '@/features/ml/train/multiclass-decision'; +import { receiveMulticlassDecisionTest } from '@/features/ml/train/multiclass-decision-state'; +import { thresholdArtifact } from '@/features/ml/store/artifacts'; +import { initialTraining, type LabState } from '@/features/ml/store/state'; + +export interface ResponseContext { + set: StoreApi['setState']; + get: () => LabState; + send: (request: WorkerRequest) => void; + attachArtifact: (patch: Partial) => void; + persistDatasetCsv: (csv: string) => Promise; +} + +export type ResponseHandlers = { + [K in WorkerResponse['kind']]: (message: Extract) => void; +}; + +export function createResponseHandlers({ + set, + get, + send, + attachArtifact, + persistDatasetCsv, +}: ResponseContext): ResponseHandlers { + return { + progress: (message) => { + set({ rowsParsed: message.rows }); + }, + parsed: (message) => { + const { meta, profiles, preview, suggestions, readFormat } = message.payload; + set({ + status: 'ready', + meta, + profiles, + preview, + baseline: suggestions, + readFormat: readFormat ?? null, + rowsParsed: meta.rowCount, + }); + }, + 'target-analyzed': (message) => { + set({ + task: message.payload.task, + targetUnsupported: message.payload.unsupportedReason ?? null, + leaks: message.payload.suggestions, + }); + }, + 'model-start': (message) => { + set({ + trainStatus: 'training', + modelProgress: { key: message.key, index: message.index, total: message.total }, + }); + }, + 'model-result': (message) => { + set({ results: [...get().results, message.result] }); + }, + 'train-complete': (message) => { + set({ trainStatus: 'done', modelProgress: null, summary: message.summary }); + // Fetch insights for the winning model right away. + // V35: the SAME ranking rule as the leaderboard — otherwise the + // table crowns one model and the insights panel opens another. + const best = bestResult(get().results, message.summary.taskType); + if (best !== null) { + send({ kind: 'model-insights', model: best.key }); + // Leaderboard-wide intervals ride along with every completed run. + send({ kind: 'uncertainty-analysis' }); + } + }, + 'train-cancelled': () => { + set({ ...initialTraining }); + }, + insights: (message) => { + set({ insights: message.payload, whatIf: null }); + // Imbalance tools ride along; the worker answers null when N/A. + send({ + kind: 'threshold-analysis', + model: message.payload.model, + focusClass: get().thresholdClass, + }); + send({ kind: 'multiclass-decision-analysis', model: message.payload.model }); + send({ kind: 'segment-analysis', model: message.payload.model }); + // First insights after a completed run = winning model → auto-save. + const state = get(); + if ( + state.trainStatus === 'done' && + state.currentRun === null && + state.meta && + state.target && + state.task && + state.summary + ) { + const createdAt = Date.now(); + const record: RunRecord = { + name: `${state.meta.name.replace(/\.[a-z]+$/i, '')} · ${state.target}`, + createdAt, + dataset: { + name: state.meta.name, + rowCount: state.meta.rowCount, + columnCount: state.meta.columnCount, + }, + target: state.target, + taskType: state.task.type, + seed: state.summary.seed, + results: state.results, + summary: state.summary, + insights: message.payload, + ...(state.savedDatasetId !== null ? { datasetId: state.savedDatasetId } : {}), + }; + set({ currentRun: record }); + // Dexie is loaded on demand so /ml renders without it. + void import('@/features/ml/projects/db').then(({ db }) => + db.runs.add(record).then((id) => { + // Match by createdAt: an artifact may have replaced the object + // meanwhile — keep it, and persist what it attached. + const current = get().currentRun; + if (current && current.createdAt === record.createdAt) { + set({ currentRun: { ...current, id } }); + if (current.artifacts) { + void db.runs.update(id, { artifacts: current.artifacts }); + } + } + }), + ); + } + }, + 'what-if-result': (message) => { + set({ whatIf: message.payload, explanation: null }); + }, + explanation: (message) => { + set({ explanation: message.payload }); + attachArtifact({ explanation: message.payload }); + }, + 'tune-progress': (message) => { + set({ + tuneProgress: { done: message.done, total: message.total, bestCv: message.bestCv }, + }); + }, + 'tune-complete': (message) => { + set({ tuneStatus: 'done', tuneProgress: null, tuneOutcome: message.payload }); + attachArtifact({ tuning: message.payload }); + }, + 'tune-cancelled': () => { + set({ tuneStatus: 'idle', tuneProgress: null }); + }, + 'curve-progress': (message) => { + set({ curveProgress: { done: message.done, total: message.total } }); + }, + 'curve-complete': (message) => { + set({ curveStatus: 'done', curveProgress: null, curveOutcome: message.payload }); + if (message.payload) attachArtifact({ learningCurve: message.payload }); + }, + 'curve-cancelled': () => { + set({ curveStatus: 'idle', curveProgress: null }); + }, + 'robust-progress': (message) => { + set({ robustProgress: { done: message.done, total: message.total } }); + }, + 'robust-complete': (message) => { + set({ robustStatus: 'done', robustProgress: null, robustOutcome: message.payload }); + attachArtifact({ robustRank: message.payload }); + }, + 'robust-cancelled': () => { + set({ robustStatus: 'idle', robustProgress: null }); + }, + 'explore-result': (message) => { + set({ exploreStatus: 'done', exploration: message.payload }); + attachArtifact({ exploration: message.payload }); + }, + 'forecast-result': (message) => { + set({ forecastStatus: 'done', forecastPayload: message.payload }); + attachArtifact({ forecast: message.payload }); + }, + 'batch-scored': (message) => { + set({ batchStatus: 'done', batchResult: message.payload, batchError: null }); + // The record keeps the numbers, never the row-level CSV. + const artifact = { ...message.payload } as Partial; + delete artifact.csv; + delete artifact.preview; + attachArtifact({ batchScore: artifact as Omit }); + }, + 'batch-error': (message) => { + set({ batchStatus: 'error', batchResult: null, batchError: message.message }); + }, + 'model-loaded': (message) => { + set({ + importedManifest: message.manifest, + importedStatus: 'idle', + importedResult: null, + importedError: null, + }); + }, + 'imported-scored': (message) => { + set({ importedResult: message.payload, importedStatus: 'idle', importedError: null }); + }, + 'import-error': (message) => { + set({ importedError: message.message, importedStatus: 'idle' }); + }, + 'threshold-result': (message) => { + const choice = { threshold: 0.5, costFp: 1, costFn: 1 }; + set({ thresholdAnalysis: message.payload, thresholdChoice: choice }); + if (message.payload) { + attachArtifact({ threshold: thresholdArtifact(message.payload, choice) }); + } + }, + 'multiclass-decision-result': (message) => { + const analysis = message.payload; + set({ + multiclassDecisionTestPending: false, + multiclassDecisionUnavailable: analysis === null && (get().task?.classes?.length ?? 0) > 2, + multiclassDecision: analysis + ? { + analysis, + editor: { + policy: { thresholds: analysis.classes.map(() => 0) }, + phase: 'draft', + test: null, + }, + } + : null, + }); + }, + 'multiclass-decision-tested': (message) => { + const state = get(); + const current = state.multiclassDecision; + if ( + current && + state.multiclassDecisionTestPending && + current.analysis.model === message.model + ) { + const editor = receiveMulticlassDecisionTest(current.editor, message.payload); + const validation = evaluateMulticlassPolicy( + current.analysis.validationLabels, + current.analysis.validationProbabilities, + editor.policy, + ); + set({ + multiclassDecision: { ...current, editor }, + multiclassDecisionTestPending: false, + }); + attachArtifact({ + multiclassDecision: { + model: current.analysis.model, + classes: current.analysis.classes, + policy: editor.policy, + validation, + test: message.payload.metrics, + exploratory: editor.phase === 'exploratory', + }, + }); + } + }, + 'segments-result': (message) => { + set({ segmentAnalysis: message.payload }); + if (message.payload) { + attachArtifact({ segments: message.payload }); + } + }, + 'uncertainty-result': (message) => { + set({ uncertaintyAnalysis: message.payload }); + if (message.payload) { + attachArtifact({ uncertainty: message.payload }); + } + }, + 'model-json': (message) => { + if (message.json !== null) { + set({ + exportedFile: { + name: `labml-${message.model}.json`, + mime: 'application/json', + content: message.json, + }, + }); + } + }, + 'dataset-csv': (message) => { + void persistDatasetCsv(message.csv); + }, + 'predictions-csv': (message) => { + set({ + exportedFile: { + name: `labml-${message.model}-predictions.csv`, + mime: 'text/csv', + content: message.csv, + }, + }); + }, + error: (message) => { + set({ status: 'error', error: message.message, ...initialTraining }); + }, + }; +} diff --git a/src/features/ml/store/state.ts b/src/features/ml/store/state.ts new file mode 100644 index 0000000..9eebe79 --- /dev/null +++ b/src/features/ml/store/state.ts @@ -0,0 +1,223 @@ +/** + * V51 — the ML Lab store's shape and starting values, split out of + * `lab-store.ts` so the store file holds the actions and nothing else. + */ +import type { ReadFormat } from '@/features/ml/data/locale'; +import type { RunRecord } from '@/features/ml/projects/types'; +import type { + ColumnSuggestion, + DatasetMeta, + ColumnProfile, + TaskInfo, +} from '@/features/ml/data/types'; +import type { BatchScore } from '@/features/ml/train/score'; +import type { ImportedManifest } from '@/features/ml/train/deserialize'; +import type { SegmentAnalysis } from '@/features/ml/train/segments'; +import type { ThresholdAnalysis } from '@/features/ml/train/threshold-analysis'; +import type { UncertaintyAnalysis } from '@/features/ml/train/uncertainty'; +import type { TunableKey, TuneOutcome } from '@/features/ml/train/search'; +import type { LearningCurveOutcome } from '@/features/ml/train/learning-curve'; +import type { RobustRankResult } from '@/features/ml/train/robust'; +import type { RankingMetric, SplitChoice } from '@/features/ml/train/types'; +import type { ExplorationPayload } from '@/features/ml/unsupervised/explore'; +import type { ForecastPayload } from '@/features/ml/timeseries/run'; +import type { ShapleyExplanation } from '@/features/ml/train/shapley'; +import type { + InsightsPayload, + ModelKey, + ModelResult, + TrainSummary, + WhatIfResult, +} from '@/features/ml/train/types'; +import type { MulticlassDecisionAnalysis } from '@/features/ml/train/multiclass-decision-analysis'; +import type { MulticlassDecisionEditorState } from '@/features/ml/train/multiclass-decision-state'; + +export type LabStatus = 'idle' | 'parsing' | 'ready' | 'error'; +export type TrainStatus = 'idle' | 'training' | 'done'; + +export const TRAIN_SEED = 42; +export const TEST_RATIO = 0.2; + +export interface LabState { + status: LabStatus; + error: string | null; + rowsParsed: number; + meta: DatasetMeta | null; + profiles: ColumnProfile[]; + preview: Record[]; + /** V38: how the file was read — null until a file has been parsed. */ + readFormat: ReadFormat | null; + /** Target-independent suggestions computed at parse time. */ + baseline: ColumnSuggestion[]; + target: string | null; + task: TaskInfo | null; + targetUnsupported: 'type' | 'tooManyClasses' | 'empty' | null; + /** Target-dependent leak suggestions. */ + leaks: ColumnSuggestion[]; + /** Manual include/exclude decisions that override the suggestions. */ + overrides: Record; + /** V35: announced non-random split, chosen in the UI. Null = seeded random. */ + splitChoice: SplitChoice | null; + /** V36: class weighting, off by default — a knob that does nothing is worse than none. */ + classWeighting: boolean; + /** V36: which metric the leaderboard ranks on. Null = the task's default. */ + rankMetric: RankingMetric | null; + /** V36: which class the threshold panel reads one-vs-rest (multiclass only). */ + thresholdClass: number; + trainStatus: TrainStatus; + modelProgress: { key: ModelKey; index: number; total: number } | null; + results: ModelResult[]; + summary: TrainSummary | null; + /** Insights bundle for the currently inspected model (defaults to the best). */ + insights: InsightsPayload | null; + whatIf: WhatIfResult | null; + /** Shapley explanation of the latest what-if row (cleared with it). */ + explanation: ShapleyExplanation | null; + tuneStatus: 'idle' | 'running' | 'done'; + tuneProgress: { done: number; total: number; bestCv: number | null } | null; + tuneOutcome: TuneOutcome | null; + curveStatus: 'idle' | 'running' | 'done'; + curveProgress: { done: number; total: number } | null; + robustStatus: 'idle' | 'running' | 'done'; + robustProgress: { done: number; total: number } | null; + robustOutcome: RobustRankResult | null; + /** null after a run = the worker refused (named): no curve theater. */ + curveOutcome: LearningCurveOutcome | null; + exploreStatus: 'idle' | 'running' | 'done'; + exploration: ExplorationPayload | null; + forecastStatus: 'idle' | 'running' | 'done' | 'error'; + forecastPayload: ForecastPayload | null; + batchStatus: 'idle' | 'scoring' | 'done' | 'error'; + batchResult: BatchScore | null; + batchError: string | null; + /** Binary + probabilistic models only — null otherwise. */ + thresholdAnalysis: ThresholdAnalysis | null; + thresholdChoice: { threshold: number; costFp: number; costFn: number }; + /** Validation-only editor for abstaining multiclass decisions. */ + multiclassDecision: { + analysis: MulticlassDecisionAnalysis; + editor: MulticlassDecisionEditorState; + } | null; + multiclassDecisionTestPending: boolean; + multiclassDecisionUnavailable: boolean; + /** Per-segment metrics of the inspected model — null when nothing sliceable. */ + segmentAnalysis: SegmentAnalysis | null; + /** Leaderboard-wide 95% intervals — belongs to the run, not the inspected model. */ + uncertaintyAnalysis: UncertaintyAnalysis | null; + /** The auto-saved record of the current run (id set once stored). */ + currentRun: RunRecord | null; + /** Local id of the stored copy of the CURRENT dataset — null if not kept. */ + savedDatasetId: number | null; + datasetSaving: boolean; + /** Refusal detail when a save does not fit the quota — named, never silent. */ + datasetQuotaError: { usedBytes: number; neededBytes: number; quotaBytes: number } | null; + /** v22: the manifest of a re-imported exported model — null when none. */ + importedManifest: ImportedManifest | null; + importedStatus: 'idle' | 'loading' | 'scoring'; + importedResult: BatchScore | null; + importedError: string | null; + /** File produced by an export action, consumed once by the UI download effect. */ + exportedFile: { name: string; mime: string; content: string } | null; + loadFile: (file: File) => void; + loadDemo: (fileName: string) => void; + saveDataset: () => void; + openDataset: (id: number) => void; + forgetDataset: (id: number) => void; + importModelFile: (file: File) => void; + importedScoreFile: (file: File) => void; + importedScoreDemo: (fileName: string) => void; + clearImported: () => void; + setTarget: (column: string | null) => void; + toggleColumn: (column: string) => void; + train: () => void; + cancelTrain: () => void; + selectInsightModel: (model: ModelKey) => void; + requestWhatIf: (values: Record) => void; + requestExplanation: (values: Record) => void; + tune: (model: TunableKey) => void; + cancelTune: () => void; + learningCurve: (model: ModelKey) => void; + robustRank: () => void; + cancelRobust: () => void; + setSplitChoice: (choice: SplitChoice | null) => void; + setClassWeighting: (on: boolean) => void; + setRankMetric: (metric: RankingMetric | null) => void; + setThresholdClass: (index: number) => void; + cancelCurve: () => void; + explore: () => void; + forecast: (dateColumn: string, valueColumn: string) => void; + scoreBatch: (file: File) => void; + scoreBatchDemo: (fileName: string) => void; + chooseThreshold: ( + partial: Partial<{ threshold: number; costFp: number; costFn: number }>, + ) => void; + setMulticlassThreshold: (classIndex: number, threshold: number) => void; + testMulticlassDecision: () => void; + exportModel: () => void; + exportPredictions: () => void; + clearExportedFile: () => void; + reset: () => void; +} + +export const initialTraining = { + trainStatus: 'idle' as TrainStatus, + modelProgress: null, + results: [], + summary: null, + insights: null, + whatIf: null, + explanation: null, + tuneStatus: 'idle' as const, + tuneProgress: null, + tuneOutcome: null, + curveStatus: 'idle' as const, + curveProgress: null, + curveOutcome: null as LearningCurveOutcome | null, + robustStatus: 'idle' as const, + robustProgress: null, + robustOutcome: null as RobustRankResult | null, + exploreStatus: 'idle' as const, + exploration: null, + forecastStatus: 'idle' as const, + forecastPayload: null, + batchStatus: 'idle' as const, + batchResult: null, + batchError: null, + thresholdAnalysis: null as ThresholdAnalysis | null, + thresholdChoice: { threshold: 0.5, costFp: 1, costFn: 1 }, + multiclassDecision: null as LabState['multiclassDecision'], + multiclassDecisionTestPending: false, + multiclassDecisionUnavailable: false, + segmentAnalysis: null as SegmentAnalysis | null, + uncertaintyAnalysis: null as UncertaintyAnalysis | null, + currentRun: null, + exportedFile: null, +}; + +export const initialData = { + status: 'idle' as LabStatus, + splitChoice: null as SplitChoice | null, + classWeighting: false, + rankMetric: null as RankingMetric | null, + thresholdClass: 0, + error: null, + rowsParsed: 0, + meta: null, + profiles: [], + preview: [], + readFormat: null, + baseline: [], + target: null, + task: null, + targetUnsupported: null, + leaks: [], + overrides: {}, + savedDatasetId: null as number | null, + datasetSaving: false, + datasetQuotaError: null as LabState['datasetQuotaError'], + importedManifest: null as ImportedManifest | null, + importedStatus: 'idle' as const, + importedResult: null as BatchScore | null, + importedError: null as string | null, + ...initialTraining, +}; diff --git a/src/features/ml/train/v35.test.ts b/src/features/ml/train/honest-evaluation.test.ts similarity index 100% rename from src/features/ml/train/v35.test.ts rename to src/features/ml/train/honest-evaluation.test.ts diff --git a/src/features/ml/train/v36.test.ts b/src/features/ml/train/imbalance-ensemble.test.ts similarity index 100% rename from src/features/ml/train/v36.test.ts rename to src/features/ml/train/imbalance-ensemble.test.ts diff --git a/src/features/ml/train/v37.test.ts b/src/features/ml/train/knn-and-helpers.test.ts similarity index 100% rename from src/features/ml/train/v37.test.ts rename to src/features/ml/train/knn-and-helpers.test.ts