From 9d2f057ca0bf6a03d248ce710ab3d1c2a15afa2c Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 12:42:56 +0300 Subject: [PATCH 1/7] docs: design and plan the opt-in accurate NLI profile Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- .../plans/2026-10-01-accurate-nli-plan.md | 177 ++++++++++++++++++ .../specs/2026-10-01-accurate-nli-design.md | 110 +++++++++++ 2 files changed, 287 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-01-accurate-nli-plan.md create mode 100644 docs/superpowers/specs/2026-10-01-accurate-nli-design.md diff --git a/docs/superpowers/plans/2026-10-01-accurate-nli-plan.md b/docs/superpowers/plans/2026-10-01-accurate-nli-plan.md new file mode 100644 index 0000000..df0684b --- /dev/null +++ b/docs/superpowers/plans/2026-10-01-accurate-nli-plan.md @@ -0,0 +1,177 @@ +# Opt-in Accurate NLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** Ship the bake-off winner as an opt-in `accurate` NLI profile. It is downloaded only with consent, falls back to the default profile with a reported reason, and the report says when it has no `low` tier. + +**Spec:** `docs/superpowers/specs/2026-10-01-accurate-nli-design.md`. The exact values to use are in its profile table and its "Tier note" section. + +## Global Constraints + +- **Python and checks:** Python 3.10+ with type annotations on every signature. `uv run ruff check`, `uv run ruff format --check` and `uv run mypy` (strict, whole `src/`; CI now runs it) must be clean. `pathlib` throughout, and every read or write passes `encoding="utf-8"`. +- **No network in unit tests.** Model downloads and `hf_hub_download` are injected or monkeypatched. +- **Product rule 4:** with no TTY, an `ask` permission becomes `deny` and the denial is reported. +- **Product rule 5:** the 643 MB model is never downloaded without consent. +- **Product rule 6:** a fallback is reported, never silent. +- **`cli` and `tui` contain no logic.** Both call `verify()` / `Engine.default`. +- **Device order** stays CUDA → CoreML → CPU, and the int8 and `quantized` files skip CoreML. +- **The default profile's behaviour and values are unchanged.** The only exception is the new `NO_LOW_TIER` note. +- **Accurate profile values:** repo `MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli`, revision `b3546ea6b0346eb6f8d5d68b13c7dc6d0376b3d7`, file `onnx/model_quantized.onnx`, k=2, decide 0.25, medium 0.252721, high 0.999142, size 643 MB. +- **Low-tier margin:** 0.01. +- **Git:** subagents never commit. The controller commits after a clean review. + +--- + +### Task 1: Profiles, tier note, config keys + +**Files:** +- `src/proofpath/entailment.py` +- `src/proofpath/pipeline.py` +- `src/proofpath/config.py` +- `src/proofpath/settings_hints.py` +- Tests: `tests/test_entailment.py`, `tests/test_pipeline.py`, `tests/test_config.py`, `tests/test_config_cli.py` + +**Produces:** +- `entailment.NliProfile`: a frozen dataclass with fields `name: str`, `repo: str`, `revision: str`, `onnx_file: Callable[[str], str]`, `k: int`, `thresholds: pipeline.Thresholds`, `size_mb: int`. + - `entailment` must not import `pipeline` at module level if that creates an import cycle. If it does, put `NliProfile` and the two profiles in a new `src/proofpath/profiles.py` and say so in the report. +- `DEFAULT_PROFILE`, `ACCURATE_PROFILE`, and `PROFILES: Mapping[str, NliProfile]`. +- `profile_installed(profile: NliProfile, cache_dir: Path, machine: str) -> bool`. It uses `huggingface_hub.try_to_load_from_cache` for `config.json`, `tokenizer.json` and the onnx file. It makes no network calls. +- `providers_for`: CoreML is also skipped when the stem contains `quantized`. +- `pipeline.NO_LOW_TIER`: one sentence in the style of `NO_HIGH_TIER`. +- `pipeline.LOW_TIER_MARGIN = 0.01`. +- `pipeline.tier_note`: returns both notes joined with `"; "` when both apply. +- `config.ModelsConfig`: one field, `nli: Literal["default", "accurate"] = "default"`. Register it in `_SECTIONS` and `_resolve_type`, and add it to `Config` as `models`. +- `Permissions.install_model: Permission = "ask"`. +- `settings_hints.MODEL_SETTING`: the `config set` hint for `permissions.install_model`, worded like `BROWSER_SETTING`. + +**Tests:** +- Profile values match the spec table. +- `providers_for` skips CoreML for `onnx/model_quantized.onnx`. +- `tier_note` returns no-high alone, no-low alone, both joined, and neither. +- `profile_installed` returns True/False against a monkeypatched `try_to_load_from_cache`. +- Config: + - `[models] nli = "accurate"` parses; + - `"fast"` is refused with a `ConfigError`; + - `install_model` parses; + - `config set models.nli accurate` round-trips through `set_value` / `render_config`. +- Existing tests that assert the default `tier_note` text must be updated to include the low-tier note. List them in the report. + +### Task 2: Model consent gate + +**Files:** +- Create `src/proofpath/model_gate.py` +- Test: `tests/test_model_gate.py` + +Model it on `browser.ConsentGate` (`src/proofpath/browser.py:243-341`) and reuse `config.resolve_permission`, `browser.Answer` / `PROMPT_CHOICES` / `TERMINAL_ANSWERS` / `TUI_ANSWERS` and `ask_terminal`. + +**Produces:** +- `model_prompt_text(profile: NliProfile, *, answers: Sequence[str]) -> str`. It follows the same three-part layout as `browser.prompt_text`: what, why, the size in MB, and the answers. +- `ModelGate`. Constructor: + - `permission: Permission` + - `interactive: bool` + - `override: bool | None = None` (True means `--accurate` given on the command line. It does **not** grant consent; it only selects the profile.) + - `prompt: Callable[[str, int | None], Answer] | None = None` + - `config_path: Path | None = None` + - `download: Callable[[NliProfile], None]` + - `installed: Callable[[NliProfile], bool]` + + Methods and state: + - `ensure(profile) -> Decision` (`config.Decision`): + - If installed, return allow with reason "installed". + - Otherwise resolve the permission. With no TTY, `ask` becomes `deny` (reason as `resolve_permission` words it). + - On `ask` with a TTY, prompt once. "always" and "never" are persisted to `permissions.install_model` through `config.set_value`; a persist failure is logged, not raised. + - On allow, call `download(profile)`. An exception means `Decision("deny", "model download failed: ")`, and no raw response text goes into the reason. + - `log: list[str]`: install and decision lines for the run. +- `download_profile(profile: NliProfile, cache_dir: Path, machine: str) -> None`: `hf_hub_download` of the three files. + +**Tests:** +- installed → no prompt; +- allow → download called; +- deny → no download; +- ask without TTY → deny, with the reason; +- ask with TTY, each of the 4 answers; +- always and never persisted (to a tmp config path); +- download raising → deny with the reason, no traceback; +- the prompt text names the repo and 643 MB. + +### Task 3: Engine, verify stage, report, CLI flag + +**Files:** +- `src/proofpath/verify.py` +- `src/proofpath/cli.py` +- `src/proofpath/report.py` (only if needed) +- Tests: `tests/test_verify.py`, `tests/test_check_cli.py`, `tests/test_report.py`, `tests/data/verify-report-golden.json` + +**Engine changes:** +- `Engine.default(config, *, …, nli: str | None = None)`. `nli` overrides `config.models.nli`; the CLI passes `"accurate"` for `--accurate`. Explicit `k` and `thresholds` arguments still win, for tests and eval. +- The Engine records `nli_requested` and a `ModelGate`. It builds the gate with `permission=config.permissions.install_model`, the same `interactive` and `prompt` as the browser gate, `installed=profile_installed`, and `download=download_profile`. +- **`Engine.default` must stay model-free and network-free.** Keep `test_engine_default_never_builds_a_model` green. +- Resolve the profile lazily, once, at the start of the verify stage, before `LOADING_MODELS`, through a method such as `engine.resolve_nli() -> NliProfile`: + - default requested → default profile; + - accurate requested → `gate.ensure(ACCURATE_PROFILE)`. Allow uses the accurate profile. Deny or failure falls back to `DEFAULT_PROFILE` and records the reason. +- The scorer factory builds `OnnxNli` from the resolved profile, passing `repo_id`, `revision` and `onnx_file(platform.machine())`. +- `engine.k` and `engine.thresholds` must be the resolved profile's values at the point where `cache_mod.model_id` is computed (`verify.py:~1293`) and wherever `pipeline.decide_indexed` gets `k` and `thresholds`. Pick the least invasive shape (resolved fields set once, or properties) and explain it in the report. +- On fallback: + - emit a Note event: "accurate NLI model not used: — using the default model. To allow the download: "; + - forward the gate's log lines as Notes, the way `install_log` is forwarded for the browser. + +**Report:** +- `_models()` gains `"k": str(k)`. +- On fallback it also gains `"nli_requested": "accurate ()"`. +- `tier_note` uses the resolved thresholds. +- Update the golden JSON for `k` and the new low-tier note. + +**CLI:** +- `check --accurate` (help: "use the larger accurate NLI model for this run (643 MB, downloaded once with consent)"). +- It is passed as `nli="accurate"`. Without the flag it passes `None`, so config decides. +- The CLI prints fallback notes on stderr through the existing Note path. + +**Tests:** +- installed accurate → k=2 and accurate thresholds reach `model_id` and decide; +- deny → default profile, the reason in `models["nli_requested"]` and a Note; +- `--accurate` reaches `Engine.default` as `nli="accurate"` (the pattern in `test_check_cli.py:592+`); +- `Engine.default` builds no model; +- the cache key differs between the two profiles. + +### Task 4: TUI + +**Files:** +- `src/proofpath/tui/widgets/config_panel.py` +- `src/proofpath/tui/app.py` +- `src/proofpath/tui/widgets/prompt.py` (only if the model prompt needs it) +- Tests: `tests/test_tui_app.py`, `tests/test_tui_rich.py` + +**Changes:** +- `/config` panel: + - a `models` section with a choice row `models.nli` (values `default` / `accurate`), its meaning text naming the size and the SciFact gain; + - a `permissions.install_model` row next to `install_browser`, built the same way. +- The TUI's engine already passes `prompt=self._prompt_for(run.id)` to `Engine.default`. Make sure the model gate's prompt reaches that same `PermissionPrompt` path, with `model_prompt_text(..., answers=TUI_ANSWERS)`. + - Generalise `PermissionPrompt` to take ready-made text if it currently only builds browser text. + - The `/allow` verb must answer a model prompt too. + +**Tests:** +- the rows exist with the right values; +- a model prompt renders and an answer settles it; +- the quit path answers "no". + +### Task 5: Docs and site + +**Files:** +- `README.md` +- `docs/superpowers/specs/2026-09-10-proofpath-design.md` (§7.1 permission table, §12 NLI row, §14 calibration) +- `CHANGELOG.md` (`[Unreleased]` → Added / Changed) +- `docs/superpowers/OPEN-ITEMS.md` (§20.11: (a) and (b) settled, (c) open) +- `site/src/data/measured.ts` (mirror the README Measured rows) + +**README:** +- A short "Accurate model (opt-in)" section with: + - what it is; + - its size and speed; + - the SciFact numbers: 0.697 against 0.580 macro-F1; + - the AVeriTeC caveat: not better on news; recommended for scientific sources; + - how to turn it on; + - the consent behaviour. +- The Measured table gains a row for the accurate profile. +- The Speed section mentions the 643 MB download. + +**Site:** if `site/` data changed, run the site build per the repo's site README and report the result. Keep wording consistent with the README. diff --git a/docs/superpowers/specs/2026-10-01-accurate-nli-design.md b/docs/superpowers/specs/2026-10-01-accurate-nli-design.md new file mode 100644 index 0000000..7bd5fb7 --- /dev/null +++ b/docs/superpowers/specs/2026-10-01-accurate-nli-design.md @@ -0,0 +1,110 @@ +# Opt-in accurate NLI model: design (2026-10-01) + +## Why + +The NLI bake-off (`docs/eval/2026-10-01-nli-bakeoff.md`) picked +`MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli` at k=2 with `max` aggregation. +Its SciFact dev macro-F1 is 0.697. The current default scores 0.580, and the best +configuration of the default model 0.608. The winner costs 643 MB instead of 244 MB and +runs at 54 ms per pair instead of 19 ms. It does **not** improve the news and fact-check +case: AVeriTeC on readable claims is 0.475, against 0.537 for the default model at k=2, +and both are below the 0.708 majority baseline. The default install must stay small +(product rule 5, spec §7.1), so the large model ships as an **opt-in** profile, not a +new default. + +The bake-off left three open points (OPEN-ITEMS 20.11). This design settles two of them: + +- **(a)** Rule-decided rows (the numeric layer, score 1.0) no longer set the `high` cut. + They fired 4–6 times on SciFact train and none was correct, which pinned `high` at + 1.000000. The accurate profile's cuts are calibrated with those rows left out of the + tier walk (they still count toward `decide`). +- **(b)** A run whose `medium` cut sits on `decide` now says that it has no `low` + tier, in the same derived way it already says when it has no `high` tier. + +**(c)** stays open, and the documentation says so: AVeriTeC is not better with the +accurate model, so it is recommended for scientific sources, not for news. + +## User-facing behaviour + +- **Config:** a new section with one key, `[models] nli = "default" | "accurate"`. The + default is `"default"`. It is set with + `proofpath config set models.nli accurate`, or from the `/config` panel. +- **One run:** `proofpath check --accurate` uses the accurate profile for one run. +- **Consent:** a new permission, `permissions.install_model = "ask" | "allow" | "deny"`. + Its default is `"ask"`. + - When the accurate profile is chosen and its files are not yet in the model cache, the + run asks once before downloading. The prompt states the model and its size + (643 MB). The answers are the browser gate's: once, always, no, never. + "Always" and "never" are saved to `permissions.install_model`. + - With no interactive terminal, `ask` is treated as `deny` and the denial is reported + (product rule 4). + - When the download is denied or fails, the run **falls back to the default + profile**. It says so as a Note, on stderr in the CLI, and on a `models` entry in the + report. Silently using another model would make the report claim a model it did + not run. + - Once the files are cached, no prompt is shown. +- **Report:** the `models` block gains `k`. When the accurate profile was requested but + not used, it also gains `nli_requested: accurate ()`. The verdict cache is keyed by + model id, k and thresholds already, so the two profiles never share cached verdicts. + +## Profiles + +An `NliProfile` (in `entailment.py`) has these fields: + +- `name`: `"default"` or `"accurate"`. +- `repo` and `revision`: the revision is the full SHA. +- `onnx_file(machine) -> str`. +- `k`. +- `thresholds`. +- `size_mb`: shown in the consent prompt. + +| profile | repo @ revision | file | k | decide | medium | high | +|---|---|---|---|---|---|---| +| default | `cross-encoder/nli-deberta-v3-base` @ `6c749ce3425cd33b46d187e45b92bbf96ee12ec7` | `pick_onnx_file` | 1 | 0.45 | 0.457948 | 0.99933 | +| accurate | `MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli` @ `b3546ea6b0346eb6f8d5d68b13c7dc6d0376b3d7` | `onnx/model_quantized.onnx` | 2 | 0.25 | 0.252721 | 0.999142 | + +The default profile's values are today's values, unchanged. The accurate profile's cuts +come from SciFact train with rule-decided rows left out of the tier walk. + +On SciFact dev these cuts give: + +- `high`: 15 of 15 right (fewer than 20 verdicts, so too few to judge); +- `medium`: 114 of 151 right (0.755); +- `low`: empty. + +`providers_for` treats a `quantized` file stem as int8 too, so CoreML is skipped for the +accurate file, as it is for the default one. Without this, a Mac would run the large +model on CoreML, which was measured 3× slower for int8 graphs. + +## Tier note + +`pipeline.tier_note(thresholds)` already returns `NO_HIGH_TIER` when `high >= 1.0`. It now +also returns `NO_LOW_TIER` when `medium - decide < 0.01` (the margin `eval_scifact` uses +for the same warning). The two notes are joined with `"; "`. + +This changes the default profile's output too: its `medium` (0.457948) sits 0.008 above +`decide` (0.45). The 2026-09-12 tier doc already says this ("`low` is effectively +empty"), so the report now admits what the measurement found. + +## Out of scope + +- Making the large model the default. +- Re-calibrating the default profile. +- A multilingual model. +- Fixing AVeriTeC. +- A TUI flag for `/check`. The TUI uses the config key. + +## Tests (no network) + +- **Profiles and providers:** profile values; `providers_for` on a `quantized` stem. +- **Tier note:** each note alone, and both together. +- **Config:** the new section and key parse; an unknown value is refused; `set_value` + round trip. +- **Model gate:** allow, deny, ask with TTY (all four answers), and ask without TTY. + Always and never are persisted. A failed download falls back. +- **Engine:** the accurate profile resolves to its k and thresholds when installed. It + falls back with the reason when consent is denied. `Engine.default` still never builds + a model. +- **CLI:** `--accurate` reaches `Engine.default`. +- **TUI:** the `/config` rows exist, and the model prompt renders. +- **Report:** `k` and `nli_requested` appear in `models`. The golden JSON is updated. From 5ff6cf1e9c325e93b53560655a993dd1b047881a Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 12:55:22 +0300 Subject: [PATCH 2/7] feat(nli): add the accurate NLI profile, its config keys and a low-tier note The accurate profile is the bake-off winner at k=2, with cuts calibrated on SciFact train and rule-decided rows left out of the tier walk. A run whose medium cut sits on decide now says it has no real low tier. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- scripts/eval_scifact.py | 2 +- src/proofpath/config.py | 20 +++++ src/proofpath/entailment.py | 7 +- src/proofpath/pipeline.py | 32 ++++++-- src/proofpath/profiles.py | 101 +++++++++++++++++++++++ src/proofpath/report.py | 8 +- src/proofpath/settings_hints.py | 7 +- tests/data/verify-report-golden.json | 2 +- tests/test_config.py | 59 ++++++++++++++ tests/test_config_cli.py | 36 +++++++- tests/test_entailment.py | 118 +++++++++++++++++++++++++++ tests/test_pipeline.py | 32 +++++++- tests/test_verify.py | 7 +- 13 files changed, 412 insertions(+), 19 deletions(-) create mode 100644 src/proofpath/profiles.py diff --git a/scripts/eval_scifact.py b/scripts/eval_scifact.py index 01e31c4..b3c03ba 100644 --- a/scripts/eval_scifact.py +++ b/scripts/eval_scifact.py @@ -47,7 +47,7 @@ CUT_DECIMALS = pipeline.CUT_DECIMALS # How close `medium` may sit to `decide` before the `low` tier stops meaning # anything: inside this margin every asserted verdict is medium or better. -LOW_TIER_MARGIN = 0.01 +LOW_TIER_MARGIN = pipeline.LOW_TIER_MARGIN PROVIDER_NAMES = { "cuda": "CUDAExecutionProvider", "coreml": "CoreMLExecutionProvider", diff --git a/src/proofpath/config.py b/src/proofpath/config.py index 14b280e..68b39ec 100644 --- a/src/proofpath/config.py +++ b/src/proofpath/config.py @@ -33,6 +33,9 @@ class ConfigError(ValueError): class Permissions: # Step 3 of the fetch ladder: ~280 MB browser engine. Never installed silently. install_browser: Permission = "ask" + # The accurate NLI profile: a 643 MB model, downloaded only on consent (rule 5). + # Denied or failed, the run falls back to the default profile and says so. + install_model: Permission = "ask" network: Permission = "allow" # Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). Only matters # once ``search.provider`` is set: configuring a provider is the consent. The @@ -91,6 +94,20 @@ class SearchConfig: results_per_claim: int = 3 +NliProfileName = Literal["default", "accurate"] +NLI_PROFILE_NAMES: tuple[str, ...] = get_args(NliProfileName) + + +@dataclass(frozen=True) +class ModelsConfig: + """Which NLI profile a run uses (``profiles.PROFILES``). ``default`` is the small + model every install ships with; ``accurate`` is the opt-in large one, better on + scientific sources and not on news (docs/superpowers/specs/ + 2026-10-01-accurate-nli-design.md).""" + + nli: NliProfileName = "default" + + @dataclass(frozen=True) class Config: permissions: Permissions = field(default_factory=Permissions) @@ -98,6 +115,7 @@ class Config: contact: Contact = field(default_factory=Contact) judge: JudgeConfig = field(default_factory=JudgeConfig) search: SearchConfig = field(default_factory=SearchConfig) + models: ModelsConfig = field(default_factory=ModelsConfig) _SECTIONS: dict[str, type] = { @@ -106,6 +124,7 @@ class Config: "contact": Contact, "judge": JudgeConfig, "search": SearchConfig, + "models": ModelsConfig, } @@ -183,6 +202,7 @@ def _resolve_type(annotation: Any) -> Any: return { "Permission": Permission, "SearchProvider": SearchProvider, + "NliProfileName": NliProfileName, "bool": bool, "int": int, "str": str, diff --git a/src/proofpath/entailment.py b/src/proofpath/entailment.py index 66343e9..dd4ab5d 100644 --- a/src/proofpath/entailment.py +++ b/src/proofpath/entailment.py @@ -62,12 +62,15 @@ def providers_for(onnx_file: str, available: Sequence[str]) -> list[str]: The project rule is CUDA -> CoreML -> CPU. For the int8 exports CoreML is skipped: measured 2026-09-11 on Apple Silicon, CoreML handles 880 of the 2,524 - nodes and the partitioning makes it 3x slower than plain CPU. + nodes and the partitioning makes it 3x slower than plain CPU. ``quantized`` is the + same dynamic int8 under another repo's name (the accurate profile's only export), + and without it a Mac would run the large model on that slow path. """ from proofpath.device import onnx_providers_from chosen = onnx_providers_from(available) - if "int8" in Path(onnx_file).stem: + stem = Path(onnx_file).stem + if "int8" in stem or "quantized" in stem: chosen = [p for p in chosen if p != "CoreMLExecutionProvider"] return chosen diff --git a/src/proofpath/pipeline.py b/src/proofpath/pipeline.py index dfad7db..7e9b60e 100644 --- a/src/proofpath/pipeline.py +++ b/src/proofpath/pipeline.py @@ -59,20 +59,40 @@ def tier(self, score: float) -> Tier: # What a report has to admit when the calibration leaves no reachable ``high`` band. NO_HIGH_TIER = ( - "this model earns no high tier on SciFact dev; medium is the strongest confidence shown" + "this model earns no high tier on SciFact dev, so medium is the strongest confidence shown" ) +# What it has to admit when ``medium`` sits on ``decide``: the ``low`` band between +# them is too thin to hold an asserted verdict, so ``low`` is in practice the tier of +# NEI alone. It names SciFact, not a split, because the two profiles were calibrated +# on different ones (dev for the default, train for the accurate one). +NO_LOW_TIER = ( + "this model has no real low tier on SciFact, so an asserted verdict is almost always " + "medium or better" +) +# How close ``medium`` may sit to ``decide`` before ``low`` stops meaning anything. +# The same margin ``scripts/eval_scifact.py`` warns at, so the harness and the report +# agree on which calibrations have two tiers rather than three. +LOW_TIER_MARGIN = 0.01 def tier_note(thresholds: Thresholds) -> str: - """What a run using these thresholds owes its reader about its top tier. + """What a run using these thresholds owes its reader about its tiers. - Derived, never hand-set: a recalibration that loses the high tier starts saying so - on its own, and one that keeps it cannot leave a stale apology in every report. A + Derived, never hand-set: a recalibration that loses a tier starts saying so on its + own, and one that keeps it cannot leave a stale apology in every report. A ``high`` cut of exactly 1.0 is ``eval.metrics.tier_cutpoints`` reporting that no score below 1.0 held the precision target — only a rule-decided verdict (spec section 10, which - scores exactly 1.0) could reach it, and that is not the model earning a tier. + scores exactly 1.0) could reach it, and that is not the model earning a tier. A + ``medium`` cut within ``LOW_TIER_MARGIN`` of ``decide`` is the other end of the same + story (docs/eval/2026-09-12-tiers.md: ``low`` held 1 of 135 asserted verdicts). Both + can hold at once, and then both are said, joined with ``"; "``. """ - return NO_HIGH_TIER if thresholds.high >= 1.0 else "" + notes: list[str] = [] + if thresholds.high >= 1.0: + notes.append(NO_HIGH_TIER) + if thresholds.medium - thresholds.decide < LOW_TIER_MARGIN: + notes.append(NO_LOW_TIER) + return "; ".join(notes) _SUPPORTED = LABEL_ORDER.index(Label.SUPPORTED) diff --git a/src/proofpath/profiles.py b/src/proofpath/profiles.py new file mode 100644 index 0000000..1341dc4 --- /dev/null +++ b/src/proofpath/profiles.py @@ -0,0 +1,101 @@ +"""NLI profiles: which model, at which k, with which calibrated cuts. + +A profile is the unit a run chooses, because the three travel together: thresholds +are calibrated for one model at one k, and moving any of them alone makes the tiers a +report prints mean something they were never measured to mean. Design and numbers: +docs/superpowers/specs/2026-10-01-accurate-nli-design.md. + +This lives beside ``entailment`` rather than in it because a profile carries +``pipeline.Thresholds`` and ``pipeline`` already imports ``entailment``: defining the +profiles there would make the two modules import each other. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType + +from proofpath.entailment import DEFAULT_REPO, DEFAULT_REVISION, pick_onnx_file +from proofpath.pipeline import DEFAULT_THRESHOLDS, Thresholds + + +@dataclass(frozen=True) +class NliProfile: + """One NLI model, pinned, with the k and cuts it was calibrated at. + + ``onnx_file`` maps ``platform.machine()`` to the export inside the repo, since a + repo may ship one graph per CPU family. ``size_mb`` is what the consent prompt + states before a download (product rule 5). + """ + + name: str + repo: str + revision: str + onnx_file: Callable[[str], str] + k: int + thresholds: Thresholds + size_mb: int + + +def _quantized_file(_machine: str) -> str: + # The repo ships one dynamic-int8 export for every CPU. + return "onnx/model_quantized.onnx" + + +# Today's model, values unchanged: SciFact dev, 2026-09-12, k=1 +# (docs/eval/2026-09-12-tiers.md). +DEFAULT_PROFILE = NliProfile( + name="default", + repo=DEFAULT_REPO, + revision=DEFAULT_REVISION, + onnx_file=pick_onnx_file, + k=1, + thresholds=DEFAULT_THRESHOLDS, + size_mb=244, +) + +# The bake-off winner (docs/eval/2026-10-01-nli-bakeoff.md): SciFact dev macro-F1 +# 0.697 against 0.580, at 643 MB and roughly 3x the time per pair. Opt-in, never the +# default. Its cuts come from SciFact train with rule-decided rows (score exactly 1.0) +# left out of the tier walk: they fired 4-6 times there and none was right, and kept +# in they pinned ``high`` at 1.0. They still count toward ``decide``. +ACCURATE_PROFILE = NliProfile( + name="accurate", + repo="MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli", + revision="b3546ea6b0346eb6f8d5d68b13c7dc6d0376b3d7", + onnx_file=_quantized_file, + k=2, + thresholds=Thresholds(decide=0.25, high=0.999142, medium=0.252721), + size_mb=643, +) + +#: By ``models.nli`` value. Read-only, so no caller can swap a profile under a run. +PROFILES: Mapping[str, NliProfile] = MappingProxyType( + {profile.name: profile for profile in (DEFAULT_PROFILE, ACCURATE_PROFILE)} +) + + +def profile_installed(profile: NliProfile, cache_dir: Path, machine: str) -> bool: + """True when every file ``OnnxNli`` would fetch for this profile is already cached. + + Reads the local Hugging Face cache only, never the network: this is what decides + whether a run has to ask before a download, so it must be answerable offline and + in CI. The ONNX file is the one this machine would run, so an arm64 cache does not + count as installed on x86_64. ``try_to_load_from_cache`` returns a path string for + a cached file and ``None`` or a "known missing" sentinel otherwise; only the + string means the file is there. + """ + from huggingface_hub import try_to_load_from_cache + + wanted = ("config.json", "tokenizer.json", profile.onnx_file(machine)) + return all( + isinstance( + try_to_load_from_cache( + profile.repo, filename, cache_dir=str(cache_dir), revision=profile.revision + ), + str, + ) + for filename in wanted + ) diff --git a/src/proofpath/report.py b/src/proofpath/report.py index 3591dbd..6a2cd67 100644 --- a/src/proofpath/report.py +++ b/src/proofpath/report.py @@ -345,8 +345,8 @@ class Report: models: dict[str, str] api_calls: int elapsed: float - # What this run's thresholds owe the reader about their top tier (spec section 14), - # from ``pipeline.tier_note``. Empty when the calibration earned a `high` band. It + # What this run's thresholds owe the reader about their tiers (spec section 14), + # from ``pipeline.tier_note``. Empty when every tier is reachable. It # belongs to the run and not to the process: two runs in one session may be # calibrated differently, and each report must say what its own numbers support. tier_note: str = "" @@ -636,8 +636,8 @@ class Footer: # Citation markers left over when no bibliography was found at all: the # denominator is 0, so ``weak`` cannot speak for them (see ``_unchecked``). unchecked_markers: int = 0 - # ``Report.tier_note`` when the calibration left no reachable ``high`` band, so - # a run never shows `medium` as if a stronger tier existed and was withheld. + # ``Report.tier_note`` when the calibration left a tier unreachable or empty, so a + # run never shows `medium` as if a stronger or weaker tier existed and was withheld. note: str | None = None # Sources that stopped at the section 7.1 consent gate, from # ``Coverage.browser_skipped``. The reason they are unread is a permission, not a diff --git a/src/proofpath/settings_hints.py b/src/proofpath/settings_hints.py index 7a186c7..4127fb7 100644 --- a/src/proofpath/settings_hints.py +++ b/src/proofpath/settings_hints.py @@ -1,7 +1,7 @@ -"""The two ``config set`` spellings a "not permitted" line points at, named once. +"""The ``config set`` spellings a "not permitted" line points at, named once. A leaf: it imports nothing from the package, so the fetch ladder, the report and the -output layer can all read the same two strings without a cycle (``report`` imports +output layer can all read the same strings without a cycle (``report`` imports ``fetch``, and ``fetch`` needs ``NETWORK_SETTING`` for its denied note). Each surface puts its own command in front: ``/config set`` in the TUI, ``proofpath config set`` on the command line and in the report. The fact a line states comes first; the @@ -13,6 +13,9 @@ #: ``permissions.install_browser`` set to ``ask``: the browser step of the fetch #: ladder asks before it installs and runs Chromium. BROWSER_SETTING = "permissions.install_browser ask" +#: ``permissions.install_model`` set to ``ask``: a run that wants the accurate NLI +#: profile asks before it downloads the model, instead of falling back to the default. +MODEL_SETTING = "permissions.install_model ask" #: ``permissions.network`` set to ``allow``: every network-bound stage runs. NETWORK_SETTING = "permissions.network allow" #: ``search.provider`` set to ``tavily``: a text that cites nothing is searched with diff --git a/tests/data/verify-report-golden.json b/tests/data/verify-report-golden.json index b3d7583..e81e18d 100644 --- a/tests/data/verify-report-golden.json +++ b/tests/data/verify-report-golden.json @@ -641,7 +641,7 @@ }, "api_calls": 0, "elapsed": 0.0, - "tier_note": "", + "tier_note": "this model has no real low tier on SciFact, so an asserted verdict is almost always medium or better", "cancelled": false, "summary": null, "summary_model": null, diff --git a/tests/test_config.py b/tests/test_config.py index d000912..7b1cf88 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -162,3 +162,62 @@ def test_set_value_takes_a_search_provider(tmp_path: Path) -> None: assert cfg.set_value("search.provider", "tavily", path).search.provider == "tavily" with pytest.raises(cfg.ConfigError, match=r"search\.provider"): cfg.set_value("search.provider", "bing", path) + + +# --- the accurate NLI profile (docs/superpowers/specs/2026-10-01-accurate-nli-design.md) + + +def test_the_default_model_is_the_default_profile_and_its_install_asks() -> None: + config = cfg.Config() + assert config.models == cfg.ModelsConfig() + assert config.models.nli == "default" + assert cfg.NLI_PROFILE_NAMES == ("default", "accurate") + # Rule 5: a 643 MB download is never made without consent. + assert config.permissions.install_model == "ask" + + +def test_the_models_section_and_the_install_permission_load_from_toml(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + path.write_text( + '[models]\nnli = "accurate"\n\n[permissions]\ninstall_model = "allow"\n', + encoding="utf-8", + ) + config = cfg.load_config(path) + assert config.models.nli == "accurate" + assert config.permissions.install_model == "allow" + + +@pytest.mark.parametrize("body", ['nli = "fast"', "nli = 1", 'nli = "Accurate"']) +def test_an_unknown_nli_profile_is_refused_by_name(tmp_path: Path, body: str) -> None: + path = tmp_path / "config.toml" + path.write_text(f"[models]\n{body}\n", encoding="utf-8") + with pytest.raises(cfg.ConfigError, match=r"models\.nli"): + cfg.load_config(path) + + +def test_a_bad_install_model_permission_is_refused(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + path.write_text('[permissions]\ninstall_model = "sometimes"\n', encoding="utf-8") + with pytest.raises(cfg.ConfigError, match=r"permissions\.install_model"): + cfg.load_config(path) + + +def test_set_value_takes_the_nli_profile_and_round_trips_it(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + updated = cfg.set_value("models.nli", "accurate", path) + assert updated.models.nli == "accurate" + text = path.read_text(encoding="utf-8") + assert "[models]" in text.splitlines() + assert 'nli = "accurate"' in text.splitlines() + assert text == cfg.render_config(updated) + assert cfg.load_config(path) == updated + with pytest.raises(cfg.ConfigError, match=r"models\.nli"): + cfg.set_value("models.nli", "fast", path) + assert cfg.load_config(path).models.nli == "accurate" + + +def test_set_value_takes_the_install_model_permission(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + assert cfg.set_value("permissions.install_model", "deny", path).permissions == ( + cfg.Permissions(install_model="deny") + ) diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py index d54824e..3edaa0b 100644 --- a/tests/test_config_cli.py +++ b/tests/test_config_cli.py @@ -28,7 +28,7 @@ def test_config_prints_path_and_every_section_as_toml(config_dir: Path) -> None: assert result.exit_code == 0, result.output assert str(config_dir / "config.toml") in result.stdout assert "not written yet" in result.stdout - for section in ("[permissions]", "[fetch]", "[contact]", "[judge]", "[search]"): + for section in ("[permissions]", "[fetch]", "[contact]", "[judge]", "[search]", "[models]"): assert section in result.stdout assert 'install_browser = "ask"' in result.stdout assert "GROQ_API_KEY" in result.stdout @@ -166,3 +166,37 @@ def test_the_search_provider_values_and_the_hinted_settings_are_the_real_ones( result = runner.invoke(app, ["config", "set", *setting.split()]) assert result.exit_code == 0, result.output assert result.stdout.startswith(f"{setting.replace(' ', ' = ', 1)} (") + + +# --- the accurate NLI profile --------------------------------------------------------- + + +def test_config_renders_the_models_section_and_the_install_model_permission( + config_dir: Path, +) -> None: + lines = runner.invoke(app, ["config"]).stdout.splitlines() + assert 'nli = "default"' in lines + assert 'install_model = "ask"' in lines + + +def test_config_set_models_nli_accurate_persists(config_dir: Path) -> None: + result = runner.invoke(app, ["config", "set", "models.nli", "accurate"]) + assert result.exit_code == 0, result.output + assert result.stdout.startswith("models.nli = accurate (") + assert 'nli = "accurate"' in runner.invoke(app, ["config"]).stdout.splitlines() + + +def test_config_set_models_nli_refuses_an_unknown_profile(config_dir: Path) -> None: + result = runner.invoke(app, ["config", "set", "models.nli", "fast"]) + assert result.exit_code == 2 + assert "models.nli" in result.output + assert not (config_dir / "config.toml").exists() + + +def test_the_model_setting_hint_is_a_command_config_set_accepts(config_dir: Path) -> None: + from proofpath.settings_hints import MODEL_SETTING + + assert MODEL_SETTING == "permissions.install_model ask" + result = runner.invoke(app, ["config", "set", *MODEL_SETTING.split()]) + assert result.exit_code == 0, result.output + assert result.stdout.startswith("permissions.install_model = ask (") diff --git a/tests/test_entailment.py b/tests/test_entailment.py index caab924..61231b3 100644 --- a/tests/test_entailment.py +++ b/tests/test_entailment.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from pathlib import Path import numpy as np import pytest @@ -93,3 +94,120 @@ def test_close_drops_the_session_and_is_idempotent() -> None: assert scorer._tokenizer is None assert scorer._input_names == [] assert scorer.close() is None + + +def test_quantized_export_also_skips_coreml() -> None: + # The accurate profile's one export is dynamic int8 under another name; without + # this a Mac would run the large model on CoreML, measured 3x slower for int8. + available = ["CoreMLExecutionProvider", "CPUExecutionProvider"] + assert ent.providers_for("onnx/model_quantized.onnx", available) == ["CPUExecutionProvider"] + assert ent.providers_for( + "onnx/model_quantized.onnx", ["CUDAExecutionProvider", *available] + ) == ["CUDAExecutionProvider", "CPUExecutionProvider"] + + +# --- NLI profiles (docs/superpowers/specs/2026-10-01-accurate-nli-design.md) ------ + + +def test_the_default_profile_is_todays_model_unchanged() -> None: + from proofpath import profiles + from proofpath.pipeline import DEFAULT_THRESHOLDS, Thresholds + + default = profiles.DEFAULT_PROFILE + assert default.name == "default" + assert default.repo == "cross-encoder/nli-deberta-v3-base" == ent.DEFAULT_REPO + assert default.revision == "6c749ce3425cd33b46d187e45b92bbf96ee12ec7" == ent.DEFAULT_REVISION + assert default.onnx_file is ent.pick_onnx_file + assert default.k == 1 + assert default.thresholds == Thresholds(decide=0.45, high=0.99933, medium=0.457948) + assert default.thresholds == DEFAULT_THRESHOLDS + assert default.size_mb == 244 + + +def test_the_accurate_profile_carries_the_bakeoff_winner() -> None: + from proofpath import profiles + from proofpath.pipeline import Thresholds + + accurate = profiles.ACCURATE_PROFILE + assert accurate.name == "accurate" + assert accurate.repo == "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli" + assert accurate.revision == "b3546ea6b0346eb6f8d5d68b13c7dc6d0376b3d7" + for machine in ("arm64", "x86_64", "riscv64"): + assert accurate.onnx_file(machine) == "onnx/model_quantized.onnx" + assert accurate.k == 2 + assert accurate.thresholds == Thresholds(decide=0.25, high=0.999142, medium=0.252721) + assert accurate.size_mb == 643 + + +def test_profiles_are_keyed_by_name_and_match_the_config_choices() -> None: + from proofpath import profiles + from proofpath.config import NLI_PROFILE_NAMES + + assert dict(profiles.PROFILES) == { + "default": profiles.DEFAULT_PROFILE, + "accurate": profiles.ACCURATE_PROFILE, + } + assert set(profiles.PROFILES) == set(NLI_PROFILE_NAMES) + + +def test_a_profile_is_frozen() -> None: + import dataclasses + + from proofpath import profiles + + with pytest.raises(dataclasses.FrozenInstanceError): + profiles.ACCURATE_PROFILE.k = 1 # type: ignore[misc] + + +def _fake_cache(cached: set[str], seen: list[tuple[str, str, str | None, str]]) -> object: + from huggingface_hub import _CACHED_NO_EXIST + + def lookup( + repo_id: str, + filename: str, + cache_dir: str | None = None, + revision: str | None = None, + repo_type: str | None = None, + ) -> object: + seen.append((repo_id, filename, revision, str(cache_dir))) + if filename in cached: + return f"/snapshots/{filename}" + # A cached "this file does not exist" is not the file being there. + return _CACHED_NO_EXIST if filename == "config.json" else None + + return lookup + + +def test_profile_installed_needs_config_tokenizer_and_the_onnx_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import huggingface_hub + + from proofpath import profiles + + seen: list[tuple[str, str, str | None, str]] = [] + every = {"config.json", "tokenizer.json", "onnx/model_quantized.onnx"} + monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", _fake_cache(every, seen)) + assert profiles.profile_installed(profiles.ACCURATE_PROFILE, tmp_path, "arm64") is True + repo, revision = profiles.ACCURATE_PROFILE.repo, profiles.ACCURATE_PROFILE.revision + assert {(r, rev, d) for r, _, rev, d in seen} == {(repo, revision, str(tmp_path))} + assert {f for _, f, _, _ in seen} == every + + for missing in sorted(every): + monkeypatch.setattr( + huggingface_hub, "try_to_load_from_cache", _fake_cache(every - {missing}, []) + ) + assert profiles.profile_installed(profiles.ACCURATE_PROFILE, tmp_path, "arm64") is False + + +def test_profile_installed_asks_for_the_file_this_machine_would_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import huggingface_hub + + from proofpath import profiles + + cached = {"config.json", "tokenizer.json", "onnx/model_qint8_arm64.onnx"} + monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", _fake_cache(cached, [])) + assert profiles.profile_installed(profiles.DEFAULT_PROFILE, tmp_path, "arm64") is True + assert profiles.profile_installed(profiles.DEFAULT_PROFILE, tmp_path, "x86_64") is False diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index b7209e7..6de2007 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -7,7 +7,9 @@ from proofpath.models import Label, Passage from proofpath.pipeline import ( DEFAULT_THRESHOLDS, + LOW_TIER_MARGIN, NO_HIGH_TIER, + NO_LOW_TIER, Thresholds, decide, decide_indexed, @@ -157,7 +159,35 @@ def test_tier_note_is_written_exactly_when_no_high_tier_can_be_earned() -> None: # leave a report claiming a tier the split did not support — or apologising for # one it did. assert tier_note(Thresholds(decide=0.45, high=1.0, medium=0.5)) == NO_HIGH_TIER - assert tier_note(Thresholds(decide=0.45, high=0.99933, medium=0.457948)) == "" + assert tier_note(Thresholds(decide=0.45, high=0.99933, medium=0.5)) == "" + + +def test_tier_note_admits_a_medium_cut_that_sits_on_decide() -> None: + # The shipped default: 0.457948 is 0.008 above 0.45, so `low` holds almost no + # asserted verdict, and the 2026-09-12 tier doc already says so. + assert LOW_TIER_MARGIN == 0.01 + assert tier_note(Thresholds(decide=0.45, high=0.99933, medium=0.457948)) == NO_LOW_TIER + assert tier_note(DEFAULT_THRESHOLDS) == NO_LOW_TIER + # Exactly on decide is the extreme case of the same thing. + assert tier_note(Thresholds(decide=0.45, high=0.99933, medium=0.45)) == NO_LOW_TIER + # One margin or more above decide is a real band. + assert tier_note(Thresholds(decide=0.45, high=0.99933, medium=0.47)) == "" + + +def test_tier_note_joins_both_notes_when_both_tiers_are_missing() -> None: + both = tier_note(Thresholds(decide=0.45, high=1.0, medium=0.455)) + assert both == f"{NO_HIGH_TIER}; {NO_LOW_TIER}" + + +def test_the_two_notes_read_as_one_sentence_each() -> None: + # Joined with "; " into one line of the footer and the markdown preamble, so + # neither may carry its own full stop or capital. + for note in (NO_HIGH_TIER, NO_LOW_TIER): + assert note == note.strip() + assert note[0].islower() + assert not note.endswith(".") + # A "; " inside one note would blur where it ends and the next begins. + assert "; " not in note def test_a_high_cut_of_one_leaves_medium_as_the_strongest_tier_a_score_can_reach() -> None: diff --git a/tests/test_verify.py b/tests/test_verify.py index 3ffdbdc..9100b73 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -1284,7 +1284,12 @@ def test_a_run_carries_the_tier_note_of_the_thresholds_it_used() -> None: no_high_tier = paper_engine() no_high_tier.thresholds = Thresholds(decide=0.45, high=1.0, medium=0.5) assert verify(text, no_high_tier).tier_note == pipeline.NO_HIGH_TIER - assert verify(text, paper_engine()).tier_note == "" + no_tier_gap = paper_engine() + no_tier_gap.thresholds = Thresholds(decide=0.45, high=0.99933, medium=0.5) + assert verify(text, no_tier_gap).tier_note == "" + # The shipped default's ``medium`` sits 0.008 above ``decide``, so its runs admit + # that ``low`` holds almost no asserted verdict. + assert verify(text, paper_engine()).tier_note == pipeline.NO_LOW_TIER def test_the_verifying_stage_emits_its_own_events() -> None: From 6146fc25cb729669519795d51531150ba82159c6 Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 12:57:10 +0300 Subject: [PATCH 3/7] feat(nli): add a consent gate for downloading the accurate model Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- src/proofpath/model_gate.py | 214 ++++++++++++++++++++++ tests/test_model_gate.py | 348 ++++++++++++++++++++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 src/proofpath/model_gate.py create mode 100644 tests/test_model_gate.py diff --git a/src/proofpath/model_gate.py b/src/proofpath/model_gate.py new file mode 100644 index 0000000..98e50de --- /dev/null +++ b/src/proofpath/model_gate.py @@ -0,0 +1,214 @@ +"""The consent gate in front of the accurate NLI profile's download. + +The accurate profile's model is 643 MB, fetched onto the user's machine, so it is +never downloaded silently (product rule 5) and never asked about without a terminal +(product rule 4). ``ModelGate`` mirrors ``browser.ConsentGate``: the same permission +values, the same four answers, "always" and "never" saved to the config. It differs +in one place on purpose: a denied or failed download is not a skipped source but a +fallback to the default profile, which the caller reports (product rule 6), so the +gate's job ends at returning a ``Decision`` whose reason a report can print. + +Design: docs/superpowers/specs/2026-10-01-accurate-nli-design.md, "User-facing +behaviour". +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import typer + +from proofpath.browser import _CHOICE_ANSWERS, PROMPT_CHOICES, TERMINAL_ANSWERS, Answer +from proofpath.config import ConfigError, Decision, Permission, resolve_permission, set_value +from proofpath.profiles import PROFILES, NliProfile + +#: The key "always" and "never" are saved under. +PERMISSION_KEY = "permissions.install_model" + +# What a model question passes as the ``host`` argument of a gate prompt. The prompt +# callable is the browser gate's, shared so the TUI answers both through one widget; +# a real host never starts with this (a ``host:port`` port is numeric), so a prompt +# can tell the two questions apart with ``subject_profile``. +_SUBJECT_PREFIX = "model:" + + +def prompt_subject(profile: NliProfile) -> str: + """The first argument the gate passes to an injected prompt for ``profile``.""" + return f"{_SUBJECT_PREFIX}{profile.name}" + + +def subject_profile(subject: str) -> NliProfile | None: + """The profile a prompt subject names, or ``None`` for a blocked host. + + Lets a shared prompt (the TUI's) render ``model_prompt_text`` instead of the + browser question when the gate asking is this one. + """ + if not subject.startswith(_SUBJECT_PREFIX): + return None + return PROFILES.get(subject[len(_SUBJECT_PREFIX) :]) + + +def model_prompt_text(profile: NliProfile, *, answers: str = TERMINAL_ANSWERS) -> str: + """The consent block for one profile, laid out like ``browser.prompt_text``: what + is wanted, what it costs, then the answers, separated by blank lines. + + ``answers`` is the last line: the terminal's keys by default, or ``TUI_ANSWERS``. + The size is stated before anything is fetched, because rule 5 is about the user + knowing what they agree to, not only about being asked. + """ + lines = [ + f" ⚠ The {profile.name} NLI profile needs a model that is not downloaded yet.", + "", + " proofpath can fetch it once into its model cache:", + "", + f" {profile.repo}", + f" revision {profile.revision[:7]} {profile.size_mb} MB", + "", + " Without it this run uses the default profile and says so.", + "", + answers, + ] + return "\n".join(lines) + + +def ask_model_terminal(profile: NliProfile) -> Answer: + """The default prompt. ``ModelGate`` never calls this without a TTY. + + Same keys and the same stderr channel as ``browser.ask_terminal`` (spec section + 13.3: ``--format json`` on stdout stays one document); only the question differs, + which is why this is not ``ask_terminal`` itself. + """ + typer.echo(model_prompt_text(profile), err=True) + while True: + raw = typer.prompt("Allow?", default="n", err=True).strip().lower() + if raw in _CHOICE_ANSWERS: + return _CHOICE_ANSWERS[raw] + typer.echo(f"Please answer one of: {', '.join(PROMPT_CHOICES)}", err=True) + + +def download_profile(profile: NliProfile, cache_dir: Path, machine: str) -> None: + """Fetch the three files ``OnnxNli`` loads for ``profile`` on ``machine``. + + Exactly the files ``profiles.profile_installed`` checks, at the pinned revision + and into the same cache, so a successful download is what makes the next run's + check say "installed" and skip the question. Imported lazily, like everywhere + else ``huggingface_hub`` is used, to keep it off the CLI's import path. + """ + from huggingface_hub import hf_hub_download + + cache_dir.mkdir(parents=True, exist_ok=True) + for filename in ("config.json", "tokenizer.json", profile.onnx_file(machine)): + hf_hub_download(profile.repo, filename, revision=profile.revision, cache_dir=str(cache_dir)) + + +def _failure_reason(exc: Exception) -> str: + """A short, safe description of a failed download. + + ``str(exc)`` is never used: Hugging Face HTTP errors carry the server's response + text, and that is not something a report should repeat. The type name, plus the + HTTP status when there is one, says enough to act on. + """ + status = getattr(getattr(exc, "response", None), "status_code", None) + if isinstance(status, int): + return f"{type(exc).__name__} (HTTP {status})" + return type(exc).__name__ + + +class ModelGate: + """Decides, once per profile per run, whether a profile's model may be used. + + ``ensure`` answers "allow" when the files are already cached, without asking. + Otherwise it resolves the stored permission, asks at most once when the + permission is ``ask`` and there is a terminal, and downloads on "allow". Every + decision, and the download itself, is written to ``log`` for the report. + + ``override`` records that ``--accurate`` chose the profile. It is not consent: + the flag says which model, not that 643 MB may be fetched, so it never changes + the decision. + """ + + def __init__( + self, + permission: Permission, + *, + interactive: bool, + override: bool | None = None, + prompt: Callable[[str, int | None], Answer] | None = None, + config_path: Path | None = None, + download: Callable[[NliProfile], None], + installed: Callable[[NliProfile], bool], + ) -> None: + self._permission = permission + self._interactive = interactive + self.override = override + self._prompt = prompt + self._config_path = config_path + self._download = download + self._installed = installed + self.log: list[str] = [] + # True once the user was shown the question this run. + self.asked = False + # By profile name: a second ``ensure`` neither asks nor downloads again, and + # a failed download is not retried within the run. + self._decided: dict[str, Decision] = {} + + def _ask(self, profile: NliProfile) -> Answer: + if self._prompt is None: + return ask_model_terminal(profile) + return self._prompt(prompt_subject(profile), None) + + def _persist(self, value: str) -> None: + """Save an "always"/"never" answer. A failure must not undo the answer already + given for this run, so it is logged, not raised.""" + try: + set_value(PERMISSION_KEY, value, self._config_path) + except (ConfigError, OSError) as exc: + self.log.append(f"could not save permission: {exc}") + + def _consent(self, profile: NliProfile) -> Decision: + decision = resolve_permission(self._permission, interactive=self._interactive) + if decision.outcome != "prompt": + return decision + self.asked = True + answer = self._ask(profile) + if answer == "once": + return Decision("allow", "user answered once") + if answer == "always": + self._persist("allow") + return Decision("allow", "user answered always") + if answer == "no": + return Decision("deny", "user answered no") + self._persist("deny") # "never" + return Decision("deny", "user answered never") + + def ensure(self, profile: NliProfile) -> Decision: + """Make ``profile``'s model usable, or say why it is not.""" + if profile.name in self._decided: + return self._decided[profile.name] + decision = self._ensure(profile) + self._decided[profile.name] = decision + return decision + + def _ensure(self, profile: NliProfile) -> Decision: + if self._installed(profile): + self.log.append(f"model {profile.name}: installed") + return Decision("allow", "installed") + + decision = self._consent(profile) + if decision.outcome != "allow": + self.log.append(f"model {profile.name}: not downloaded ({decision.reason})") + return decision + + self.log.append( + f"model {profile.name}: downloading {profile.repo} ({profile.size_mb} MB;" + f" {decision.reason})" + ) + try: + self._download(profile) + except Exception as exc: # a failed download falls back, it never crashes + failed = Decision("deny", f"model download failed: {_failure_reason(exc)}") + self.log.append(f"model {profile.name}: {failed.reason}") + return failed + self.log.append(f"model {profile.name}: downloaded") + return decision diff --git a/tests/test_model_gate.py b/tests/test_model_gate.py new file mode 100644 index 0000000..db08eeb --- /dev/null +++ b/tests/test_model_gate.py @@ -0,0 +1,348 @@ +"""ModelGate: the consent gate in front of the accurate NLI profile's download. + +No network and no real model: ``prompt``, ``download`` and ``installed`` are always +injected, ``hf_hub_download`` is monkeypatched, and the config lives under tmp_path. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from proofpath import model_gate as mg +from proofpath.browser import TERMINAL_ANSWERS, TUI_ANSWERS, Answer +from proofpath.config import load_config +from proofpath.profiles import ACCURATE_PROFILE, DEFAULT_PROFILE, NliProfile + + +@pytest.fixture(autouse=True) +def config_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + monkeypatch.setenv("PROOFPATH_CONFIG_DIR", str(tmp_path)) + return tmp_path + + +def raising_prompt(subject: str, status: int | None) -> Answer: + raise AssertionError("prompt must not be called") + + +def raising_download(profile: NliProfile) -> None: + raise AssertionError("download must not be called") + + +class Recorder: + """A ``download`` that remembers what it was asked to fetch.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def __call__(self, profile: NliProfile) -> None: + self.calls.append(profile.name) + + +def missing(profile: NliProfile) -> bool: + return False + + +# --- installed -------------------------------------------------------------------- + + +def test_installed_allows_without_prompt_or_download() -> None: + gate = mg.ModelGate( + "deny", + interactive=True, + prompt=raising_prompt, + download=raising_download, + installed=lambda p: True, + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "allow" + assert decision.reason == "installed" + + +# --- stored permission ------------------------------------------------------------ + + +def test_allow_permission_downloads_without_prompt() -> None: + download = Recorder() + gate = mg.ModelGate( + "allow", interactive=False, prompt=raising_prompt, download=download, installed=missing + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "allow" + assert decision.reason == "permission is set to allow" + assert download.calls == ["accurate"] + assert any("643 MB" in line for line in gate.log) + + +def test_deny_permission_never_downloads() -> None: + gate = mg.ModelGate( + "deny", + interactive=True, + prompt=raising_prompt, + download=raising_download, + installed=missing, + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "deny" + assert decision.reason == "permission is set to deny" + assert any("permission is set to deny" in line for line in gate.log) + + +def test_ask_without_tty_denies_and_reports() -> None: + gate = mg.ModelGate( + "ask", + interactive=False, + prompt=raising_prompt, + download=raising_download, + installed=missing, + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "deny" + assert decision.reason == "permission is set to ask but there is no interactive terminal" + assert any("no interactive terminal" in line for line in gate.log) + + +def test_override_does_not_grant_consent() -> None: + """``--accurate`` picks the profile; the 643 MB still needs a yes (rule 5).""" + gate = mg.ModelGate( + "ask", + interactive=False, + override=True, + prompt=raising_prompt, + download=raising_download, + installed=missing, + ) + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + + +# --- ask with a TTY: each answer ---------------------------------------------------- + + +@pytest.mark.parametrize( + ("answer", "outcome", "reason", "downloads"), + [ + ("once", "allow", "user answered once", ["accurate"]), + ("always", "allow", "user answered always", ["accurate"]), + ("no", "deny", "user answered no", []), + ("never", "deny", "user answered never", []), + ], +) +def test_ask_with_tty_each_answer( + answer: Answer, outcome: str, reason: str, downloads: list[str] +) -> None: + asked: list[tuple[str, int | None]] = [] + + def prompt(subject: str, status: int | None) -> Answer: + asked.append((subject, status)) + return answer + + download = Recorder() + gate = mg.ModelGate( + "ask", interactive=True, prompt=prompt, download=download, installed=missing + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == outcome + assert decision.reason == reason + assert download.calls == downloads + assert asked == [(mg.prompt_subject(ACCURATE_PROFILE), None)] + assert any(reason in line for line in gate.log) + + +def test_ask_is_asked_once_per_run() -> None: + asked: list[str] = [] + + def prompt(subject: str, status: int | None) -> Answer: + asked.append(subject) + return "no" + + gate = mg.ModelGate( + "ask", interactive=True, prompt=prompt, download=raising_download, installed=missing + ) + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + assert len(asked) == 1 + + +def test_download_runs_once_per_run() -> None: + download = Recorder() + gate = mg.ModelGate( + "allow", interactive=False, prompt=raising_prompt, download=download, installed=missing + ) + gate.ensure(ACCURATE_PROFILE) + gate.ensure(ACCURATE_PROFILE) + assert download.calls == ["accurate"] + + +# --- persistence -------------------------------------------------------------------- + + +def test_always_persists_allow(tmp_path: Path) -> None: + path = tmp_path / "custom" / "config.toml" + gate = mg.ModelGate( + "ask", + interactive=True, + prompt=lambda s, st: "always", + download=Recorder(), + installed=missing, + config_path=path, + ) + gate.ensure(ACCURATE_PROFILE) + assert load_config(path).permissions.install_model == "allow" + # The browser permission is a different key and stays as it was. + assert load_config(path).permissions.install_browser == "ask" + + +def test_never_persists_deny(tmp_path: Path) -> None: + path = tmp_path / "custom" / "config.toml" + gate = mg.ModelGate( + "ask", + interactive=True, + prompt=lambda s, st: "never", + download=raising_download, + installed=missing, + config_path=path, + ) + gate.ensure(ACCURATE_PROFILE) + assert load_config(path).permissions.install_model == "deny" + + +@pytest.mark.parametrize("answer", ["once", "no"]) +def test_once_and_no_are_not_persisted(tmp_path: Path, answer: Answer) -> None: + path = tmp_path / "config.toml" + gate = mg.ModelGate( + "ask", + interactive=True, + prompt=lambda s, st: answer, + download=Recorder(), + installed=missing, + config_path=path, + ) + gate.ensure(ACCURATE_PROFILE) + assert not path.exists() + + +def test_persist_failure_is_logged_not_raised(tmp_path: Path) -> None: + blocked = tmp_path / "not-a-directory" + blocked.write_text("nope", encoding="utf-8") + gate = mg.ModelGate( + "ask", + interactive=True, + prompt=lambda s, st: "always", + download=Recorder(), + installed=missing, + config_path=blocked / "config.toml", + ) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "allow" # the in-run answer is kept + assert any("could not save permission" in line for line in gate.log) + + +# --- download failure --------------------------------------------------------------- + + +class FakeResponse: + status_code = 503 + text = "secret upstream body" + + +class FakeHttpError(Exception): + def __init__(self) -> None: + super().__init__("503 Server Error: secret upstream body") + self.response = FakeResponse() + + +def test_download_failure_denies_with_a_short_reason() -> None: + def download(profile: NliProfile) -> None: + raise FakeHttpError + + gate = mg.ModelGate("allow", interactive=False, download=download, installed=missing) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.outcome == "deny" + assert decision.reason == "model download failed: FakeHttpError (HTTP 503)" + assert all("secret upstream body" not in line for line in gate.log) + assert all("Traceback" not in line for line in gate.log) + + +def test_download_failure_without_a_response_names_the_type() -> None: + def download(profile: NliProfile) -> None: + raise ConnectionError("dns failure for huggingface.co: raw details") + + gate = mg.ModelGate("allow", interactive=False, download=download, installed=missing) + decision = gate.ensure(ACCURATE_PROFILE) + assert decision.reason == "model download failed: ConnectionError" + + +def test_download_failure_is_not_retried() -> None: + calls: list[str] = [] + + def download(profile: NliProfile) -> None: + calls.append(profile.name) + raise OSError("disk full") + + gate = mg.ModelGate("allow", interactive=False, download=download, installed=missing) + gate.ensure(ACCURATE_PROFILE) + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + assert calls == ["accurate"] + + +# --- the prompt text ----------------------------------------------------------------- + + +def test_prompt_text_names_repo_and_size() -> None: + text = mg.model_prompt_text(ACCURATE_PROFILE) + assert ACCURATE_PROFILE.repo in text + assert "643 MB" in text + assert text.endswith(TERMINAL_ANSWERS) + + +def test_prompt_text_has_three_parts_like_the_browser_prompt() -> None: + text = mg.model_prompt_text(ACCURATE_PROFILE, answers=TUI_ANSWERS) + assert text.endswith(TUI_ANSWERS) + # what, why + cost, answers: separated by blank lines. + assert len([block for block in text.split("\n\n") if block.strip()]) >= 3 + + +def test_prompt_subject_round_trips() -> None: + subject = mg.prompt_subject(ACCURATE_PROFILE) + assert mg.subject_profile(subject) is ACCURATE_PROFILE + assert mg.subject_profile(mg.prompt_subject(DEFAULT_PROFILE)) is DEFAULT_PROFILE + # A blocked host is never mistaken for a model question. + assert mg.subject_profile("nature.com") is None + assert mg.subject_profile("model:unknown") is None + + +def test_default_prompt_reads_the_terminal(monkeypatch: pytest.MonkeyPatch) -> None: + echoed: list[str] = [] + replies = iter(["maybe", "a"]) + monkeypatch.setattr(mg.typer, "echo", lambda message, err=False: echoed.append(message)) + monkeypatch.setattr(mg.typer, "prompt", lambda *args, **kwargs: next(replies)) + assert mg.ask_model_terminal(ACCURATE_PROFILE) == "always" + assert ACCURATE_PROFILE.repo in echoed[0] + assert any("Please answer one of" in line for line in echoed) + + +# --- download_profile ---------------------------------------------------------------- + + +def test_download_profile_fetches_the_three_files( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import huggingface_hub + + calls: list[tuple[str, str, str | None, str | None]] = [] + + def fake_download( + repo_id: str, filename: str, *, revision: str | None = None, cache_dir: str | None = None + ) -> str: + calls.append((repo_id, filename, revision, cache_dir)) + return str(tmp_path / filename) + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download) + cache = tmp_path / "models" + mg.download_profile(ACCURATE_PROFILE, cache, "arm64") + assert [c[1] for c in calls] == ["config.json", "tokenizer.json", "onnx/model_quantized.onnx"] + assert {c[0] for c in calls} == {ACCURATE_PROFILE.repo} + assert {c[2] for c in calls} == {ACCURATE_PROFILE.revision} + assert {c[3] for c in calls} == {str(cache)} + assert cache.is_dir() From c210978369b1d27b8a5f2e2246ff57daed5a1312 Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 13:13:12 +0300 Subject: [PATCH 4/7] feat(nli): resolve the NLI profile per run, with consent and a reported fallback check --accurate or [models] nli = "accurate" selects the large model. It is downloaded once, only with consent; when the download is refused or fails the run uses the default model and says so in the report, as a note, and on a CLI line that survives -q. The models block now records k. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- src/proofpath/cli.py | 19 ++ src/proofpath/model_gate.py | 45 ++- src/proofpath/report.py | 14 + src/proofpath/settings_hints.py | 3 + src/proofpath/verify.py | 223 ++++++++++++-- tests/data/verify-report-golden.json | 1 + tests/test_check_cli.py | 71 +++++ tests/test_model_gate.py | 78 ++++- tests/test_report.py | 26 ++ tests/test_verify.py | 428 ++++++++++++++++++++++++++- 10 files changed, 871 insertions(+), 37 deletions(-) diff --git a/src/proofpath/cli.py b/src/proofpath/cli.py index 81833b8..125c0ec 100644 --- a/src/proofpath/cli.py +++ b/src/proofpath/cli.py @@ -534,6 +534,16 @@ def check( help="Do not search the web for evidence when the text cites no source.", ), ] = False, + accurate: Annotated[ + bool, + typer.Option( + "--accurate", + help=( + "Use the larger accurate NLI model for this run " + "(643 MB, downloaded once with consent)." + ), + ), + ] = False, out_path: Annotated[ Path | None, typer.Option( @@ -586,6 +596,9 @@ def on_event(event: events.Event) -> None: # nothing else; the escalation stage is what ``--judge`` buys. escalate=judge, search=not no_search, + # Which model, not consent to download it: the engine's model gate + # still asks. Without the flag ``models.nli`` in the config decides. + nli="accurate" if accurate else None, ) as engine, _interruptible(cancel), ): @@ -640,6 +653,12 @@ def on_event(event: events.Event) -> None: for item in report_mod.render_diagnostics(report): ui.diagnostic(out, item) ui.blank(out) + fallback = report_mod.nli_fallback_line(report) + if fallback is not None: + # Same rule as the ``judge`` and ``summary`` lines below: the note that says + # so is dropped under ``-q``, and a run that used another model than the + # one asked for must not read like one that did not (product rule 6). + ui.kv(out, "nli", fallback) unanswered = report_mod.judge_unavailable_line(report) if unanswered is not None: # Same rule as the ``summary`` line below: the stage row and the note are diff --git a/src/proofpath/model_gate.py b/src/proofpath/model_gate.py index 98e50de..9a7bcb7 100644 --- a/src/proofpath/model_gate.py +++ b/src/proofpath/model_gate.py @@ -16,6 +16,7 @@ from collections.abc import Callable from pathlib import Path +from typing import Literal import typer @@ -26,6 +27,15 @@ #: The key "always" and "never" are saved under. PERMISSION_KEY = "permissions.install_model" +#: Why ``ensure`` did not allow a profile, as data rather than as the reason's words, +#: so the fallback line can give advice that fits it: +#: +#: * ``no_terminal`` -- the permission is ``ask`` and nobody can be asked (rule 4); +#: * ``refused`` -- the permission is ``deny``, or the user answered "never"; +#: * ``declined`` -- the user answered "no", for this run only; +#: * ``download_failed`` -- consent was given and the download did not finish. +Refusal = Literal["no_terminal", "refused", "declined", "download_failed"] + # What a model question passes as the ``host`` argument of a gate prompt. The prompt # callable is the browser gate's, shared so the TUI answers both through one widget; # a real host never starts with this (a ``host:port`` port is numeric), so a prompt @@ -138,6 +148,7 @@ def __init__( config_path: Path | None = None, download: Callable[[NliProfile], None], installed: Callable[[NliProfile], bool], + on_log: Callable[[str], None] | None = None, ) -> None: self._permission = permission self._interactive = interactive @@ -147,11 +158,26 @@ def __init__( self._download = download self._installed = installed self.log: list[str] = [] + # Called with each ``log`` line as it is written, so a caller can say + # "downloading ... 643 MB" before the download rather than after it. Public + # and reassignable: the verifying stage points it at the run's listener. + self.on_log = on_log # True once the user was shown the question this run. self.asked = False # By profile name: a second ``ensure`` neither asks nor downloads again, and # a failed download is not retried within the run. self._decided: dict[str, Decision] = {} + self._refusals: dict[str, Refusal] = {} + + def _log(self, line: str) -> None: + self.log.append(line) + if self.on_log is not None: + self.on_log(line) + + def refusal(self, profile: NliProfile) -> Refusal | None: + """Why ``ensure`` did not allow ``profile`` this run, or ``None`` if it did (or + has not been asked).""" + return self._refusals.get(profile.name) def _ask(self, profile: NliProfile) -> Answer: if self._prompt is None: @@ -164,10 +190,14 @@ def _persist(self, value: str) -> None: try: set_value(PERMISSION_KEY, value, self._config_path) except (ConfigError, OSError) as exc: - self.log.append(f"could not save permission: {exc}") + self._log(f"could not save permission: {exc}") def _consent(self, profile: NliProfile) -> Decision: decision = resolve_permission(self._permission, interactive=self._interactive) + if decision.outcome == "deny": + # ``ask`` only resolves to ``deny`` when there is no terminal to ask on. + no_terminal = self._permission == "ask" + self._refusals[profile.name] = "no_terminal" if no_terminal else "refused" if decision.outcome != "prompt": return decision self.asked = True @@ -178,8 +208,10 @@ def _consent(self, profile: NliProfile) -> Decision: self._persist("allow") return Decision("allow", "user answered always") if answer == "no": + self._refusals[profile.name] = "declined" return Decision("deny", "user answered no") self._persist("deny") # "never" + self._refusals[profile.name] = "refused" return Decision("deny", "user answered never") def ensure(self, profile: NliProfile) -> Decision: @@ -192,15 +224,15 @@ def ensure(self, profile: NliProfile) -> Decision: def _ensure(self, profile: NliProfile) -> Decision: if self._installed(profile): - self.log.append(f"model {profile.name}: installed") + self._log(f"model {profile.name}: installed") return Decision("allow", "installed") decision = self._consent(profile) if decision.outcome != "allow": - self.log.append(f"model {profile.name}: not downloaded ({decision.reason})") + self._log(f"model {profile.name}: not downloaded ({decision.reason})") return decision - self.log.append( + self._log( f"model {profile.name}: downloading {profile.repo} ({profile.size_mb} MB;" f" {decision.reason})" ) @@ -208,7 +240,8 @@ def _ensure(self, profile: NliProfile) -> Decision: self._download(profile) except Exception as exc: # a failed download falls back, it never crashes failed = Decision("deny", f"model download failed: {_failure_reason(exc)}") - self.log.append(f"model {profile.name}: {failed.reason}") + self._refusals[profile.name] = "download_failed" + self._log(f"model {profile.name}: {failed.reason}") return failed - self.log.append(f"model {profile.name}: downloaded") + self._log(f"model {profile.name}: downloaded") return decision diff --git a/src/proofpath/report.py b/src/proofpath/report.py index 6a2cd67..ee70856 100644 --- a/src/proofpath/report.py +++ b/src/proofpath/report.py @@ -582,6 +582,20 @@ def summary_unavailable(report: Report) -> str | None: return f"{JUDGE} {found.removeprefix(f'{SUMMARY} ')}" +def nli_fallback_line(report: Report) -> str | None: + """The ``nli`` line's value when the requested NLI profile was not the one used. + + ``models["nli_requested"]`` is only there on a fallback. The run's Note says the + same thing, but notes are dropped under ``-q``, and a run that decided with + another model than the one asked for must not read like one that did not + (product rule 6), so the terminal prints this whatever the verbosity. + """ + requested = report.models.get("nli_requested") + if requested is None: + return None + return f"default model chosen; requested {requested}" + + # ``pipeline.py`` writes this reason when a rule, not the model, refuted a claim. It # is parsed back out rather than re-derived, so the caret line and the verdict can # never disagree about which figure was wrong. diff --git a/src/proofpath/settings_hints.py b/src/proofpath/settings_hints.py index 4127fb7..03da4ef 100644 --- a/src/proofpath/settings_hints.py +++ b/src/proofpath/settings_hints.py @@ -16,6 +16,9 @@ #: ``permissions.install_model`` set to ``ask``: a run that wants the accurate NLI #: profile asks before it downloads the model, instead of falling back to the default. MODEL_SETTING = "permissions.install_model ask" +#: ``permissions.install_model`` set to ``allow``: the download goes ahead without a +#: question, which is the only way it can where there is no terminal to ask on (CI). +MODEL_ALLOW_SETTING = "permissions.install_model allow" #: ``permissions.network`` set to ``allow``: every network-bound stage runs. NETWORK_SETTING = "permissions.network allow" #: ``search.provider`` set to ``tavily``: a text that cites nothing is searched with diff --git a/src/proofpath/verify.py b/src/proofpath/verify.py index 0a20f2a..5284401 100644 --- a/src/proofpath/verify.py +++ b/src/proofpath/verify.py @@ -26,6 +26,7 @@ from __future__ import annotations +import platform import re import sqlite3 import threading @@ -48,7 +49,7 @@ from proofpath.browser import Answer, ConsentGate from proofpath.cache import Cache from proofpath.claims import Claims -from proofpath.config import Config +from proofpath.config import Config, NliProfileName from proofpath.document import CitationMarker, Claim, Document, Locator, Reference from proofpath.entailment import OnnxNli, Scorer from proofpath.events import ( @@ -63,11 +64,13 @@ ) from proofpath.fetch import Fetched, Fetcher, Outcome from proofpath.judge import Judge, JudgeCost, JudgeItem, JudgeOpinion +from proofpath.model_gate import ModelGate, Refusal, download_profile from proofpath.models import Label, Passage, Verdict from proofpath.oa import OpenAccess from proofpath.paths import models_dir from proofpath.pipeline import DEFAULT_THRESHOLDS, Thresholds from proofpath.polite import PoliteClient, ProviderError, user_agent +from proofpath.profiles import DEFAULT_PROFILE, PROFILES, NliProfile, profile_installed from proofpath.providers import ( # noqa: F401 - two names re-exported, see below ARXIV_PREFIX, DOI_PREFIX, @@ -119,7 +122,13 @@ from proofpath.search.providers import build_searcher from proofpath.search.queries import plan_queries from proofpath.secrets import CREDENTIALS_MISSING -from proofpath.settings_hints import BROWSER_SETTING, SEARCH_SETTING, SEARXNG_SETTING +from proofpath.settings_hints import ( + BROWSER_SETTING, + MODEL_ALLOW_SETTING, + MODEL_SETTING, + SEARCH_SETTING, + SEARXNG_SETTING, +) # Stage names, as the report and the TUI print them (spec section 13.2). PARSING = "Parsing" @@ -283,11 +292,31 @@ class Engine: gate: ConsentGate embedder: Callable[[], Embedder] scorer: Callable[[], Scorer] - # k=1 and the thresholds beside it are the SciFact dev calibration of 2026-09-12 - # (spec section 14, docs/eval/2026-09-12-tiers.md): k=2 ties on accuracy and k=3 - # is worse, so the cheapest of the tied settings is the one that ships. + # The k and the cuts this run decides with. The defaults are the default profile's + # (SciFact dev, 2026-09-12; docs/eval/2026-09-12-tiers.md: k=2 ties on accuracy and + # k=3 is worse, so the cheapest of the tied settings ships). ``resolve_nli`` replaces + # each one with the resolved profile's when its ``*_from_profile`` switch is on, + # which is how ``Engine.default`` builds an engine unless its caller pinned the + # value; an engine assembled by hand keeps what it was given (see ``resolve_nli``). k: int = 1 thresholds: Thresholds = DEFAULT_THRESHOLDS + k_from_profile: bool = False + thresholds_from_profile: bool = False + # The NLI profile this run asked for (``models.nli`` or ``--accurate``) and the gate + # in front of a profile that has to be downloaded first (product rule 5). The + # default profile never goes through the gate; ``None`` here means a profile that + # needs it cannot be used, and the run falls back to the default and says so. + nli_requested: NliProfileName = "default" + model_gate: ModelGate | None = None + # Set once by ``resolve_nli``: the profile the run actually uses, and why it is not + # the requested one when it is not (empty otherwise). A report states both, since + # silently running another model would make it claim a model it did not run. + nli_profile: NliProfile | None = None + nli_fallback: str = "" + # The gate's ``Refusal`` behind ``nli_fallback``, so the line that reports it can + # give advice that fits (a failed download wants a retry, not a setting). ``None`` + # with a fallback means there was no gate to ask. + nli_refusal: Refusal | None = None device: str = "cpu" # The opt-in second opinion. ``None`` is the default path, and the default path # makes zero LLM calls (spec section 11). Nothing downstream branches on it @@ -339,17 +368,29 @@ def default( browser: bool | None = None, no_cache: bool = False, prompt: Callable[[str, int | None], Answer] | None = None, - k: int = 1, - thresholds: Thresholds = DEFAULT_THRESHOLDS, + k: int | None = None, + thresholds: Thresholds | None = None, judge: Judge | None = None, escalate: bool = True, search: bool = True, + nli: NliProfileName | None = None, ) -> Engine: """The real wiring: the same one ``proofpath fetch`` builds, one layer up. ``interactive`` is passed down rather than sniffed here, so a caller that knows it is in CI (or under a pipe) gets ``ask`` treated as ``deny`` all the way through — the gate never prompts without a terminal (product rule 4). + + ``nli`` names the NLI profile for this run (``--accurate``); ``None`` leaves it + to ``config.models.nli``. Nothing about the profile is looked at here: the + model cache is checked, and the user asked, only when the verifying stage + needs the model (``resolve_nli``), so building an engine stays model-free and + network-free. + + ``k`` and ``thresholds`` pin those values for tests and evaluations. Each one + given wins over the resolved profile's; each one left ``None`` follows the + profile the run resolves to. They are independent, so pinning ``k`` alone + still decides with the profile's cuts. """ closers: list[Callable[[], None]] = [] if judge is not None: @@ -381,6 +422,17 @@ def default( closers.append(resolver_client.close) resolver = Resolver(contact_email=email, client=resolver_client) device_name = device_mod.onnx_device_name() + machine = platform.machine() + # The same ``prompt`` as the browser gate: in the TUI that is its own + # question block, so a model question never falls back to reading stdin. + model_gate = ModelGate( + config.permissions.install_model, + interactive=interactive, + override=None if nli is None else True, + prompt=prompt, + download=lambda profile: download_profile(profile, models_dir(), machine), + installed=lambda profile: profile_installed(profile, models_dir(), machine), + ) except Exception: _close_all(closers) raise @@ -393,12 +445,20 @@ def embedder() -> Embedder: ) def scorer() -> Scorer: - # No ``providers`` and no ``onnx_file``: ``entailment.providers_for`` - # then picks them from the export it actually chose, which is what - # keeps CoreML out of the int8 path (measured 3x slower there). - return OnnxNli(cache_dir=models_dir()) + # The profile the run resolved to; ``decide_all`` resolves it first, so this + # only ever reads the answer. The default profile's file is the one + # ``OnnxNli`` would pick by itself. No ``providers``: + # ``entailment.providers_for`` picks them from the export, which is what + # keeps CoreML out of the int8 and ``quantized`` paths (measured 3x slower). + profile = built.resolve_nli() + return OnnxNli( + cache_dir=models_dir(), + repo_id=profile.repo, + revision=profile.revision, + onnx_file=profile.onnx_file(machine), + ) - return cls( + built = cls( config=config, cache=cache, resolver=resolver, @@ -408,8 +468,12 @@ def scorer() -> Scorer: client=client, embedder=embedder, scorer=scorer, - k=k, - thresholds=thresholds, + k=DEFAULT_PROFILE.k if k is None else k, + thresholds=DEFAULT_PROFILE.thresholds if thresholds is None else thresholds, + k_from_profile=k is None, + thresholds_from_profile=thresholds is None, + nli_requested=config.models.nli if nli is None else nli, + model_gate=model_gate, device=device_name, judge=judge, escalate=escalate, @@ -419,6 +483,39 @@ def scorer() -> Scorer: search=search and config.permissions.web_search == "allow", _closers=closers, ) + return built + + def resolve_nli(self) -> NliProfile: + """The NLI profile this run uses, decided once and then remembered. + + The default profile is used as it is: it never asks and its first download + behaves as it always has. Any other profile goes through ``model_gate``, which + may find it installed, ask, download it, or refuse; a refusal or a failed + download falls back to the default profile and records why in + ``nli_fallback`` (product rule 6). After this, ``k`` and ``thresholds`` are the + resolved profile's unless they were pinned (``*_from_profile`` off), so the + cache key and every decision read the values of the model that actually runs. + """ + if self.nli_profile is not None: + return self.nli_profile + profile = DEFAULT_PROFILE + wanted = PROFILES[self.nli_requested] + if wanted is not DEFAULT_PROFILE: + if self.model_gate is None: + self.nli_fallback = "no consent gate on this engine" + else: + decision = self.model_gate.ensure(wanted) + if decision.outcome == "allow": + profile = wanted + else: + self.nli_fallback = decision.reason + self.nli_refusal = self.model_gate.refusal(wanted) + if self.k_from_profile: + self.k = profile.k + if self.thresholds_from_profile: + self.thresholds = profile.thresholds + self.nli_profile = profile + return profile def get_embedder(self) -> Embedder: """The run's embedder, built at most once and owned by this engine.""" @@ -1277,6 +1374,10 @@ def check() -> None: if jobs: # A run told to stop before it started must not pay for two ONNX sessions. check() + # Before the models load: the profile decides which NLI model is built and + # the k and cuts the key below is made of, and its question (if any) comes + # before the wait for the models, not hidden behind it. + _resolve_nli(engine, emit) emit(Note(LOADING_MODELS, transient=True)) # Built through the engine, not called as factories: the engine keeps # them, so nothing here outlives ``Engine.close()`` holding a session. @@ -1371,7 +1472,7 @@ def check() -> None: api_calls=0 if judge_cost is None else judge_cost.calls - judge_cost.local_calls, judge_cost=judge_cost, elapsed=time.monotonic() - prepared.started, - tier_note=pipeline.tier_note(engine.thresholds), + tier_note=pipeline.tier_note(_tuning(engine)[1]), cancelled=cancelled, search=prepared.search, # The hint (no fallback, but one is worth adding) rides the same one-line @@ -1736,17 +1837,91 @@ def _coverage(prepared: Prepared, engine: Engine) -> Coverage: ) +def _resolve_nli(engine: Engine, emit: Listener) -> None: + """Resolve the run's NLI profile and say what that took, as Notes. + + The gate's log lines go out as Notes the moment they are written, not after + ``ensure`` returns: "downloading ... 643 MB" is worth saying before the long wait, + not after it. The gate's ``on_log`` points at this run's listener for exactly as + long as the resolution takes, then goes back to what it was. A fallback adds one + line that says why and what would change it, so a run that wanted the accurate + model and did not get it never reads like one that did. + """ + gate = engine.model_gate + if gate is None: + engine.resolve_nli() + else: + before = gate.on_log + gate.on_log = lambda line: emit(Note(line)) + try: + engine.resolve_nli() + finally: + gate.on_log = before + if engine.nli_fallback: + emit( + Note(_nli_fallback_note(engine.nli_requested, engine.nli_fallback, engine.nli_refusal)) + ) + + +def _nli_fallback_note(requested: str, reason: str, refusal: Refusal | None) -> str: + """The fallback line, with the advice that fits why the profile was not used. + + Decided by the gate's ``Refusal``, never by the reason's wording: a failed download + is retried, not re-permitted; a run with no terminal cannot be asked, so ``ask`` + would change nothing and only ``allow`` (or a terminal) can; a refused permission + is what the ``ask`` setting changes; a "no" for this run is asked again next time. + """ + head = f"{requested} NLI model not used: {reason} — using the default model." + if refusal == "download_failed": + return f"{head} Retry when online." + if refusal == "no_terminal": + return f"{head} To download it, run in a terminal or set {MODEL_ALLOW_SETTING}" + if refusal == "refused": + # ``ask`` only helps where a terminal can answer; ``allow`` works anywhere. + return ( + f"{head} To allow the download: {MODEL_SETTING} (asks in a terminal) " + f"or {MODEL_ALLOW_SETTING}" + ) + if refusal == "declined": + return f"{head} The next run asks again." + return head + + +def _tuning(engine: Engine) -> tuple[int, Thresholds]: + """The k and cuts a report states for this run. + + After resolution they are the engine's own. A run that never reached the models + resolved nothing and asked nobody, so the values that follow the profile are the + requested profile's static ones: read from ``PROFILES``, with no gate call and no + I/O, rather than the default's, which nothing in the run chose. + """ + if engine.nli_profile is not None: + return engine.k, engine.thresholds + planned = PROFILES[engine.nli_requested] + return ( + planned.k if engine.k_from_profile else engine.k, + planned.thresholds if engine.thresholds_from_profile else engine.thresholds, + ) + + def _models(embedder: Embedder | None, scorer: Scorer | None, engine: Engine) -> dict[str, str]: """What decided this run. ``NO_MODEL`` where none was loaded, never a name.""" - thresholds = engine.thresholds - models = { - "nli": NO_MODEL if scorer is None else scorer.name, - "embedder": NO_MODEL if embedder is None else embedder.name, - "device": engine.device, - "thresholds": ( - f"decide={thresholds.decide:g};high={thresholds.high:g};medium={thresholds.medium:g}" - ), - } + k, thresholds = _tuning(engine) + models = {"nli": NO_MODEL if scorer is None else scorer.name} + if engine.nli_fallback: + # Beside the model that did run, so the two are read together. + models["nli_requested"] = f"{engine.nli_requested} ({engine.nli_fallback})" + models.update( + { + "embedder": NO_MODEL if embedder is None else embedder.name, + "device": engine.device, + "k": str(k), + "thresholds": ( + f"decide={thresholds.decide:g};high={thresholds.high:g};" + f"medium={thresholds.medium:g}" + ), + } + ) if engine.judge is not None: # Only when one was asked. A ``NO_MODEL`` dash here would put a judge row in # every report, and a run that consulted nobody has no judge to name. diff --git a/tests/data/verify-report-golden.json b/tests/data/verify-report-golden.json index e81e18d..e40f964 100644 --- a/tests/data/verify-report-golden.json +++ b/tests/data/verify-report-golden.json @@ -637,6 +637,7 @@ "nli": "keyword", "embedder": "words", "device": "cpu", + "k": "1", "thresholds": "decide=0.45;high=0.99933;medium=0.457948" }, "api_calls": 0, diff --git a/tests/test_check_cli.py b/tests/test_check_cli.py index ebe9d7e..5de07ee 100644 --- a/tests/test_check_cli.py +++ b/tests/test_check_cli.py @@ -28,9 +28,11 @@ from proofpath.events import Cancelled from proofpath.fetch import Fetched, FetchStats, Outcome from proofpath.judge import Completion, Judge, JudgeCost, JudgeUnavailable +from proofpath.model_gate import ModelGate from proofpath.oa import Attempt, Evidence, Location from proofpath.resolve import Candidate, ResolveResult, Retraction, State from proofpath.retrieval import Embedder +from proofpath.settings_hints import MODEL_ALLOW_SETTING from proofpath.verify import Engine from tests.fakes import TableScorer, WordEmbedder @@ -619,6 +621,75 @@ def test_no_cache_reaches_the_engine(monkeypatch: pytest.MonkeyPatch) -> None: assert seen["no_cache"] is True +def test_accurate_reaches_the_engine_as_the_accurate_profile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen = install(monkeypatch) + result = runner.invoke(app, ["check", "-", "--accurate"], input=draft(CLEAN_BODY)) + assert result.exit_code == 0, result.output + assert seen["nli"] == "accurate" + + +def test_without_accurate_the_config_decides_the_profile(monkeypatch: pytest.MonkeyPatch) -> None: + seen = install(monkeypatch) + result = runner.invoke(app, ["check", "-"], input=draft(CLEAN_BODY)) + assert result.exit_code == 0, result.output + assert seen["nli"] is None + + +def test_a_refused_accurate_model_is_said_on_stderr_and_in_the_report( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Piped, so ``ask`` is a denial (rule 4); the run goes on with the default model + # and says so where a human reads it, leaving stdout one JSON document. + install(monkeypatch, refused_accurate_engine()) + result = runner.invoke( + app, ["check", "-", "--accurate", "--format", "json"], input=draft(CLEAN_BODY) + ) + + assert result.exit_code == 0, result.output + assert "accurate NLI model not used" in result.stderr + # No terminal: ``ask`` is already the setting, so the advice is ``allow``. + assert MODEL_ALLOW_SETTING in result.stderr + models = json.loads(result.stdout)["models"] + assert models["nli_requested"].startswith("accurate (permission is set to ask") + + +def refused_accurate_engine() -> Engine: + """A stub engine asked for the accurate profile with no terminal to consent on.""" + built = built_engine() + built.nli_requested = "accurate" + built.model_gate = ModelGate( + "ask", interactive=False, installed=lambda _p: False, download=lambda _p: None + ) + return built + + +def test_a_quiet_run_still_says_the_accurate_model_was_not_used( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # ``-q`` drops the note; the ``nli`` line is a state of the run and stays (rule 6). + install(monkeypatch, refused_accurate_engine()) + result = runner.invoke(app, ["-q", "check", "-", "--accurate"], input=draft(CLEAN_BODY)) + + assert result.exit_code == 0, result.output + assert "NLI model not used" not in result.output + line = next(row for row in result.stdout.splitlines() if row.startswith("nli ")) + assert line.split(maxsplit=1)[1] == ( + "default model chosen; requested accurate" + " (permission is set to ask but there is no interactive terminal)" + ) + + +def test_a_run_that_used_the_model_it_asked_for_prints_no_nli_line( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install(monkeypatch) + result = runner.invoke(app, ["-q", "check", "-"], input=draft(CLEAN_BODY)) + assert result.exit_code == 0, result.output + assert not any(row.startswith("nli ") for row in result.stdout.splitlines()) + + # --- --format json ------------------------------------------------------------------- diff --git a/tests/test_model_gate.py b/tests/test_model_gate.py index db08eeb..d78edb4 100644 --- a/tests/test_model_gate.py +++ b/tests/test_model_gate.py @@ -6,13 +6,14 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path import pytest from proofpath import model_gate as mg from proofpath.browser import TERMINAL_ANSWERS, TUI_ANSWERS, Answer -from proofpath.config import load_config +from proofpath.config import Permission, load_config from proofpath.profiles import ACCURATE_PROFILE, DEFAULT_PROFILE, NliProfile @@ -346,3 +347,78 @@ def fake_download( assert {c[2] for c in calls} == {ACCURATE_PROFILE.revision} assert {c[3] for c in calls} == {str(cache)} assert cache.is_dir() + + +# --- why a profile was refused, and the log as it is written ----------------------- + + +def answering(answer: Answer) -> Callable[[str, int | None], Answer]: + return lambda _subject, _status: answer + + +@pytest.mark.parametrize( + ("permission", "interactive", "answer", "expected"), + [ + ("ask", False, None, "no_terminal"), + ("deny", True, None, "refused"), + ("ask", True, "never", "refused"), + ("ask", True, "no", "declined"), + ], +) +def test_a_refusal_is_recorded_as_its_kind( + permission: Permission, + interactive: bool, + answer: Answer | None, + expected: mg.Refusal, +) -> None: + gate = mg.ModelGate( + permission, + interactive=interactive, + prompt=raising_prompt if answer is None else answering(answer), + download=raising_download, + installed=missing, + ) + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + assert gate.refusal(ACCURATE_PROFILE) == expected + + +def test_a_failed_download_is_recorded_as_its_kind() -> None: + def broken(profile: NliProfile) -> None: + raise OSError("offline") + + gate = mg.ModelGate("allow", interactive=False, download=broken, installed=missing) + assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" + assert gate.refusal(ACCURATE_PROFILE) == "download_failed" + + +@pytest.mark.parametrize("answer", ["once", "always"]) +def test_an_allowed_profile_has_no_refusal(answer: Answer) -> None: + gate = mg.ModelGate( + "ask", + interactive=True, + prompt=answering(answer), + download=Recorder(), + installed=missing, + ) + assert gate.ensure(ACCURATE_PROFILE).outcome == "allow" + assert gate.refusal(ACCURATE_PROFILE) is None + + +def test_on_log_hears_each_line_before_the_download_starts() -> None: + # "downloading ... 643 MB" has to reach the user before the wait, not after it. + order: list[str] = [] + gate = mg.ModelGate( + "allow", + interactive=False, + download=lambda _profile: order.append("download"), + installed=missing, + on_log=lambda line: order.append(line), + ) + gate.ensure(ACCURATE_PROFILE) + + downloading = next(i for i, item in enumerate(order) if "downloading" in item) + assert "643 MB" in order[downloading] + assert downloading < order.index("download") + assert order[-1] == "model accurate: downloaded" + # ``log`` still holds every line, in the same order. + assert gate.log == [item for item in order if item != "download"] diff --git a/tests/test_report.py b/tests/test_report.py index 9d6d398..69c2b62 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -33,6 +33,7 @@ judge_detail, judge_unavailable, judge_unavailable_line, + nli_fallback_line, no_bibliography, reason_label, render_diagnostics, @@ -1063,6 +1064,31 @@ def test_markdown_carries_the_model_preamble() -> None: assert "- thresholds: decide=0.5" in text +def test_markdown_preamble_states_k_and_a_profile_that_was_not_used() -> None: + # A run that wanted the accurate model and fell back says so beside the model that + # did run (product rule 6), and the k it decided at travels with the cuts. + models = { + "nli": "cross-encoder", + "nli_requested": "accurate (user answered no)", + "k": "1", + "thresholds": "decide=0.45", + } + text = render_markdown(dataclasses.replace(report_with(), models=models), written_at=WHEN) + assert "- nli: cross-encoder\n- nli_requested: accurate (user answered no)\n" in text + assert "- k: 1" in text + + +def test_the_nli_line_is_there_only_when_the_requested_profile_was_not_used() -> None: + assert nli_fallback_line(report_with()) is None + fell_back = dataclasses.replace( + report_with(), models={"nli": "x", "nli_requested": "accurate (user answered no)"} + ) + assert ( + nli_fallback_line(fell_back) + == "default model chosen; requested accurate (user answered no)" + ) + + def test_markdown_preamble_states_the_tier_note() -> None: report = dataclasses.replace(report_with(), tier_note="no high tier on this model") assert "- tiers: no high tier on this model" in render_markdown(report, written_at=WHEN) diff --git a/tests/test_verify.py b/tests/test_verify.py index 9100b73..1523566 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -10,23 +10,25 @@ from __future__ import annotations import json +import platform import re import sqlite3 import threading from collections.abc import Callable, Sequence -from dataclasses import replace +from dataclasses import dataclass, replace from pathlib import Path -from typing import NoReturn +from typing import Any, ClassVar, NoReturn import pytest from proofpath import ingest as ingest_mod from proofpath import oa, pipeline, retrieval -from proofpath.browser import ConsentGate +from proofpath import verify as verify_mod +from proofpath.browser import Answer, ConsentGate from proofpath.cache import Cache -from proofpath.config import Config, Permissions +from proofpath.config import Config, ModelsConfig, Permission, Permissions from proofpath.document import Claim, Document, Locator, PageError -from proofpath.entailment import Scorer +from proofpath.entailment import Scorer, pick_onnx_file from proofpath.events import ( Cancelled, Emitted, @@ -38,10 +40,12 @@ ) from proofpath.fetch import Fetched, FetchStats, Outcome from proofpath.judge import Completion, Judge, JudgeCost, JudgeUnavailable +from proofpath.model_gate import ModelGate from proofpath.models import Label, Passage, Verdict from proofpath.oa import ABSTRACT_ONLY, Attempt, Evidence, Location from proofpath.pipeline import Thresholds from proofpath.polite import ProviderError +from proofpath.profiles import ACCURATE_PROFILE, DEFAULT_PROFILE, NliProfile from proofpath.report import ( ABSTRACT_BASIS, JUDGE_DETAIL_PREFIX, @@ -55,7 +59,7 @@ ) from proofpath.resolve import Candidate, ResolveResult, Retraction, State from proofpath.retrieval import Embedder -from proofpath.settings_hints import BROWSER_SETTING +from proofpath.settings_hints import BROWSER_SETTING, MODEL_ALLOW_SETTING, MODEL_SETTING from proofpath.ui import json_text from proofpath.verify import ( CACHE_BY, @@ -927,6 +931,133 @@ def test_engine_default_never_builds_a_model() -> None: assert callable(built.scorer) +def test_engine_default_never_builds_a_model_nor_asks_for_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Building the engine for the accurate profile neither reads the model cache nor + # asks: both wait for the verifying stage, which a document may never reach. + monkeypatch.setattr(verify_mod, "profile_installed", never_called) + monkeypatch.setattr(verify_mod, "OnnxNli", never_called) + with Engine.default(Config(), interactive=False, no_cache=True, nli="accurate") as built: + assert built.nli_requested == "accurate" + assert built.nli_profile is None + assert built.k == 1 + + +def never_called(*_args: object, **_kwargs: object) -> NoReturn: + raise AssertionError("nothing may be loaded or checked while the engine is built") + + +class RecordedNli: + """Stands in for ``OnnxNli``: records how it was built, scores nothing.""" + + built: ClassVar[list[dict[str, object]]] = [] + name = "recorded" + + def __init__(self, **kwargs: object) -> None: + RecordedNli.built.append(kwargs) + + +@pytest.fixture +def recorded_nli(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: + RecordedNli.built = [] + monkeypatch.setattr(verify_mod, "OnnxNli", RecordedNli) + return RecordedNli.built + + +def test_engine_default_builds_the_accurate_scorer_when_it_is_installed( + monkeypatch: pytest.MonkeyPatch, recorded_nli: list[dict[str, object]] +) -> None: + monkeypatch.setattr(verify_mod, "profile_installed", lambda *_args: True) + with Engine.default(Config(), interactive=False, no_cache=True, nli="accurate") as built: + built.get_scorer() + assert built.nli_profile is ACCURATE_PROFILE + assert (built.k, built.thresholds) == (2, ACCURATE_PROFILE.thresholds) + + [kwargs] = recorded_nli + assert kwargs["repo_id"] == ACCURATE_PROFILE.repo + assert kwargs["revision"] == ACCURATE_PROFILE.revision + assert kwargs["onnx_file"] == "onnx/model_quantized.onnx" + + +def test_engine_default_builds_today_s_scorer_for_the_default_profile( + monkeypatch: pytest.MonkeyPatch, recorded_nli: list[dict[str, object]] +) -> None: + # The default profile never asks: not even the cache is checked for it. + monkeypatch.setattr(verify_mod, "profile_installed", never_called) + with Engine.default(Config(), interactive=False, no_cache=True) as built: + built.get_scorer() + assert built.nli_profile is DEFAULT_PROFILE + assert (built.k, built.thresholds) == (1, DEFAULT_PROFILE.thresholds) + + [kwargs] = recorded_nli + assert kwargs["repo_id"] == DEFAULT_PROFILE.repo + assert kwargs["revision"] == DEFAULT_PROFILE.revision + # The file ``OnnxNli`` would have picked by itself, so nothing changes for it. + assert kwargs["onnx_file"] == pick_onnx_file(platform.machine()) + + +def test_engine_default_reads_the_profile_from_the_config_unless_told( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configured = Config(models=ModelsConfig(nli="accurate")) + with Engine.default(configured, interactive=False, no_cache=True) as built: + assert built.nli_requested == "accurate" + with Engine.default(configured, interactive=False, no_cache=True, nli="default") as built: + assert built.nli_requested == "default" + with Engine.default(Config(), interactive=False, no_cache=True, nli="accurate") as built: + assert built.nli_requested == "accurate" + + +def test_engine_default_explicit_k_and_thresholds_outlive_the_profile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(verify_mod, "profile_installed", lambda *_args: True) + pinned = Thresholds(decide=0.5, high=0.99, medium=0.6) + with Engine.default( + Config(), interactive=False, no_cache=True, nli="accurate", k=3, thresholds=pinned + ) as built: + assert built.resolve_nli() is ACCURATE_PROFILE + assert (built.k, built.thresholds) == (3, pinned) + # One pinned, the other still the profile's. + with Engine.default(Config(), interactive=False, no_cache=True, nli="accurate", k=3) as built: + built.resolve_nli() + assert (built.k, built.thresholds) == (3, ACCURATE_PROFILE.thresholds) + + +def test_engine_default_hands_the_model_gate_the_caller_s_prompt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The TUI's prompt, not the terminal one: a TUI run must never read stdin. + asked: list[tuple[str, int | None]] = [] + + def prompt(subject: str, port: int | None) -> Answer: + asked.append((subject, port)) + return "no" + + monkeypatch.setattr(verify_mod, "profile_installed", lambda *_args: False) + monkeypatch.setattr(verify_mod, "download_profile", never_called) + with Engine.default( + Config(), interactive=True, no_cache=True, prompt=prompt, nli="accurate" + ) as built: + assert built.resolve_nli() is DEFAULT_PROFILE + assert built.nli_fallback == "user answered no" + assert asked == [("model:accurate", None)] + + +def test_engine_default_downloads_into_the_models_dir_on_consent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fetched: list[tuple[NliProfile, Path, str]] = [] + monkeypatch.setattr(verify_mod, "models_dir", lambda: tmp_path) + monkeypatch.setattr(verify_mod, "profile_installed", lambda *_args: False) + monkeypatch.setattr(verify_mod, "download_profile", lambda *args: fetched.append(args)) + allowed = Config(permissions=Permissions(install_model="allow")) + with Engine.default(allowed, interactive=False, no_cache=True, nli="accurate") as built: + assert built.resolve_nli() is ACCURATE_PROFILE + assert fetched == [(ACCURATE_PROFILE, tmp_path, platform.machine())] + + def test_prepare_is_typed_as_it_is_declared() -> None: text, doc = drafted("A claim [1].", [REAL]) ready = prepare(text, engine(resolver=StubResolver({"Vaswani": resolved()}))) @@ -1273,6 +1404,7 @@ def test_the_verifying_stage_names_the_device_and_counts_the_verdicts() -> None: "device": "cpu", # The calibrated defaults, spelled out rather than derived: a recalibration # has to come past this line and past docs/eval/2026-09-12-tiers.md together. + "k": "1", "thresholds": "decide=0.45;high=0.99933;medium=0.457948", } @@ -1320,6 +1452,290 @@ def test_no_source_text_loads_no_model_at_all() -> None: assert report.models["device"] == "cpu" +# --- the NLI profile --------------------------------------------------------- +# +# The default profile is today's model and never asks; the accurate one sits behind +# ``ModelGate`` (product rule 5) and falls back, saying so, when it may not be used +# (product rule 6). Every gate here is a real ``ModelGate`` with its two side effects +# injected, so nothing touches the Hugging Face cache or the network. + +PROFILE_TABLE = {SUPPORTING: SUPPORTED_ROW, FIGURE: NEI_ROW, FILLER: NEI_ROW} + + +def never(_profile: NliProfile) -> NoReturn: + raise AssertionError("the model gate may not be consulted on this path") + + +def model_gate( + permission: Permission = "deny", + *, + installed: bool = False, + download: Callable[[NliProfile], None] = never, +) -> ModelGate: + return ModelGate( + permission, + interactive=False, + installed=lambda _profile: installed, + download=download, + ) + + +def accurate(built: Engine, gate: ModelGate) -> Engine: + """``built`` set up the way ``Engine.default(..., nli="accurate")`` sets one up.""" + built.nli_requested = "accurate" + built.model_gate = gate + built.k_from_profile = True + built.thresholds_from_profile = True + return built + + +@dataclass +class Spied: + """What the verifying stage handed ``cache.model_id`` and ``decide_indexed``.""" + + keys: list[tuple[int, Thresholds]] + decided: list[tuple[int, Thresholds]] + + +@pytest.fixture +def spied(monkeypatch: pytest.MonkeyPatch) -> Spied: + seen = Spied(keys=[], decided=[]) + real_key, real_decide = verify_mod.cache_mod.model_id, verify_mod.pipeline.decide_indexed + + def model_id(*, nli: str, embedder: str, k: int, thresholds: Thresholds) -> str: + seen.keys.append((k, thresholds)) + return real_key(nli=nli, embedder=embedder, k=k, thresholds=thresholds) + + def decide_indexed(*args: Any, k: int, thresholds: Thresholds, **kwargs: Any) -> Verdict: + seen.decided.append((k, thresholds)) + return real_decide(*args, k=k, thresholds=thresholds, **kwargs) + + monkeypatch.setattr(verify_mod.cache_mod, "model_id", model_id) + monkeypatch.setattr(verify_mod.pipeline, "decide_indexed", decide_indexed) + return seen + + +def test_an_installed_accurate_profile_decides_at_its_own_k_and_cuts(spied: Spied) -> None: + built = accurate(paper_engine(table=PROFILE_TABLE), model_gate(installed=True)) + events, listener = collected() + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + wanted = (ACCURATE_PROFILE.k, ACCURATE_PROFILE.thresholds) + assert spied.keys == [wanted] + assert spied.decided and set(spied.decided) == {wanted} + assert report.models["k"] == "2" + assert report.models["thresholds"] == "decide=0.25;high=0.999142;medium=0.252721" + assert "nli_requested" not in report.models + assert report.tier_note == pipeline.tier_note(ACCURATE_PROFILE.thresholds) + # The gate's own line is passed on, and nothing says the profile was not used. + assert Note("model accurate: installed") in events + assert not any(isinstance(e, Note) and "not used" in e.text for e in events) + + +def test_the_profile_is_resolved_before_the_models_load() -> None: + built = accurate(paper_engine(table=PROFILE_TABLE), model_gate(installed=True)) + events, listener = collected() + verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + notes = [e for e in events if isinstance(e, Note)] + assert notes.index(Note("model accurate: installed")) < notes.index( + Note(LOADING_MODELS, transient=True) + ) + + +def test_a_denied_accurate_profile_falls_back_and_says_why(spied: Spied) -> None: + # No terminal, permission ``ask``: rule 4 makes it a denial, and the run goes on + # with the default profile rather than stopping or claiming the large model. + built = accurate(paper_engine(table=PROFILE_TABLE), model_gate("ask")) + events, listener = collected() + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + reason = "permission is set to ask but there is no interactive terminal" + default = (DEFAULT_PROFILE.k, DEFAULT_PROFILE.thresholds) + assert spied.keys == [default] + assert set(spied.decided) == {default} + assert report.models["nli_requested"] == f"accurate ({reason})" + assert report.models["k"] == "1" + assert report.models["thresholds"] == "decide=0.45;high=0.99933;medium=0.457948" + assert Note(f"model accurate: not downloaded ({reason})") in events + assert ( + Note( + f"accurate NLI model not used: {reason} — using the default model." + f" To download it, run in a terminal or set {MODEL_ALLOW_SETTING}" + ) + in events + ) + + +def answered(answer: Answer, config_path: Path) -> ModelGate: + """A gate with a terminal whose user answers ``answer``; any save lands in tmp.""" + return ModelGate( + "ask", + interactive=True, + prompt=lambda _subject, _status: answer, + config_path=config_path, + installed=lambda _profile: False, + download=never, + ) + + +def broken_download(_profile: NliProfile) -> None: + raise OSError("offline") + + +REFUSED_HINT = ( + f" To allow the download: {MODEL_SETTING} (asks in a terminal) or {MODEL_ALLOW_SETTING}" +) + + +@pytest.mark.parametrize( + ("build", "advice"), + [ + # Nobody can be asked: ``ask`` changes nothing there, ``allow`` or a terminal does. + ( + lambda _tmp: model_gate("ask"), + f" To download it, run in a terminal or set {MODEL_ALLOW_SETTING}", + ), + # Refused by the setting, or for good: ``ask`` changes it where a terminal can + # answer, ``allow`` anywhere. + ( + lambda _tmp: model_gate("deny"), + REFUSED_HINT, + ), + ( + lambda tmp: answered("never", tmp / "config.toml"), + REFUSED_HINT, + ), + # "No" holds for this run only, so there is nothing to change. + (lambda tmp: answered("no", tmp / "config.toml"), " The next run asks again."), + # Permitted, and the network let it down: a setting would not help. + (lambda _tmp: model_gate("allow", download=broken_download), " Retry when online."), + ], + ids=["no-terminal", "deny", "never", "no", "download-failed"], +) +def test_the_fallback_note_gives_the_advice_that_fits_the_refusal( + build: Callable[[Path], ModelGate], advice: str, tmp_path: Path +) -> None: + built = accurate(paper_engine(table=PROFILE_TABLE), build(tmp_path)) + events, listener = collected() + verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + [line] = [e.text for e in events if isinstance(e, Note) and "NLI model not used" in e.text] + assert line == ( + f"accurate NLI model not used: {built.nli_fallback} — using the default model.{advice}" + ) + + +def test_without_a_gate_the_fallback_note_gives_no_advice() -> None: + built = paper_engine(table=PROFILE_TABLE) + built.nli_requested = "accurate" + events, listener = collected() + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + assert report.models["nli_requested"] == "accurate (no consent gate on this engine)" + assert ( + Note( + "accurate NLI model not used: no consent gate on this engine — using the default model." + ) + in events + ) + + +def test_the_download_is_announced_before_it_starts() -> None: + # The gate's lines go out as they are written: a 643 MB wait is explained first. + events, listener = collected() + + def download(_profile: NliProfile) -> None: + assert any(isinstance(e, Note) and "downloading" in e.text for e in events) + + gate = model_gate("allow", download=download) + built = accurate(paper_engine(table=PROFILE_TABLE), gate) + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + assert report.models["k"] == "2" + notes = [e.text for e in events if isinstance(e, Note)] + assert notes.count("model accurate: downloaded") == 1 # said once, not twice + assert gate.on_log is None # handed back once the profile is resolved + + +def test_a_failed_download_falls_back_with_a_short_reason() -> None: + def broken(_profile: NliProfile) -> None: + raise RuntimeError("the server said something long and private") + + built = accurate(paper_engine(table=PROFILE_TABLE), model_gate("allow", download=broken)) + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built) + + assert report.models["nli_requested"] == "accurate (model download failed: RuntimeError)" + assert report.models["k"] == "1" + assert "private" not in json_text(report) + + +def test_the_default_profile_never_consults_the_model_gate(spied: Spied) -> None: + built = paper_engine(table=PROFILE_TABLE) + built.model_gate = ModelGate("ask", interactive=True, installed=never, download=never) + built.k_from_profile = built.thresholds_from_profile = True + events, listener = collected() + report = verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) + + assert spied.keys == [(1, DEFAULT_PROFILE.thresholds)] + assert "nli_requested" not in report.models + assert built.model_gate.log == [] + assert not any( + isinstance(e, Note) and ("NLI" in e.text or e.text.startswith("model ")) for e in events + ) + + +def test_explicit_k_and_thresholds_win_over_the_profile(spied: Spied) -> None: + # What ``Engine.default(k=..., thresholds=...)`` leaves behind: both pinned. + built = accurate(paper_engine(table=PROFILE_TABLE), model_gate(installed=True)) + built.k_from_profile = built.thresholds_from_profile = False + pinned = Thresholds(decide=0.5, high=0.99, medium=0.6) + built.k, built.thresholds = 3, pinned + verify(draft(ONE_SOURCE_BODY, [REAL]), built) + + assert spied.keys == [(3, pinned)] + + +def test_a_run_with_nothing_to_verify_does_not_ask_for_the_model() -> None: + # A gate that fails the test if it is asked anything: no claim reached a source. + built = accurate(engine(), ModelGate("ask", interactive=True, installed=never, download=never)) + report = verify(draft("A claim [1].", [GHOSTLY]), built) + + assert report.models["nli"] == NO_MODEL + assert built.nli_profile is None + # Nothing was resolved, so the report states the requested profile's own k and + # cuts -- not the default's, which nothing in this run chose. + assert report.models["k"] == "2" + assert report.models["thresholds"] == "decide=0.25;high=0.999142;medium=0.252721" + assert report.tier_note == pipeline.tier_note(ACCURATE_PROFILE.thresholds) + assert "nli_requested" not in report.models + + +def test_nothing_to_verify_still_reports_pinned_values() -> None: + built = accurate(engine(), ModelGate("ask", interactive=True, installed=never, download=never)) + built.k_from_profile = built.thresholds_from_profile = False + built.k = 3 + report = verify(draft("A claim [1].", [GHOSTLY]), built) + assert report.models["k"] == "3" + assert report.models["thresholds"] == "decide=0.45;high=0.99933;medium=0.457948" + + +def test_the_two_profiles_never_share_cached_verdicts(tmp_path: Path) -> None: + text = draft(ONE_SOURCE_BODY, [REAL]) + with Cache(tmp_path / "c.sqlite3") as cache: + default = paper_engine(table=PROFILE_TABLE, cache=cache) + ready = prepare(text, default) + source_row(cache, f"doi:{DOI}", PAPER) + first = decide_all(ready, default) + assert [item.from_cache for item in first.results] == [False, False] + + large = accurate(paper_engine(table=PROFILE_TABLE, cache=cache), model_gate(installed=True)) + second = decide_all(ready, large) + # Same text, same embedder, same scorer name: only k and the cuts differ, and + # that alone keeps the default model's answers out of the accurate run. + assert [item.from_cache for item in second.results] == [False, False] + + # --- the cache --------------------------------------------------------------- From 0bc67a72021fc5aa73dbbfff7f356937473c153a Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 13:23:12 +0300 Subject: [PATCH 5/7] feat(tui): choose the NLI profile in /config and answer the model prompt Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- src/proofpath/tui/app.py | 26 +-- src/proofpath/tui/widgets/__init__.py | 2 + src/proofpath/tui/widgets/config_panel.py | 32 +++- src/proofpath/tui/widgets/prompt.py | 28 ++- tests/test_tui_app.py | 203 ++++++++++++++++++++-- tests/test_tui_rich.py | 9 +- 6 files changed, 265 insertions(+), 35 deletions(-) diff --git a/src/proofpath/tui/app.py b/src/proofpath/tui/app.py index 2011715..7946256 100644 --- a/src/proofpath/tui/app.py +++ b/src/proofpath/tui/app.py @@ -85,6 +85,7 @@ Suggestions, accent_for, panelled, + permission_question, ) from proofpath.verify import Engine @@ -414,21 +415,24 @@ def on_proofpath_app_judge_switched(self, message: JudgeSwitched) -> None: # --- the section 7.1 prompt, inline --------------------------------------------- def _prompt_for(self, owner: int) -> Callable[[str, int | None], Answer]: - """The gate's prompt, for one asker. Runs on that asker's worker thread. - - The thread blocks on a future while the event loop draws the question and - waits for a button or an ``/allow`` line. An app that is already gone answers - ``no``: nothing large is installed without an answer (rule 5). + """The gates' prompt, for one asker. Runs on that asker's worker thread. + + One callable for both consent gates: the browser gate passes the host that + blocked a fetch, the model gate a ``model:`` subject, and + :func:`permission_question` draws whichever block that is. The thread blocks + on a future while the event loop draws the question and waits for a button + or an ``/allow`` line. An app that is already gone answers ``no``: nothing + large is installed or downloaded without an answer (rule 5). """ - def ask(host: str, status: int | None) -> Answer: + def ask(subject: str, status: int | None) -> Answer: if self._quitting: return "no" # asked while the app was already leaving answer: Future[Answer] = Future() try: # Textual types the callback as ``Callable[..., T | Awaitable[T]]``, and # mypy solves ``T`` for a coroutine function as ``Never``: a stub limit. - self.call_from_thread(self._open_prompt, owner, host, status, answer) # type: ignore[arg-type] + self.call_from_thread(self._open_prompt, owner, subject, status, answer) # type: ignore[arg-type] except Exception: return "no" return answer.result() @@ -436,7 +440,7 @@ def ask(host: str, status: int | None) -> Answer: return ask async def _open_prompt( - self, owner: int, host: str, status: int | None, answer: Future[Answer] + self, owner: int, subject: str, status: int | None, answer: Future[Answer] ) -> None: """Mount the question where it happened. Loop thread only.""" if self._quitting: @@ -446,7 +450,8 @@ async def _open_prompt( answer.set_result("no") return self.pending[owner] = answer - widget = PermissionPrompt(owner, host, status, self._out, self._theme) + question = permission_question(subject, status) + widget = PermissionPrompt(owner, question, self._out, self._theme) home: RunBlock | CommandBlock | None = ( self._blocks.get(owner) if owner > 0 else self._command_blocks.get(owner) ) @@ -1038,7 +1043,8 @@ def _allow(self, arg: str) -> None: """ if not self.pending: self._note( - "nothing to allow — the question appears under the stage a site blocks; " + "nothing to allow — the question appears under the stage that needs it " + "(a blocked site or a model download); " "answer it there, or with /allow " + "|".join(commands.ALLOW_ANSWERS) ) return diff --git a/src/proofpath/tui/widgets/__init__.py b/src/proofpath/tui/widgets/__init__.py index 399a42e..1ddf41c 100644 --- a/src/proofpath/tui/widgets/__init__.py +++ b/src/proofpath/tui/widgets/__init__.py @@ -21,6 +21,7 @@ PermissionPrompt, Prompt, PromptFrame, + permission_question, ) from proofpath.tui.widgets.run_block import ( CommandBlock, @@ -58,5 +59,6 @@ "Suggestions", "accent_for", "panelled", + "permission_question", "textual_colour", ] diff --git a/src/proofpath/tui/widgets/config_panel.py b/src/proofpath/tui/widgets/config_panel.py index 6e640fc..5e493ec 100644 --- a/src/proofpath/tui/widgets/config_panel.py +++ b/src/proofpath/tui/widgets/config_panel.py @@ -42,12 +42,14 @@ from proofpath import ui from proofpath.commands import ConfigView, config_set, config_view from proofpath.config import ( + NLI_PROFILE_NAMES, PERMISSION_VALUES, SEARCH_PROVIDERS, ConfigError, Contact, FetchConfig, JudgeConfig, + ModelsConfig, Permissions, SearchConfig, ) @@ -128,7 +130,7 @@ def sections() -> tuple[Section, ...]: """The panel's rows in the config's own order. Built, not constant, because the provider list is the judge module's, and the defaults are the dataclasses' own.""" permissions, fetch, judge, contact = Permissions(), FetchConfig(), JudgeConfig(), Contact() - search = SearchConfig() + search, models = SearchConfig(), ModelsConfig() return ( Section( "permissions", @@ -140,6 +142,13 @@ def sections() -> tuple[Section, ...]: "step 3 of the fetch ladder: a ~280 MB browser engine, spec section 7.1", _default(permissions.install_browser), ), + Row( + "permissions.install_model", + "choice", + PERMISSION_VALUES, + "the accurate NLI model, a 643 MB download; deny = the default profile", + _default(permissions.install_model), + ), Row( "permissions.network", "choice", @@ -231,6 +240,20 @@ def sections() -> tuple[Section, ...]: ), ), ), + Section( + "models", + ( + # The measured gain, so the choice is made knowing what it buys and + # what it does not (spec 2026-10-01, "Why"): scientific sources only. + Row( + "models.nli", + "choice", + NLI_PROFILE_NAMES, + "accurate: 643 MB, on consent; SciFact F1 0.697 vs 0.580, not news", + _default(models.nli), + ), + ), + ), ) @@ -864,7 +887,11 @@ def _draw_legend(self) -> Text: ) def _draw_summary(self) -> Text: - """The one-line record: the settings a run decides by, as the file has them.""" + """The one-line record: the settings a run decides by, as the file has them. + + ``install_model`` is left out to keep it one line at 100 columns: it only matters + once ``nli=accurate`` is chosen, and the open panel shows it. + """ config = self._view.config parts = ( f"install_browser={config.permissions.install_browser}", @@ -872,5 +899,6 @@ def _draw_summary(self) -> Text: f"respect_robots={_default(config.fetch.respect_robots)}", f"judge={config.judge.provider}", f"search={config.search.provider}", + f"nli={config.models.nli}", ) return Text(" config " + " ".join(parts)) diff --git a/src/proofpath/tui/widgets/prompt.py b/src/proofpath/tui/widgets/prompt.py index 3006286..f8de284 100644 --- a/src/proofpath/tui/widgets/prompt.py +++ b/src/proofpath/tui/widgets/prompt.py @@ -17,6 +17,7 @@ from proofpath import ui from proofpath.browser import TUI_ANSWERS, Answer, prompt_text +from proofpath.model_gate import model_prompt_text, subject_profile from proofpath.tui import wordmark from proofpath.tui.history import History from proofpath.tui.theme import Theme @@ -55,8 +56,27 @@ def tinted(accent: str) -> Color: return Color(base.r, base.g, base.b, TINT_ALPHA) +def permission_question(subject: str, status: int | None) -> str: + """The block a gate's question is drawn with, in the TUI's words. + + Both consent gates ask through the same prompt callable: the browser gate with + the host that blocked a fetch, the model gate with a ``model:`` subject + (:func:`proofpath.model_gate.prompt_subject`). Each gets the block its terminal + prompt prints, so the two front ends ask the same question in the same words; + only the last line differs, naming the buttons and ``/allow`` instead of keys. + """ + profile = subject_profile(subject) + if profile is not None: + return model_prompt_text(profile, answers=TUI_ANSWERS) + return prompt_text(subject, status, answers=TUI_ANSWERS) + + class PermissionPrompt(Vertical): - """The section 7.1 question, asked where it happened (spec section 13.1). + """A consent question, asked where it happened (spec section 13.1). + + Either section 7.1's browser question or the accurate NLI profile's download + question: the widget takes the block ready-made (:func:`permission_question`) + and is the same four buttons and the same ``/allow`` answers for both. Never a modal: it is mounted inside the block of the run that hit the wall, under the stage that hit it, and the log keeps scrolling around it. The four buttons are @@ -71,14 +91,14 @@ class PermissionPrompt(Vertical): thing that may act on it, and the install still happens behind it (rule 5). """ - def __init__(self, owner: int, host: str, status: int | None, out: ui.Ui, theme: Theme) -> None: + def __init__(self, owner: int, question: str, out: ui.Ui, theme: Theme) -> None: super().__init__(classes="permission") #: The run this question belongs to, or the negative id of a ``/fetch`` block. self.owner = owner - #: The section 7.1 block, verbatim: the terminal and the TUI ask the same + #: The gate's block, verbatim: the terminal and the TUI ask the same #: question, in the same words, about the same download. Only the last line #: differs: it names the buttons and the ``/allow`` answers, not the keys. - self.question = prompt_text(host, status, answers=TUI_ANSWERS) + self.question = question self._out = out self._theme = theme #: The answer once it was given, so a scrolled-back log still says what was diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index 90bc40b..3699090 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -31,8 +31,10 @@ from proofpath.document import Document, Locator, Reference from proofpath.events import Emitted, Event, Note, Progress, Prompted, StageEnd, StageStart from proofpath.judge import JudgeCost, JudgeError, provider_defaults +from proofpath.model_gate import ModelGate, model_prompt_text, prompt_subject from proofpath.models import Label, Passage, Verdict from proofpath.paths import config_path +from proofpath.profiles import ACCURATE_PROFILE from proofpath.report import Coverage, Finding, Kind, Report, summary_silence from proofpath.resolve import Candidate, ResolveResult from proofpath.resolve import State as ResolveState @@ -1297,6 +1299,89 @@ async def test_allow_with_nothing_pending_says_so() -> None: assert "/allow once|always|no|never" in note +# --- the accurate NLI profile's download question, in the same widget -------------- + + +def ask_model_from_a_worker(app: ProofpathApp, owner: int) -> Future[Any]: + """Ask the app's prompt the way ``ModelGate`` asks it: the model subject, no status. + + The engine hands both gates the same callable, so this is the one a run's model + gate blocks on when the accurate profile is not downloaded yet. + """ + ask = app._prompt_for(owner) + return in_a_worker(lambda: ask(prompt_subject(ACCURATE_PROFILE), None)) + + +async def test_a_model_question_shows_the_models_own_text_and_a_click_settles_it() -> None: + app, schedulers = build_app() + async with app.run_test(size=SIZE) as pilot: + await submit(pilot, "/check draft.md") + answer = ask_model_from_a_worker(app, schedulers[0].runs[0].id) + await until(pilot, lambda: app.query(PermissionPrompt), "the prompt to be mounted") + prompt = app.query_one(PermissionPrompt) + text = prompt.question + # The terminal's block word for word, with the bar's answers as the last line + # (rule 5: the size is on screen before anything is fetched). + assert text == model_prompt_text(ACCURATE_PROFILE, answers=browser.TUI_ANSWERS) + assert ACCURATE_PROFILE.repo in text + assert "643 MB" in text + # Not the browser's question about a blocked site. + assert browser.BROWSER_SIZE not in text + assert "model:" not in text + await click(pilot, "#allow-once") + assert await answered(pilot, answer) == "once" + assert prompt.answer == "once" + assert app.pending == {} + + +async def test_allow_answers_a_model_question_like_a_browser_one() -> None: + app, schedulers = build_app() + async with app.run_test(size=SIZE) as pilot: + await submit(pilot, "/check draft.md") + answer = ask_model_from_a_worker(app, schedulers[0].runs[0].id) + await until(pilot, lambda: app.query(PermissionPrompt), "the prompt") + await submit(pilot, "/allow always") + assert await answered(pilot, answer) == "always" + prompt = app.query_one(PermissionPrompt) + assert prompt.answer == "always" + assert all(button.disabled for button in prompt.query(Button)) + assert app.pending == {} + + +async def test_quitting_answers_an_open_model_question_with_no() -> None: + """Rule 5: an app on its way out installs nothing; "no" downloads nothing.""" + app, schedulers = build_app() + async with app.run_test(size=SIZE) as pilot: + await submit(pilot, "/check draft.md") + answer = ask_model_from_a_worker(app, schedulers[0].runs[0].id) + await until(pilot, lambda: app.query(PermissionPrompt), "the prompt") + await submit(pilot, "/quit") + await pilot.pause() + assert answer.result(timeout=15) == "no" + + +async def test_the_tui_model_question_drives_the_real_gate_to_a_download() -> None: + """The gate's own prompt call reaches the widget, and its answer reaches the gate.""" + downloaded: list[str] = [] + app, schedulers = build_app() + async with app.run_test(size=SIZE) as pilot: + await submit(pilot, "/check draft.md") + gate = ModelGate( + "ask", + interactive=True, + prompt=app._prompt_for(schedulers[0].runs[0].id), + download=lambda profile: downloaded.append(profile.name), + installed=lambda profile: False, + ) + decision = in_a_worker(lambda: gate.ensure(ACCURATE_PROFILE)) + await until(pilot, lambda: app.query(PermissionPrompt), "the prompt") + assert ACCURATE_PROFILE.repo in app.query_one(PermissionPrompt).question + await submit(pilot, "/allow once") + result = await answered(pilot, decision) + assert result.outcome == "allow" and result.reason == "user answered once" + assert downloaded == ["accurate"] + + def test_the_footer_hint_for_skipped_sources_names_the_setting() -> None: """The docked caveat line says which setting lets the browser run next time.""" from proofpath.report import BROWSER_SKIPPED_REASON, render_footer @@ -3005,6 +3090,19 @@ async def written(pilot: Any, key: str, value: str) -> None: ) +async def select_row(pilot: Any, panel: ConfigPanel, key: str) -> None: + """Move the selection to ``key``'s row with the arrow keys, the way a user does. + + By the row's key rather than a count of presses, so a row added to the panel + does not silently send every later test to the row above the one it meant. + """ + index = next(i for i, row in enumerate(panel.rows) if row.key == key) + steps = index - panel.selected + if steps: + await pilot.press(*["down" if steps > 0 else "up"] * abs(steps)) + assert panel.selected == index + + @pytest.mark.usefixtures("no_keys") async def test_config_opens_the_settings_panel_and_reads_the_file() -> None: app, _ = build_app() @@ -3020,7 +3118,7 @@ async def test_config_opens_the_settings_panel_and_reads_the_file() -> None: assert "(not written yet, showing defaults)" in lines[0] assert lines[1].startswith(" key GROQ_API_KEY") assert "not set" in lines[1] and "GROQ_API_KEY=" in lines[1] - for heading in ("permissions", "fetch", "judge", "contact"): + for heading in ("permissions", "fetch", "judge", "contact", "models"): assert f" {heading}" in lines @@ -3054,6 +3152,7 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside panel = await open_panel(pilot) assert [row.key for row in panel.rows] == [ "permissions.install_browser", + "permissions.install_model", "permissions.network", "permissions.web_search", "fetch.respect_robots", @@ -3067,6 +3166,7 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside "search.api_key_env", "search.api_key", "contact.email", + "models.nli", ] assert [row.kind for row in panel.rows] == [ "choice", @@ -3074,6 +3174,7 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside "choice", "choice", "choice", + "choice", "text", "text", "text", @@ -3083,14 +3184,15 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside "text", "secret", "text", + "choice", ] assert panel.selected == 0 assert _panel_row(panel, "install_browser").startswith("> install_browser") - await pilot.press(*["down"] * 13) - assert panel.selected == 13 - assert _panel_row(panel, "email").startswith("> email") + await pilot.press(*["down"] * 15) + assert panel.selected == 15 + assert _panel_row(panel, "nli").startswith("> nli") assert _panel_row(panel, "install_browser").startswith(" install_browser") - await pilot.press(*["up"] * 14) + await pilot.press(*["up"] * 16) assert panel.selected == 0 # The file was only read: moving through the rows writes nothing. assert not config_path().exists() @@ -3463,14 +3565,72 @@ async def test_a_choice_row_writes_at_once_and_wraps() -> None: assert str(config_path()) in _notes(app) +@pytest.mark.usefixtures("no_keys") +async def test_the_model_rows_show_their_choices_and_meanings() -> None: + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + rows = {row.key: row for row in panel.rows} + lines = _shown(panel) + install = rows["permissions.install_model"] + assert install.values == ("ask", "allow", "deny") + assert install.default == "ask" + nli = rows["models.nli"] + assert nli.values == ("default", "accurate") + assert nli.default == "default" + assert " install_model [ask] allow deny" in lines + assert " nli [default] accurate" in lines + # Each meaning sits on the line under its row: the download it costs, and what + # the larger model buys (and does not). + install_meaning = lines[lines.index(" install_model [ask] allow deny") + 1] + assert "643 MB" in install_meaning + nli_meaning = lines[lines.index(" nli [default] accurate") + 1] + for part in ("643 MB", "consent", "SciFact", "0.697", "0.580", "news"): + assert part in nli_meaning, part + # Both fit inside the panel's frame on an 80-column terminal. + assert len(install_meaning) <= SIZE[0] - 6 and len(nli_meaning) <= SIZE[0] - 6 + + +@pytest.mark.usefixtures("no_keys") +async def test_the_model_rows_write_through_the_panel() -> None: + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + await select_row(pilot, panel, "permissions.install_model") + await pilot.press("right") + await written(pilot, "permissions.install_model", "allow") + assert load_config().permissions.install_model == "allow" + assert app._config.permissions.install_model == "allow" + assert _panel_row(panel, "install_model") == "> install_model ask [allow] deny" + await pilot.press("backspace") + await written(pilot, "permissions.install_model", "ask") + assert load_config().permissions.install_model == "ask" + + await select_row(pilot, panel, "models.nli") + await pilot.press("right") + await written(pilot, "models.nli", "accurate") + assert load_config().models.nli == "accurate" + assert app._config.models.nli == "accurate" + assert panel.value("models.nli") == "accurate" + assert _panel_row(panel, "nli") == "> nli default [accurate]" + await pilot.press("right") # wraps back to the first choice + await written(pilot, "models.nli", "default") + assert load_config().models.nli == "default" + await pilot.press("left", "escape") + await pilot.pause() + assert panel.collapsed + assert _panel_text(panel).endswith(" nli=accurate") + + @pytest.mark.usefixtures("no_keys") async def test_the_network_row_drives_the_banner() -> None: app, _ = build_app() async with app.run_test(size=SIZE) as pilot: row = wordmark.rows(80, version=__version__, context=run_context(Config()), hint=HINT) assert " online " in _banner_lines(app)[row.version] - await open_panel(pilot) - await pilot.press("down", "right") # network: allow -> deny + panel = await open_panel(pilot) + await select_row(pilot, panel, "permissions.network") + await pilot.press("right") # network: allow -> deny await written(pilot, "permissions.network", "deny") assert app._config.permissions.network == "deny" await until( @@ -3493,7 +3653,8 @@ async def test_a_bool_row_round_trips_through_the_file() -> None: app, _ = build_app() async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - await pilot.press("down", "down", "down", "right") + await select_row(pilot, panel, "fetch.respect_robots") + await pilot.press("right") await written(pilot, "fetch.respect_robots", "false") assert load_config().fetch.respect_robots is False assert panel.value("fetch.respect_robots") == "false" @@ -3508,8 +3669,9 @@ async def test_a_provider_switch_writes_four_and_moves_the_key_line() -> None: app, _ = build_app() async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - assert panel.rows[4].values == ("gemini", "groq", "ollama") - await pilot.press("down", "down", "down", "down", "left") # groq -> gemini + await select_row(pilot, panel, "judge.provider") + assert panel.rows[panel.selected].values == ("gemini", "groq", "ollama") + await pilot.press("left") # groq -> gemini await written(pilot, "judge.api_key_env", "GEMINI_API_KEY") gemini = provider_defaults("gemini") assert load_config().judge == gemini @@ -3537,7 +3699,7 @@ async def test_a_provider_outside_the_known_choices_is_drawn_honestly() -> None: panel = await open_panel(pilot) row = _panel_row(panel, "provider") assert row == " provider gemini groq ollama [openrouter]" - await pilot.press("down", "down", "down", "down") # install_browser -> ... -> provider + await select_row(pilot, panel, "judge.provider") await pilot.press("right") # cycles from the first choice, not from "nowhere" await written(pilot, "judge.provider", "gemini") @@ -3547,7 +3709,8 @@ async def test_a_provider_without_a_key_variable_says_none_is_needed() -> None: app, _ = build_app() async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - await pilot.press("down", "down", "down", "down", "right") # groq -> ollama + await select_row(pilot, panel, "judge.provider") + await pilot.press("right") # groq -> ollama await written(pilot, "judge.provider", "ollama") text = _panel_text(panel) assert " key not needed" in text @@ -3562,7 +3725,8 @@ async def test_a_text_row_is_edited_in_place() -> None: default = JudgeConfig().model async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - await pilot.press("down", "down", "down", "down", "down", "enter") + await select_row(pilot, panel, "judge.model") + await pilot.press("enter") await pilot.pause() assert panel.editing edit = panel.query_one(Input) @@ -3596,7 +3760,8 @@ async def test_an_empty_text_row_shows_unset_and_its_note() -> None: assert _panel_row(panel, "email") == ( " email (unset) , optional, for the Crossref / OpenAlex polite pools" ) - await pilot.press(*["down"] * 13, "enter") + await select_row(pilot, panel, "contact.email") + await pilot.press("enter") await pilot.pause() await until(pilot, lambda: app.focused is panel.query_one(Input), "the edit") assert panel.query_one(Input).value == "" @@ -3617,7 +3782,8 @@ async def test_backspace_resets_a_row_to_its_default() -> None: await written(pilot, "permissions.install_browser", "ask") assert load_config().permissions.install_browser == "ask" assert panel.value("permissions.install_browser") == "ask" - await pilot.press("down", "down", "down", "down", "left") + await select_row(pilot, panel, "judge.provider") + await pilot.press("left") await written(pilot, "judge.provider", "gemini") before = _notes(app).count("judge.") await pilot.press("backspace") @@ -3680,14 +3846,15 @@ async def test_escape_collapses_the_panel_to_one_line_and_focuses_the_bar() -> N app, _ = build_app() async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - await pilot.press("down", "right", "right") # network: allow -> deny -> ask + await select_row(pilot, panel, "permissions.network") + await pilot.press("right", "right") # network: allow -> deny -> ask await written(pilot, "permissions.network", "ask") await pilot.press("escape") await pilot.pause() assert panel.collapsed assert _panel_text(panel) == ( - " config install_browser=ask network=ask respect_robots=true" - " judge=groq search=off" + " config install_browser=ask network=ask" + " respect_robots=true judge=groq search=off nli=default" ) await until(pilot, lambda: app.focused is app.query_one(Prompt), "the bar to take focus") # A collapsed panel is a record: the keys do nothing to it any more. diff --git a/tests/test_tui_rich.py b/tests/test_tui_rich.py index 4333c70..182b404 100644 --- a/tests/test_tui_rich.py +++ b/tests/test_tui_rich.py @@ -963,8 +963,15 @@ async def test_the_config_panel_marks_the_row_badges_the_value_and_wears_the_acc assert spans["allow"] == RICH.tone("muted") assert spans[marker] == ACCENT assert plain[-1] == " ↑↓ row ←→ change enter edit text backspace default esc close" + # The NLI profile row is a choice row like the rest: the default profile badged. + nli = next(line for line in lines if line.plain.startswith(" nli ")) + assert nli.plain == " nli default accurate" + nli_spans = {nli.plain[span.start : span.end]: str(span.style) for span in nli.spans} + assert nli_spans[" default "] == f"{BADGE_INK} on {ACCENT}" + assert nli_spans["accurate"] == RICH.tone("muted") # The marker moves with the selection and nothing else about the rows changes. - assert next(line for line in moved if "network" in line).startswith(f"{marker} network") + selected = next(line for line in moved if "install_model" in line) + assert selected.startswith(f"{marker} install_model") assert next(line for line in moved if "install_browser" in line).startswith(" install") # Single-cell glyphs throughout, and every row but the two that carry a path (the # machine's, as long as it is) fits inside the panel at 100 columns. From bde04fe4238d20183f9e2f00a2f74ba40504410d Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 13:23:12 +0300 Subject: [PATCH 6/7] docs: document the opt-in accurate model, its consent and its limits Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- CHANGELOG.md | 33 +++++++++++ README.md | 58 ++++++++++++++++++- docs/superpowers/OPEN-ITEMS.md | 3 +- .../specs/2026-09-10-proofpath-design.md | 31 +++++++++- site/src/data/measured.ts | 11 ++++ 5 files changed, 133 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bef957..7032265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,39 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +### Added +- **An opt-in accurate NLI model.** The bake-off's winner, + `MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli`, runs at k=2 with cuts + calibrated on SciFact train. Its SciFact dev macro-F1 is 0.697, against 0.580 for + the default model. It is not better on news: on AVeriTeC claims with a readable + source it scores 0.475, against 0.537 for the default model at k=2. It is + recommended for scientific sources, not for news. It is 643 MB, against 244 MB, + and the default install does not change. +- **`proofpath check --accurate`** uses the accurate model for one run. + **`[models] nli = "accurate"`** uses it for every run; set it with + `proofpath config set models.nli accurate` or in the `/config` panel. +- **`permissions.install_model`** guards the 643 MB download. It is `ask` by + default: the run asks once, naming the model and its size, with the browser + question's answers (once, always, no, never), and "always" and "never" are saved. + With no terminal, `ask` is treated as `deny` and the denial is reported. A denied + or failed download falls back to the default model and says so in three places: + a note with the reason and what would change it (dropped by `-q`, and on stderr + under `--format json` or `sarif`); in text mode, an + `nli default model chosen; requested accurate ()` line that `-q` keeps; + and `nli_requested: accurate ()` in the report's `models` block. That + block now also names the `k` that ran. + +### Changed +- **A note when a run has no real low tier.** A run whose `medium` cut sits within + 0.01 of `decide` now says "this model has no real low tier on SciFact, so an + asserted verdict is almost always medium or better". Default runs say it too: the + default's `medium` (0.457948) sits 0.008 above `decide` (0.45), which the + 2026-09-12 tier sweep already measured. When the no-high-tier note also applies, + the two are joined with "; ". +- **The no-high-tier note** now reads "this model earns no high tier on SciFact dev, + so medium is the strongest confidence shown". +- **CI runs strict mypy** over the whole of `src/`. + ### Measured - **An NLI bake-off** (`scripts/eval_nli_bakeoff.py`, report in `docs/eval/2026-10-01-nli-bakeoff.md`). It compares three NLI models, three diff --git a/README.md b/README.md index 80d508f..c51d292 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,13 @@ was near-certain here", not as a guarantee of 85 % precision. And `medium` lands almost exactly on `decide`, so `low` is practically empty among asserted verdicts — the display is effectively **two tiers**: near-certain, and asserted at all. +The report says this itself. A run whose `medium` cut sits within 0.01 of `decide` — +the default model's does, and so does the [accurate model's](#accurate-model-opt-in) — +prints `this model has no real low tier on SciFact, so an asserted verdict is almost +always medium or better`. A calibration that leaves `high` out of reach prints `this +model earns no high tier on SciFact dev, so medium is the strongest confidence shown`. +When both hold, both are printed. + ## Citations it reads Numeric markers — `[12]`, `[12,15]`, `[12-15]`, Nature-style superscripts — and, since @@ -241,13 +248,61 @@ Apple Silicon Mac, models already downloaded: | 1-page markdown draft, 7 references | 79.3 s (v0.1 cold) | **1.35 s** ([v0.2](docs/eval/2026-09-15-v0.2-live.md)) — 15.8 s in v0.1 | | 19-page arXiv PDF, 68 references | 12 m 57 s | 3 m 42 s ([v0.1](docs/eval/2026-09-12-v0.1-live.md)) | -The **first ever** run also downloads about 250 MB of ONNX models. Since v0.2 a +The **first ever** run also downloads about 250 MB of ONNX models. The opt-in +[accurate model](#accurate-model-opt-in) is a further one-time 643 MB download, asked +before it starts. Since v0.2 a cached re-run asks the network for nothing: reference resolution and the retraction check are cached (resolutions 30 days; a retraction hit 30 days, a miss 7 days), the fetched text for 7 days, and chunks and verdicts for as long as the text is unchanged. The models are still loaded, and any source whose text has expired or was never read is fetched again — the PDF re-run above (v0.1) still spent 22 s fetching and re-scored 3 of 102 claims. +## Accurate model (opt-in) + +The default entailment model is `cross-encoder/nli-deberta-v3-base` (244 MB). The +**accurate** profile swaps it for `MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli` +(643 MB), reads the top **two** passages per claim instead of one, and uses cuts +calibrated for that pairing on SciFact train. The default install does not change. + +```bash +proofpath check paper.pdf --accurate # this run only +proofpath config set models.nli accurate # every run; or the models.nli row in /config +``` + +**What it buys.** On SciFact dev it scores **0.697 macro-F1, against 0.580** for the +default model calibrated and scored the same way ([the bake-off](docs/eval/2026-10-01-nli-bakeoff.md)). +Read 0.697 as optimistic: the same dev split chose this model out of 27 combinations and +reports its score. With the cuts it ships with, its tiers on dev are `high` 15 of 15 +right (too few to judge), `medium` 114 of 151 (0.755) and `low` empty +([design](docs/superpowers/specs/2026-10-01-accurate-nli-design.md)), so its reports +carry the low-tier note above. + +**What it does not buy.** It is **not better on news and fact-check pages**. On +AVeriTeC claims with a readable source it scores 0.475, against 0.537 for the default +model at k=2 (the shipped default, at k=1, scores 0.438), and all of them are below the +0.708 majority baseline. Use it for scientific sources, not for news. + +**What it costs.** A one-time 643 MB download, and 54 ms per claim–passage pair against +19 ms (batched, in the eval harness) — with twice as many pairs per claim. + +**Consent.** The download is never silent. When the model is not cached yet, the run +asks once, naming the model and its size, with the browser question's answers: once, +always, no, never. "Always" and "never" are saved to `permissions.install_model` +(`ask` by default; `allow` and `deny` skip the question). Piped or in CI there is no +prompt: `ask` is treated as `deny` and reported. When the download is denied or fails, +the run uses the default model and says so, so a report never names a model it did +not run: + +- a note, `accurate NLI model not used: — using the default model.`, with what + would change it. It is printed with the run's other notes, so `-q` drops it; under + `--format json` or `sarif` it goes to stderr; +- in text mode, a line that `-q` does not drop: + `nli default model chosen; requested accurate ()`; +- `nli_requested: accurate ()` in the report's `models` block, which also names + the `k` that ran. + +Once the files are cached, nothing is asked. + ## Looking inside the cache Everything proofpath fetches, embeds and decides lands in one plain SQLite file: @@ -408,6 +463,7 @@ switch to, the run says so and tells you how to add one, on the same channels. | What | Set | Result | |---|---|---| | Retrieval + entailment | SciFact dev, 340 pairs | 0.609 accuracy, 0.597 macro-F1, against a 0.406 trivial baseline ([details](docs/eval/2026-09-12-scifact-dev.md)) | +| Retrieval + entailment, [accurate model](#accurate-model-opt-in) (opt-in) | SciFact dev, 340 pairs | 0.697 accuracy, 0.697 macro-F1, against 0.591 / 0.580 for the default model in the same 2026-10-01 run. Cuts were fitted on SciFact train there, so the default's figures differ from the row above. Dev both chose this model and scores it, so 0.697 is optimistic; on AVeriTeC it is no better than the default model at k=2 (0.475 against 0.537) ([details](docs/eval/2026-10-01-nli-bakeoff.md)) | | Reference resolution | hand-built ghost set, 274 references | 0.0 % false-ghost, 99.1 % ghost recall ([details](docs/eval/2026-09-12-ghosts.md)) | | Source access | 50 DOIs | 72 % full text, 18 % abstract only, 10 % nothing ([details](docs/eval/2026-09-11-coverage.md)) — a real biomedical paper in the live runs reached 33 % full text | | Citation pairing, numeric | 61 hand-built passages | 0.99 ([details](docs/eval/2026-09-11-pairing.md)) | diff --git a/docs/superpowers/OPEN-ITEMS.md b/docs/superpowers/OPEN-ITEMS.md index 91955b5..f2b91f5 100644 --- a/docs/superpowers/OPEN-ITEMS.md +++ b/docs/superpowers/OPEN-ITEMS.md @@ -608,4 +608,5 @@ five packages. The user's decisions: | 20.8 | Bibliography parsing that was measured but never shipped | two-column rejoin (9.10/12.1), multi-block headless fallback (11.11), more headings (16.5), `[1]-[3]` ranges (9.4) | | 20.9 | The judge's quote is never checked | `judge._collect` does not verify that the quoted span occurs in the passage. Checking it is a cheap way to enforce product rule 1 | | 20.10 | Eval honesty | add a `--judge` flag to `scripts/eval_averitec.py`; do one run with the browser allowed; show the 0.404 search score in the README "Measured" table | -| 20.11 | NLI bake-off result | **Winner: `large-fever` (MoritzLaurer DeBERTa-v3-large mnli-fever-anli-ling-wanli) × k=2 × `max`.** SciFact dev macro-F1 0.697, against 0.580 for the `base` × k=1 × `max` baseline and 0.608 for the best `base` row. Accuracy 0.697. AVeriTeC 3-way 0.438, or 0.475 on the readable subset, against a 0.708 majority baseline. Cost: 643 MB and 54 ms/pair, against 244 MB and 19 ms/pair. Full report: `docs/eval/2026-10-01-nli-bakeoff.md`. Next: a follow-up spec for an opt-in "accurate" NLI model (consent download, spec §7.1). It must settle three open points. (a) Exclude rule-decided rows from the tier walk: the numeric layer fires 4–6 times on train with 0 correct, so it pins `high` at 1.000000. (b) The winner's `medium` cut sits on `decide`, so the report shows only two tiers. (c) On AVeriTeC the winner is below `base` × k=2 (0.475 against 0.537), and `Supported` is still 2/19 | +| 20.11 | NLI bake-off result | **Winner: `large-fever` (MoritzLaurer DeBERTa-v3-large mnli-fever-anli-ling-wanli) × k=2 × `max`.** SciFact dev macro-F1 0.697, against 0.580 for the `base` × k=1 × `max` baseline and 0.608 for the best `base` row. Accuracy 0.697. AVeriTeC 3-way 0.438, or 0.475 on the readable subset, against a 0.708 majority baseline. Cost: 643 MB and 54 ms/pair, against 244 MB and 19 ms/pair. Full report: `docs/eval/2026-10-01-nli-bakeoff.md`. Next: (a) and (b) are settled by the accurate-NLI design (`docs/superpowers/specs/2026-10-01-accurate-nli-design.md`), which adds the winner as an opt-in "accurate" profile behind a consent download (spec §7.1); (c) stays open. (a) **settled** — rule-decided rows are left out of the tier walk (the numeric layer fired 4–6 times on train with 0 correct and pinned `high` at 1.000000); the accurate cuts are `high` 0.999142. (b) **settled** — the `medium` cut still sits on `decide`, and the report now says so itself (`NO_LOW_TIER`, default runs included). (c) **still open** — on AVeriTeC the winner is below `base` × k=2 (0.475 against 0.537), and `Supported` is still 2/19; the README recommends the profile for scientific sources, not for news | +| 20.12 | Accurate-NLI follow-ups | (1) A shared `browser.read_terminal_answer` helper: `model_gate.ask_model_terminal` repeats `browser.ask_terminal`'s prompt loop and imports the private `_CHOICE_ANSWERS` to do it. (2) A multilingual NLI model is still deferred (out of scope in the accurate-NLI design); non-English claims still need the judge to translate | diff --git a/docs/superpowers/specs/2026-09-10-proofpath-design.md b/docs/superpowers/specs/2026-09-10-proofpath-design.md index f84f5c6..ec79ef9 100644 --- a/docs/superpowers/specs/2026-09-10-proofpath-design.md +++ b/docs/superpowers/specs/2026-09-10-proofpath-design.md @@ -202,6 +202,7 @@ settings: install_browser = "ask" # step 3: playwright/patchright + chromium, ~280 MB # default confirmed 2026-09-11: "deny" would hide that # a blocked source was recoverable +install_model = "ask" # the opt-in accurate NLI model, 643 MB (added 2026-10-01) network = "allow" # outbound HTTP at all [fetch] @@ -245,6 +246,22 @@ The prompt appears at most once per run. A denied answer is not re-asked, and th report states plainly how many sources were skipped because of it, so a low-coverage run is never mistaken for a clean one. +> *2026-10-01:* `install_model` guards the opt-in accurate NLI model +> (`[models] nli = "accurate"` or `check --accurate`, §12) the same way, with the same +> four answers. It differs in what a refusal costs: no source is skipped, the run +> falls back to the default model and reports it +> (`docs/superpowers/specs/2026-10-01-accurate-nli-design.md`). + +| `install_model` situation | Behaviour | +|---|---| +| files already cached | no prompt, whatever the setting | +| `allow` | download once into the model cache, then use it | +| `deny`, or the answer *never* | default model; a note, the text-mode line `nli default model chosen; requested accurate ()` (kept under `-q`), and `nli_requested: accurate ()` in the report's `models` block | +| `ask` + interactive TTY | prompt naming the model and its size (643 MB) | +| `ask` + **no TTY** | **never prompt** — treat as `deny` and report it | +| download fails | default model; the same note, `nli` line and `nli_requested` entry as a denial, with the failure as the reason | +| `--accurate` flag | chooses the profile for that run; it is **not** consent | + ## 8. Reference resolution — corrected **The naive approach does not work.** Crossref's `query.bibliographic` matcher always @@ -505,7 +522,7 @@ unsuppressed terminal line; `-q` cannot hide it. The SARIF log carries findings | Scrapling | browser engines are heavy | core + `curl_cffi` bundled (2.7 MB); browser engine installed on demand with explicit consent (§7.1) | | **Inference runtime** | **`torch` + `sentence-transformers` is ~800 MB — larger than the browser engine we ask consent for** | **ONNX runtime by default (base install stays small). `torch` moves to an opt-in `[gpu]` extra.** | | Compute device | CUDA on Windows, MPS on Mac | preference order is always CUDA → MPS → CPU; under ONNX the equivalent is CUDA → CoreML → CPU | -| **NLI model** | a ready-made ONNX cross-encoder had to exist for the "no torch" decision to hold | **verified 2026-09-11:** `cross-encoder/nli-deberta-v3-base` @ `6c749ce3425cd33b46d187e45b92bbf96ee12ec7` ships ONNX (`onnx/model.onnx` 739 MB, `model_O4` 388 MB, int8 variants 244 MB) and `tokenizer.json`. The int8 file is chosen by `platform.machine()` (`qint8_arm64` / `quint8_avx2`). `-small` / `-xsmall` ship the same layout as lighter fallbacks. No `optimum` export, no `transformers` | +| **NLI model** | a ready-made ONNX cross-encoder had to exist for the "no torch" decision to hold | **verified 2026-09-11:** `cross-encoder/nli-deberta-v3-base` @ `6c749ce3425cd33b46d187e45b92bbf96ee12ec7` ships ONNX (`onnx/model.onnx` 739 MB, `model_O4` 388 MB, int8 variants 244 MB) and `tokenizer.json`. The int8 file is chosen by `platform.machine()` (`qint8_arm64` / `quint8_avx2`). `-small` / `-xsmall` ship the same layout as lighter fallbacks. No `optimum` export, no `transformers`. **2026-10-01:** an opt-in **accurate** profile, `MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli` @ `b3546ea6b0346eb6f8d5d68b13c7dc6d0376b3d7` (`onnx/model_quantized.onnx`, 643 MB, int8, so CoreML is skipped as for the default) at **k=2**, the NLI bake-off's winner (`docs/eval/2026-10-01-nli-bakeoff.md`). Downloaded only with `install_model` consent (§7.1); the default stays the base model | | Embedding model | must run under ONNX too | `fastembed` with a pinned model; first candidate `BAAI/bge-small-en-v1.5`, compared against one alternative in the Phase 1 results table | | Evaluation data | `allenai/scifact` on Hugging Face is a script-based loader that `datasets ≥ 4` refuses | the original AI2 tarball (`scifact.s3-us-west-2.amazonaws.com/release/latest/data.tar.gz`, verified live 2026-09-11) is downloaded with `httpx` and its sha256 pinned in the loader. No `datasets` dependency | | Model cache | different cache roots per OS | `platformdirs` | @@ -917,6 +934,18 @@ rule-decided verdict scores exactly 1.0 (§10) and so a cut of 1.0 can be reache `tier_note` asks whether **`high >= 1.0`**, because that is the shipped threshold and a tier only the numeric rule can enter is not the model earning one. +> *2026-10-01:* the accurate profile (§12) has its own cuts — **k=2, `decide` 0.25, +> `medium` 0.252721, `high` 0.999142** — fitted on SciFact **train**, not dev. Its +> calibration leaves rule-decided rows (score exactly 1.0) out of the tier walk: the +> numeric layer fired 4–6 times on train with none correct, and kept in, those rows +> pinned `high` at 1.0. They still count toward `decide`. On SciFact dev these cuts +> give `high` 15 of 15 (too few to judge), `medium` 114 of 151 (0.755), `low` empty. +> `tier_note` now also derives the other end: when `medium - decide < 0.01` (the +> margin the harness warns at) it returns `NO_LOW_TIER`, joined to `NO_HIGH_TIER` with +> `"; "` when both hold. Both profiles meet it — the default's `medium` sits 0.008 +> above `decide` — so the default's reports now admit what the 2026-09-12 sweep found. +> `docs/superpowers/specs/2026-10-01-accurate-nli-design.md`. + ## 15. Error handling and honesty states Every non-verdict is an explicit, reported state. Absence of evidence is never diff --git a/site/src/data/measured.ts b/site/src/data/measured.ts index b0ddc1b..11d22b1 100644 --- a/site/src/data/measured.ts +++ b/site/src/data/measured.ts @@ -40,6 +40,17 @@ export const measures: readonly Measure[] = [ set: 'SciFact dev, 340 pairs', doc: '2026-09-12-scifact-dev.md', }, + { + id: 'accurate', + figure: '0.697', + reading: 'macro-F1 · 0.697 accuracy', + against: + 'against 0.580 macro-F1 for the default model in the same run, with cuts fitted on SciFact train', + what: 'Retrieval and entailment, accurate model (opt-in)', + set: 'SciFact dev, 340 pairs · 643 MB, two passages per claim', + doc: '2026-10-01-nli-bakeoff.md', + note: 'dev both chose this model and scores it, so 0.697 is optimistic · not better on news: AVeriTeC readable 0.475 against 0.537 for the default model at k=2 — for scientific sources, not news', + }, { id: 'nei', figure: '135 / 340', From 8ee4cbea789aafda1e816dac18c52ad8558ab134 Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Thu, 1 Oct 2026 13:32:16 +0300 Subject: [PATCH 7/7] fix(nli): pin eval scripts to the default model and quiet the installed note The published AVeriTeC number describes the default install, so the eval scripts no longer follow a user's models.nli. A cached accurate model no longer adds a note to every run. The default verdict-cache key is pinned by a test so it cannot drift. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KRNnJrkUuFNdBcR5vXRgTp --- docs/superpowers/OPEN-ITEMS.md | 2 +- .../specs/2026-10-01-accurate-nli-design.md | 2 +- scripts/eval_averitec.py | 12 +++++-- scripts/zero_network_check.py | 5 ++- src/proofpath/model_gate.py | 11 ++---- src/proofpath/verify.py | 5 +-- tests/test_cache.py | 35 +++++++++++++++++++ tests/test_model_gate.py | 13 ------- tests/test_verify.py | 12 +++---- 9 files changed, 63 insertions(+), 34 deletions(-) diff --git a/docs/superpowers/OPEN-ITEMS.md b/docs/superpowers/OPEN-ITEMS.md index f2b91f5..4cb3975 100644 --- a/docs/superpowers/OPEN-ITEMS.md +++ b/docs/superpowers/OPEN-ITEMS.md @@ -609,4 +609,4 @@ five packages. The user's decisions: | 20.9 | The judge's quote is never checked | `judge._collect` does not verify that the quoted span occurs in the passage. Checking it is a cheap way to enforce product rule 1 | | 20.10 | Eval honesty | add a `--judge` flag to `scripts/eval_averitec.py`; do one run with the browser allowed; show the 0.404 search score in the README "Measured" table | | 20.11 | NLI bake-off result | **Winner: `large-fever` (MoritzLaurer DeBERTa-v3-large mnli-fever-anli-ling-wanli) × k=2 × `max`.** SciFact dev macro-F1 0.697, against 0.580 for the `base` × k=1 × `max` baseline and 0.608 for the best `base` row. Accuracy 0.697. AVeriTeC 3-way 0.438, or 0.475 on the readable subset, against a 0.708 majority baseline. Cost: 643 MB and 54 ms/pair, against 244 MB and 19 ms/pair. Full report: `docs/eval/2026-10-01-nli-bakeoff.md`. Next: (a) and (b) are settled by the accurate-NLI design (`docs/superpowers/specs/2026-10-01-accurate-nli-design.md`), which adds the winner as an opt-in "accurate" profile behind a consent download (spec §7.1); (c) stays open. (a) **settled** — rule-decided rows are left out of the tier walk (the numeric layer fired 4–6 times on train with 0 correct and pinned `high` at 1.000000); the accurate cuts are `high` 0.999142. (b) **settled** — the `medium` cut still sits on `decide`, and the report now says so itself (`NO_LOW_TIER`, default runs included). (c) **still open** — on AVeriTeC the winner is below `base` × k=2 (0.475 against 0.537), and `Supported` is still 2/19; the README recommends the profile for scientific sources, not for news | -| 20.12 | Accurate-NLI follow-ups | (1) A shared `browser.read_terminal_answer` helper: `model_gate.ask_model_terminal` repeats `browser.ask_terminal`'s prompt loop and imports the private `_CHOICE_ANSWERS` to do it. (2) A multilingual NLI model is still deferred (out of scope in the accurate-NLI design); non-English claims still need the judge to translate | +| 20.12 | Accurate-NLI follow-ups | (1) A shared `browser.read_terminal_answer` helper: `model_gate.ask_model_terminal` repeats `browser.ask_terminal`'s prompt loop and imports the private `_CHOICE_ANSWERS` to do it. (2) The `-q` `nli` line carries the fallback reason but not the advice. (3) Ctrl-C and `/stop` do not interrupt a running 643 MB download. (4) An "installed" but corrupt accurate model fails the run instead of falling back. (5) A multilingual NLI model is still deferred (out of scope in the accurate-NLI design); non-English claims still need the judge to translate | diff --git a/docs/superpowers/specs/2026-10-01-accurate-nli-design.md b/docs/superpowers/specs/2026-10-01-accurate-nli-design.md index 7bd5fb7..df2bde9 100644 --- a/docs/superpowers/specs/2026-10-01-accurate-nli-design.md +++ b/docs/superpowers/specs/2026-10-01-accurate-nli-design.md @@ -49,7 +49,7 @@ accurate model, so it is recommended for scientific sources, not for news. ## Profiles -An `NliProfile` (in `entailment.py`) has these fields: +An `NliProfile` (in `profiles.py`, not `entailment.py`: that would be an import cycle with `pipeline`) has these fields: - `name`: `"default"` or `"accurate"`. - `repo` and `revision`: the revision is the full SHA. diff --git a/scripts/eval_averitec.py b/scripts/eval_averitec.py index 425caa9..48f0d5f 100644 --- a/scripts/eval_averitec.py +++ b/scripts/eval_averitec.py @@ -37,7 +37,7 @@ from proofpath import claims as claims_mod from proofpath import ingest, pipeline, retrieval -from proofpath.config import SearchConfig, load_config +from proofpath.config import NliProfileName, SearchConfig, load_config from proofpath.eval import averitec from proofpath.fetch import Fetched from proofpath.models import Label, Passage @@ -412,6 +412,9 @@ def _decide_claim( ) +NLI_PROFILE: NliProfileName = "default" + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--limit", type=int, default=100, help="first N dev claims") @@ -456,7 +459,12 @@ def main(argv: list[str] | None = None) -> int: # ``interactive=False`` so an `ask` permission is denied and reported rather than # prompted for (product rule 4); ``--no-browser`` forces the browser step off # outright, otherwise the config's own permission decides. - engine = Engine.default(config, interactive=False, browser=False if args.no_browser else None) + # ``nli="default"`` is pinned: the published numbers describe the default install, + # so the user's ``models.nli`` must not change what this script measures. + engine = Engine.default( + config, interactive=False, browser=False if args.no_browser else None, nli=NLI_PROFILE + ) + print(f"nli {NLI_PROFILE} profile (pinned; ignores models.nli)") searcher: Searcher | None = None if args.search: # ``Engine.default`` already resolved the configured provider onto its own diff --git a/scripts/zero_network_check.py b/scripts/zero_network_check.py index d17a964..0811a80 100644 --- a/scripts/zero_network_check.py +++ b/scripts/zero_network_check.py @@ -64,11 +64,14 @@ def guard(*_args: Any, **_kwargs: Any) -> NoReturn: httpx.Client.send = guard # type: ignore[method-assign] curl_cffi.requests.get = guard # type: ignore[assignment] - with verify.Engine.default(load_config(), interactive=False) as engine: + # Pinned to the default profile: the check describes the default install, not + # whatever ``models.nli`` the user has set. + with verify.Engine.default(load_config(), interactive=False, nli="default") as engine: report = verify.verify(target, engine) full, abstract, unverified = report.coverage.pct() print(f"target {target}") + print(f"nli {engine.nli_requested} profile (pinned)") print(f"exit code {report.exit_code()}") print(f"coverage fulltext {full}% / abstract {abstract}% / unverified {unverified}%") for stage in report.stages: diff --git a/src/proofpath/model_gate.py b/src/proofpath/model_gate.py index 9a7bcb7..4a508d7 100644 --- a/src/proofpath/model_gate.py +++ b/src/proofpath/model_gate.py @@ -133,9 +133,9 @@ class ModelGate: permission is ``ask`` and there is a terminal, and downloads on "allow". Every decision, and the download itself, is written to ``log`` for the report. - ``override`` records that ``--accurate`` chose the profile. It is not consent: - the flag says which model, not that 643 MB may be fetched, so it never changes - the decision. + ``--accurate`` picks the profile but is not consent: the flag says which model, + not that 643 MB may be fetched, so the gate takes no flag and the permission + alone decides. """ def __init__( @@ -143,7 +143,6 @@ def __init__( permission: Permission, *, interactive: bool, - override: bool | None = None, prompt: Callable[[str, int | None], Answer] | None = None, config_path: Path | None = None, download: Callable[[NliProfile], None], @@ -152,7 +151,6 @@ def __init__( ) -> None: self._permission = permission self._interactive = interactive - self.override = override self._prompt = prompt self._config_path = config_path self._download = download @@ -162,8 +160,6 @@ def __init__( # "downloading ... 643 MB" before the download rather than after it. Public # and reassignable: the verifying stage points it at the run's listener. self.on_log = on_log - # True once the user was shown the question this run. - self.asked = False # By profile name: a second ``ensure`` neither asks nor downloads again, and # a failed download is not retried within the run. self._decided: dict[str, Decision] = {} @@ -200,7 +196,6 @@ def _consent(self, profile: NliProfile) -> Decision: self._refusals[profile.name] = "no_terminal" if no_terminal else "refused" if decision.outcome != "prompt": return decision - self.asked = True answer = self._ask(profile) if answer == "once": return Decision("allow", "user answered once") diff --git a/src/proofpath/verify.py b/src/proofpath/verify.py index 5284401..4e49963 100644 --- a/src/proofpath/verify.py +++ b/src/proofpath/verify.py @@ -428,7 +428,6 @@ def default( model_gate = ModelGate( config.permissions.install_model, interactive=interactive, - override=None if nli is None else True, prompt=prompt, download=lambda profile: download_profile(profile, models_dir(), machine), installed=lambda profile: profile_installed(profile, models_dir(), machine), @@ -1852,7 +1851,9 @@ def _resolve_nli(engine: Engine, emit: Listener) -> None: engine.resolve_nli() else: before = gate.on_log - gate.on_log = lambda line: emit(Note(line)) + # "installed" stays in the gate's log only: it is the normal case, not news. + # Only lines about asking, downloading or failing are worth a Note. + gate.on_log = lambda line: None if line.endswith(": installed") else emit(Note(line)) try: engine.resolve_nli() finally: diff --git a/tests/test_cache.py b/tests/test_cache.py index 19706bc..99e95ad 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -832,3 +832,38 @@ def test_migrating_a_v4_file_forgets_every_cached_ghost_and_keeps_the_rest( ) assert db.get_resolution(GHOST_REFERENCE, now=NOW) is None assert db.get_resolution(RAW_REFERENCE, now=NOW) == RESOLVED + + +@pytest.mark.parametrize( + ("machine", "expected"), + [ + ( + "arm64", + "cross-encoder/nli-deberta-v3-base@6c749ce:model_qint8_arm64|BAAI/bge-small-en-v1.5" + "|k=1|decide=0.450000|high=0.999330|medium=0.457948", + ), + ( + "x86_64", + "cross-encoder/nli-deberta-v3-base@6c749ce:model_quint8_avx2|BAAI/bge-small-en-v1.5" + "|k=1|decide=0.450000|high=0.999330|medium=0.457948", + ), + ], +) +def test_default_profile_verdict_cache_key_is_pinned(machine: str, expected: str) -> None: + """The default profile's key must equal what main produced, byte for byte. + + A change here silently invalidates every user's cached verdicts. If this test has + to change on purpose, that is a cache-schema decision, not a test fix. + """ + from pathlib import Path + + from proofpath.entailment import DEFAULT_REPO, DEFAULT_REVISION, pick_onnx_file + from proofpath.pipeline import DEFAULT_THRESHOLDS + from proofpath.verify import EMBEDDING_MODEL + + # Built the way ``OnnxNli.__init__`` builds its ``name``. + nli_name = f"{DEFAULT_REPO}@{DEFAULT_REVISION[:7]}:{Path(pick_onnx_file(machine)).stem}" + key = cache_mod.model_id( + nli=nli_name, embedder=EMBEDDING_MODEL, k=1, thresholds=DEFAULT_THRESHOLDS + ) + assert key == expected diff --git a/tests/test_model_gate.py b/tests/test_model_gate.py index d78edb4..fd92ece 100644 --- a/tests/test_model_gate.py +++ b/tests/test_model_gate.py @@ -104,19 +104,6 @@ def test_ask_without_tty_denies_and_reports() -> None: assert any("no interactive terminal" in line for line in gate.log) -def test_override_does_not_grant_consent() -> None: - """``--accurate`` picks the profile; the 643 MB still needs a yes (rule 5).""" - gate = mg.ModelGate( - "ask", - interactive=False, - override=True, - prompt=raising_prompt, - download=raising_download, - installed=missing, - ) - assert gate.ensure(ACCURATE_PROFILE).outcome == "deny" - - # --- ask with a TTY: each answer ---------------------------------------------------- diff --git a/tests/test_verify.py b/tests/test_verify.py index 1523566..e7b83b8 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -1527,8 +1527,8 @@ def test_an_installed_accurate_profile_decides_at_its_own_k_and_cuts(spied: Spie assert report.models["thresholds"] == "decide=0.25;high=0.999142;medium=0.252721" assert "nli_requested" not in report.models assert report.tier_note == pipeline.tier_note(ACCURATE_PROFILE.thresholds) - # The gate's own line is passed on, and nothing says the profile was not used. - assert Note("model accurate: installed") in events + # "installed" is the normal case: it stays in the gate's log, it is not a Note. + assert not any(isinstance(e, Note) and "installed" in e.text for e in events) assert not any(isinstance(e, Note) and "not used" in e.text for e in events) @@ -1537,10 +1537,10 @@ def test_the_profile_is_resolved_before_the_models_load() -> None: events, listener = collected() verify(draft(ONE_SOURCE_BODY, [REAL]), built, on_event=listener) - notes = [e for e in events if isinstance(e, Note)] - assert notes.index(Note("model accurate: installed")) < notes.index( - Note(LOADING_MODELS, transient=True) - ) + assert Note(LOADING_MODELS, transient=True) in events + assert built.nli_profile is ACCURATE_PROFILE + assert built.model_gate is not None + assert built.model_gate.log == ["model accurate: installed"] def test_a_denied_accurate_profile_falls_back_and_says_why(spied: Spied) -> None: