diff --git a/.github/workflows/reader-package.yml b/.github/workflows/reader-package.yml index bbcf72aa..89ef96d3 100644 --- a/.github/workflows/reader-package.yml +++ b/.github/workflows/reader-package.yml @@ -7,6 +7,7 @@ on: - "scriptorium/**" - "tests/**" - "tools/reader_package_smoke.py" + - "tools/private_compare_smoke.py" - ".github/workflows/reader-package.yml" push: branches: [master] @@ -15,6 +16,7 @@ on: - "scriptorium/**" - "tests/**" - "tools/reader_package_smoke.py" + - "tools/private_compare_smoke.py" - ".github/workflows/reader-package.yml" workflow_dispatch: @@ -51,6 +53,12 @@ jobs: cd "$RUNNER_TEMP" "$RUNNER_TEMP/reader-venv/bin/python" -I "$GITHUB_WORKSPACE/tools/reader_package_smoke.py" "$GITHUB_WORKSPACE" "$RUNNER_TEMP/reader-venv/bin/scriptorium" --help + - name: Exercise private comparison with the installed analyzer + shell: bash + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/reader-venv/bin/python" -I "$GITHUB_WORKSPACE/tools/private_compare_smoke.py" "$GITHUB_WORKSPACE" - name: Verify installed source-free gallery and deterministic output shell: bash run: | diff --git a/README.md b/README.md index 6f8da37e..585147ba 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ **Understand the language of a text without uploading the manuscript.** Scriptorium measures vocabulary, sentence length, dialogue and punctuation, then produces -readable offline reports and reproducible data. This integrated **0.2.0a1 preview** brings -the reader site and the local analyzer into the canonical repository. +readable offline reports and reproducible data. This **0.4.0a1 preview** adds private +comparison of two analyses to the canonical Reader and its real-work gallery. ## Try your own text @@ -26,11 +26,28 @@ No runtime dependencies, account, server or telemetry are required. Installation fetch build tools; see [offline wheel installation](docs/READER_PREVIEW.md). `python -m scriptorium` exposes the same commands from a checkout. +## Compare your own analyses + +```bash +scriptorium analyze second-manuscript.txt --output second-analysis.json +scriptorium compare analysis.json second-analysis.json --output comparison.html +scriptorium compare analysis.json second-analysis.json --format json --output comparison.json +scriptorium compare analysis.json second-analysis.json --format csv --output comparison.csv +``` + +Open `comparison.html` for the same 29 measurements, explicit units, source identities +and **right minus left** differences. Percentage differences use percentage points. +Unknown measurements do not become zero; different methods or dictionaries are labelled +incomparable where applicable. JSON decimal observations and their differences retain +precision. Contradictory counts, means, punctuation rates and same-text results are rejected. +Imported JSON and hashes are declarations, not authenticated proof: rerun the analyzer +on the original bytes to verify them. [Comparison contract and examples](docs/PRIVATE_COMPARISON.md). + ## Open a real literary result -The repository already contains a source-free, historical full-work diagnostic for -**Anna Karenina**. It has **28 expected/actual/delta rows: 23 numeric results and five -unavailable values**. No novel download is needed: +The repository contains a source-free, historical full-work diagnostic for +**Anna Karenina**: **28 expected/actual/delta rows**, with 23 numeric results and five +unavailable values. No novel download is needed: ```bash scriptorium diagnostic benchmarks/fantlab/work270306-wikisource-diagnostic.json \ @@ -38,12 +55,12 @@ scriptorium diagnostic benchmarks/fantlab/work270306-wikisource-diagnostic.json scriptorium site --repo-root . --output build/site ``` -Open `anna-report.html` or `build/site/index.html`. The canonical site retains its four -earlier evidence pages and adds the full-work report with a working homepage action. -**A successful local build is not a live GitHub Pages deployment.** The existing Pages -activation interlock is unchanged; this preview does not enable publishing by itself. -Atomic site replacement currently requires Linux `renameat2`; unsupported platforms -fail without deleting the previous build. Private text analysis is a separate capability. +Open `anna-report.html` or `build/site/index.html`. The canonical site retains the five +earlier evidence/report routes and the Reader 0.3 two-book comparison route. +**A successful local build is not a live GitHub Pages deployment.** The Pages activation +interlock is unchanged; this preview does not enable publishing by itself. Atomic site +replacement currently requires Linux `renameat2`; unsupported platforms fail without +deleting the previous build. Private text analysis and comparison are separate capabilities. ## Compare two already-analyzed books @@ -57,18 +74,21 @@ python -m scriptorium.full_work_gallery --repo-root . --format csv > build/full- ``` Open `build/full-work-gallery.html` for ten exact measurements and seven row-wise charts. -These are Scriptorium-side results, not FantLab parity or author-style scores. The comparison now also has a pinned canonical site route: -`build/site/works/two-full-work-differences/index.html`, with downloadable -`measurements.json` and `measurements.csv`. The offline command remains available. -Building this route does not prove GitHub Pages is deployed. -Shell redirection replaces its destination; use new filenames to keep older exports. [Method and limits](docs/FULL_WORK_GALLERY.md). +These are Scriptorium-side results, not FantLab parity or author-style scores. The pinned +canonical site route is `build/site/works/two-full-work-differences/index.html`, with +`measurements.json` and `measurements.csv` downloads. Navigation and export links have +44-pixel minimum touch targets; visible focus and reduced-motion styles are retained. +Newly approved replacement evidence must declare compatible source units before display; +legacy Grin diagnostics are not assigned invented source-unit fields. +Building this route does not prove GitHub Pages is deployed. Shell redirection replaces +its destination; use new filenames to keep older exports. [Method and limits](docs/FULL_WORK_GALLERY.md). ## What the results mean FantLab-shaped metrics are **inferred compatibility candidates**, not demonstrated parity. Scriptorium-only features are **extensions**. The historical Anna diagnostic has no established identity match to FantLab's input text. **M2 remains 0/5**; none of -M1–M5 is declared complete by this release. The AOT-lineage morphology target and +M1–M5 is declared complete by this preview. The AOT-lineage morphology target and field-by-field parity criteria remain unchanged. Texts shorter than 300,000 characters including spaces are valid inputs with explicit @@ -79,20 +99,22 @@ Hashes can identify known works; share derived reports deliberately. ## Safety and reproducibility -The reader CLI bounds manuscripts to 32 MiB and dictionaries to 8 MiB, rejects special -files and symlinks, and uses a create-only atomic writer. Operational errors do not echo -private paths or source text. CSV neutralizes formula-like **text** while preserving -numeric cells; spreadsheet import/save behavior is not universally guaranteed. +The CLI bounds manuscripts to 32 MiB and dictionaries and comparison reports to 8 MiB +per file, rejects special files and symlinks, and uses a create-only atomic writer. +Operational errors do not echo private paths or source text. CSV neutralizes formula-like +**text** while preserving numeric cells; spreadsheet import/save behavior is not universally guaranteed. ```bash python -m unittest discover -s tests -v python -m pip wheel --no-deps . --wheel-dir dist ``` -The Reader package workflow tests the installed wheel outside the checkout, including -the source-free gallery; the Pages workflow runs the complete canonical suite and a -byte-identical site rebuild. The full Python public API, morphology modules, benchmark -data and corpus records remain in place. No external package-registry release is performed by CI. +The Reader workflow runs the complete canonical suite, builds a full wheel and tests it +outside the checkout: actual manuscript analysis/comparison, HTML/JSON/CSV, historical +report, safe failures and deterministic source-free gallery/site. No analyzer stubs are +used in the private-comparison acceptance. Package, project and CLI versions must agree. +The full Python public API, morphology modules, benchmark data and corpus records remain +in place. No external package-registry release is performed by CI. Site locks are persistent regular files whose kernel locks are released on process death. Do not delete a lock file while a publisher could be using it. Legacy directory locks @@ -102,11 +124,13 @@ from an interrupted older preview fail closed and are not automatically deleted. [Reader installation and commands](docs/READER_PREVIEW.md) · [Local analysis and dictionaries](docs/LOCAL_ANALYSIS.md) · +[Private comparison](docs/PRIVATE_COMPARISON.md) · [Frozen diagnostic methodology](docs/FROZEN_DIAGNOSTIC_REPORT.md) · [Canonical site and publication boundary](docs/FULL_WORK_SITE_REPORT.md) · [Roadmap and repository assessment](project/ROADMAP.md) -The previous detailed README is preserved as [Research and evidence index](RESEARCH_INDEX.md). -It retains the project history, corpus investigations and original reference links. -The remaining unpublished EPUB/batch Workbench and Comparison Desk prototypes are -**not** advertised as released features; they need focused adoption and repository tests. +The detailed [Research and evidence index](RESEARCH_INDEX.md) retains project history, +corpus investigations and original reference links. Unpublished EPUB/batch Workbench +and browser Comparison Desk prototypes are **not** advertised as released features; +they require focused adoption and real repository tests. Code/data licensing and Pages +activation retain their separately tracked gates (#284/#285). diff --git a/docs/PRIVATE_COMPARISON.md b/docs/PRIVATE_COMPARISON.md new file mode 100644 index 00000000..0b50339c --- /dev/null +++ b/docs/PRIVATE_COMPARISON.md @@ -0,0 +1,62 @@ +# Private comparison of two Reader analyses + +The `0.4.0a1` preview adds a complete local `analyze -> compare` path without +uploading manuscripts or report files. It uses the canonical 29-metric registry, +not a second analyzer or a synthetic fixture runtime. + +```bash +scriptorium analyze first.txt --output first.json +scriptorium analyze second.txt --output second.json +scriptorium compare first.json second.json --output comparison.html +scriptorium compare first.json second.json --format json --output comparison.json +scriptorium compare first.json second.json --format csv --output comparison.csv +``` + +Open the HTML locally. All formats preserve metric IDs, source digests, declared +algorithm/dependency identities and measurement units. HTML includes a keyboard +skip link, focusable scroll region, source details and a table alternative. Existing +outputs are never overwritten. Shell redirection is not create-only; prefer `--output`. + +## Interpretation + +Differences are **right minus left**. A percentage difference is in percentage +points, never relative percentage change. Unknown measurements remain unavailable; +different algorithm profiles make all rows incomparable. A different dictionary +makes only the five dictionary-dependent vocabulary metrics incomparable. Dictionary +identity is content-based, not JSON key-order-based. Revision labels are provenance, +not a replacement for algorithm versioning. + +Input JSON decimals are retained as exact decimal observations; subtraction chooses +sufficient precision instead of rounding at the default Decimal context. The analyzer's +existing floating-point measurements are not retrospectively upgraded to exact science. + +Reports and hashes are unauthenticated declarations. Validation catches contradictions, +not every possible forgery. Rerun analysis on the original bytes to verify a report. +Shared normalized digests and method identities cannot assert contradictory measurements; +dictionary changes do not excuse dialogue or punctuation drift. + +## Rejection and privacy boundaries + +Each UTF-8 report is limited to 8 MiB. Duplicate keys, nonfinite/oversized numbers, +unknown/missing fields, malformed digests, incorrect units/evidence statuses, missing +mandatory counts, impossible means, broken vocabulary partitions and inconsistent +punctuation rates are rejected. Symlinks and non-regular input files fail closed. +Create-only output uses the same atomic writer as the analyzer. Operational errors do +not print manuscript text or private paths. CSV protects text cells from formula-like +prefixes while retaining signed numeric differences. Hashes can identify known works; +share reports deliberately. No telemetry or external requests are made. + +Short texts are valid inputs with the existing representativeness limitation. No corpus +entry, source edition identity, FantLab parity or M1-M5 completion is established by +this comparison. **M2 remains 0/5**, and AOT-lineage compatibility remains the target. + +## Reproducible acceptance + +`tests/test_compare.py` and `tests/test_imported_mean_integrity.py` construct their +reports with the real `build_local_analysis()`, not analyzer stubs. The Reader workflow +also builds the full repository wheel, installs it outside the checkout and runs +`tools/private_compare_smoke.py`: real texts, 29 rows, HTML/JSON/CSV, exact deltas, +deterministic output, create-only behavior and negative input rejection. + +The separate browser Comparison Desk prototype is tracked in #280 and is not advertised +as an integrated feature of this CLI. GitHub Pages activation remains separately gated. diff --git a/project/CHANGELOG.d/2026-10-22-reader-private-comparison.md b/project/CHANGELOG.d/2026-10-22-reader-private-comparison.md new file mode 100644 index 00000000..3f1d87c0 --- /dev/null +++ b/project/CHANGELOG.d/2026-10-22-reader-private-comparison.md @@ -0,0 +1,10 @@ +# 2026-10-22 — Reader 0.4 delivery batch + +- Accepted and merged earlier Reader 0.3 PR #278 as `5c03ee428df6cc36ebdeff2443d35ac9f7e72dca`; #277 closed by native GitHub. Exact-head baseline: 775 tests and installed-wheel checks passed. Pages deployment is still interlocked. +- Opened #288 on the reused `scrip-compare-003-local-cli` branch. Adopted #279: 29 canonical metrics, strict input/cross-report consistency, exact decimal deltas, offline HTML/JSON/CSV and private create-only output. +- Replaced legacy fixture-runtime mean tests with real `build_local_analysis()` reports. First full CI: 809 tests passed; an old hardcoded 0.2.0a1 smoke expectation then failed. Fixed it by requiring pyproject, installed metadata and exact CLI version agreement, retaining all installed tests and adding `-I` isolation. All six workflows passed at `dab93b3143ee8f494b2f2761bb21d25bf8153ad7`. +- Adopted #286's repin-aware source-unit guard without a second metric registry. Eight negative/positive tests plus one actual frozen-record check passed locally (43 local tests with the comparator). Full canonical CI must revalidate this successor before acceptance. +- Improved two-work navigation/export touch targets to a minimum of 44 pixels, retaining focus and reduced-motion styles. This is partial #283, not a claim of full browser/accessibility acceptance. +- Updated product-first README and existing compact queue; version is `0.4.0a1`. No external registry publication, new branch or access/administration change. +- Recovered distinct unpublished POS benchmark-join debt as #287 after deduplicating open and closed records. Existing #279-#286 remain their canonical scopes. +- New implementation remains pending later fresh acceptance. Browser HTTP navigation was blocked by the runtime: VISUAL_NOT_VERIFIED, no live-publication claim. Source data, scientific thresholds and M2 0/5 are unchanged. diff --git a/project/STATE_AND_QUEUE.md b/project/STATE_AND_QUEUE.md index 041dcbe1..ed926b4b 100644 --- a/project/STATE_AND_QUEUE.md +++ b/project/STATE_AND_QUEUE.md @@ -1,62 +1,56 @@ # STATE_AND_QUEUE — Scriptorium -STATE_REVISION: 293 +STATE_REVISION: 294 PHASE: M1 — Deterministic FantLab surface -LAST_COMMITTED_RUN_AT: 2026-10-18 -LAST_RESULT: Owner-requested cleanup accepted #268 as 6338bedbeaf94b00374a95423bba5dd82368e593 and #272 as 9412ee54ab1a56c7c81f6ff4d290e39d83f49630; #267/#271 closed completed. #275 owns the new successor Reader repair PR, reusing the existing gallery branch and preserving #274's integration. Syntax and crash-safe publication fixes are authored here, not merged. Native GitHub owns current check/merge/deployment facts. -LAST_VERIFIED_PROGRESS: Complete current renderer blob revalidated and syntax corrected. Publisher baseline reproduced two crash failures; revised publisher plus existing and new bounded subprocess/filesystem regressions passed 11/11 locally. Full successor CI and installed-wheel/site checks require actual exact-head provider evidence. All frozen source/benchmark/publication data remain unchanged; M2 stays 0/5. +LAST_COMMITTED_RUN_AT: 2026-10-22 +LAST_RESULT: Owner-requested interactive delivery accepted and merged Reader 0.3 #278 as 5c03ee428df6cc36ebdeff2443d35ac9f7e72dca; native GitHub closed #277. Reader 0.4 implementation is in #288 on the reused scrip-compare-003-local-cli branch. It is authored work, not accepted or merged. Native records own current heads, checks and deployment facts. +LAST_VERIFIED_PROGRESS: #288 head dab93b3143ee8f494b2f2761bb21d25bf8153ad7 passed all six applicable workflows after fixing the stale hardcoded smoke-test version. Its preceding full suite passed 809 tests. This successor adds nine gallery unit/touch-structure regressions, which passed locally with the actual pinned source-free records; exact-head full CI must pass again. No frozen source/benchmark data or scientific gate changed. -## Current unit +## Current delivery and acceptance ```text -UNIT_ID: SCRIP-QA-READER-REPAIR -ISSUE: #275 (covers the remaining executable #269 and #273) +UNIT_ID: SCRIP-READER-0.4-DELIVERY +ISSUES: #279 private CLI comparison; #286 source-unit guard; partial #283 touch targets STATUS: IMPLEMENTED_PENDING_ACCEPTANCE -PR: Resolve the successor PR linked from #275; native head/check facts are authoritative. -BRANCH: scrip-site-two-full-work-gallery -BASE: 9412ee54ab1a56c7c81f6ff4d290e39d83f49630 -INPUT_HEADS: Reader #274 at 62e6c078cb70ec77775007a1668fa0c3ede4f5fa; - unfinished gallery at c411e29030e7e778f99cd9689441d348f75a7715. -NEXT_ACTION: Fix actual successor CI findings in this same branch, then later fresh acceptance. - Inspect the exact diff, full suite, installed-wheel smoke and canonical rebuild. - Never merge this authored repair in the same run. +PR: #288 +BRANCH: scrip-compare-003-local-cli +BASE: 5c03ee428df6cc36ebdeff2443d35ac9f7e72dca +ACCEPTANCE: Fresh exact-head diff, full canonical suite, installed complete wheel, + deterministic gallery/site, private create-only negative cases. + Later acceptance only; no same-session merge of authored changes. ``` -The 2026-10-18 owner explicitly requested a new PR for ready issues and qualifying merges. -#268/#272 are now merged, not pending dependencies; do not merge or reimplement them again. -#270 has unresolved FIFO/path-private-error findings in its old head; #274 has the known -syntax regression. Their useful code is retained in the successor, and the #273 claim is -prospectively redirected through #275. Preserve #270/#274 as provenance until successor -acceptance; do not keep developing competing integrations or blindly adopt old ZIPs. -#266 is already merged. Interactive writes succeeded in this pass, not proof of hourly -scheduled-context capability. No scheduler/admin/deployment activation change was made. +Interactive GitHub writes and expected-head merge succeeded. Historical scheduled +write denials are not current provider facts and must not be retried merely by age. +No second worker, new branch, scheduler change or administrative/deployment activation +was created. Existing earlier cleanup evidence remains in CHANGELOG and native PRs; +#266/#278 are merged, not pending work. Revalidate superseded PRs before acting. ## Next user-visible milestone and compact queue -Milestone: install the canonical Reader, analyze a local text to JSON/HTML/CSV, open the -real full-work report through the five-route static site, and inspect a source-free -Grin/Beketova offline comparison. Required acceptance: exact-head full CI, installed package -outside checkout, deterministic builds, safe errors/writes and honest scientific labels. -This incremental availability does not close M1-M5. +Milestone: install one canonical Reader; analyze and compare private UTF-8 texts to +readable HTML and exact JSON/CSV; inspect retained full-work results and exports through +the six-route static site. Actual CLI reports, not a parallel synthetic analyzer, are +the comparison contract. Incremental availability does not close M1-M5. | Priority | Unit | Deliverable and acceptance | Scoped dependency | | --- | --- | --- | --- | -| P1 | #275 / #269 / #273 | Finish exact-head Reader CI, accept in a later run, merge and reconcile superseded PRs | Real checks/protection; no same-run acceptance | -| P2 | SITE — full-work gallery route | Integrate the two pinned derived works into an explicit public allow-list; real links/downloads; desktop/mobile/keyboard checks and preserved last build | Reader publisher; admin interlock blocks DEPLOYED only | -| P3 | METRIC — comparison contract | Adopt one pending comparison slice using one shared versioned metric/method/unit/dictionary registry; actual CLI JSON -> viewer -> export tests | Revalidate pending code against canonical Reader; no parallel stub implementation | -| P4 | MORPH — AOT ambiguity diagnostic | Add a reproducible pinned-provider noun/cardinal ambiguity experiment and inspectable POS deltas | AOT provenance stays pinned; no inferred mapping promoted to parity | -| P5 | REPRO/CORPUS — evidence readiness | One machine-readable cross-work readiness/delta report from retained evidence; explicit identity/rounding/NOT_RUN gates, with a bounded new source-identity consequence | Unknown FantLab input blocks parity only; no repeated scan metadata loop | +| P1 | #288 / #279 / #286 | Finish any real exact-head CI findings, then later independent acceptance and merge; source-unit guards must retain current frozen outputs | Actual checks and branch rules; not a separate human unless genuinely required | +| P2 | #280 / #283 | Adopt browser Comparison Desk against actual Reader-generated reports; preserve private files, invalidation and exact exports; verify desktop/mobile/keyboard/touch flows | Python comparison contract and permitted browser execution; no in-memory preview presented as live | +| P3 | #281 | Adopt readiness/delta tool against the real retained catalog and reconcile proven frozen metadata; separate rights/body/size/input-identity/rounding gates | Source-unmatched comparisons remain diagnostic; one difficult source does not block other work | +| P4 | #287 | Recover POS distribution joins from the private outbox into the canonical benchmark with source/profile accounting and real CLI tests | Existing POS artifact contract; unresolved service/runtime-N/homonyms remain unknown, not zero | +| P5 | #282 | One reproducible AOT noun/cardinal ambiguity experiment with inspectable POS deltas | Pinned AOT lineage and honest inferred/extension labels | -Execution order lives only here. [ROADMAP.md](ROADMAP.md) records the acceptance details -and repository assessment. Reserve engineering attention for install/CI friction and -explicit code/data licensing before any broad release. #265 is scoped administrative -follow-through; do not let unavailable deletion/settings block P1-P5. +Execution order lives here. ROADMAP records broader acceptance and repository assessment. +Retain release attention for code/data licensing #284 and publication #285 without making +those blockers prevent independent product implementation. Existing #279-#286 already own +the corresponding outbox debts; do not duplicate them. Distinct POS-join debt is #287. ## Retained evidence and scientific gates -Detailed immutable evidence remains in `corpus/candidates/`, `benchmarks/`, `showcase/`, -`project/run-receipts/`, `project/CHANGELOG.md` and `CHANGELOG.d/`; reread candidate-specific -records before acting. This cleanup does not recalculate, admit or relabel any work. +Immutable detail remains in `corpus/candidates/`, `benchmarks/`, `showcase/`, +`project/run-receipts/`, `project/CHANGELOG.md` and `CHANGELOG.d/`. Reread the relevant +candidate records before acting; this delivery does not recalculate, admit or relabel works. - Anna Karenina: 239 pinned revisions, 1,705,605 characters; source identity unknown. - Resurrection: 129 revisions, 890,835 characters; source identity unknown. @@ -87,7 +81,8 @@ own manifest gates. Short excerpts/fixtures do not count as corpus admission. ## Scoped operational blockers -- #265: active master ruleset prevents deletion/non-fast-forward and requires PRs, with zero required approvals. The observed ruleset does not require named CI checks or resolved threads. Current connector has no branch-deletion/admin-setting mutation; login has push/triage but not admin. Reopen those operations only when supported authority/capability is present; no endpoint/ref bypass. -- Pages: repository reports has_pages=false. Existing deployment variable/configuration interlock remains untouched. BUILT is not DEPLOYED or LIVE_VERIFIED. -- Local full checkout: Git clone failed DNS in this run. Exact files and focused publisher tests were verified locally; hosted full-repository checks remain the integrated execution authority. -- Browser route execution must be tested in a permitted environment. Prior blocked navigation or in-memory previews are not visual/live acceptance for this candidate. +- #265: default-branch rules prevent deletion/non-fast-forward and require PRs, with zero required approvals in the read ruleset. Connector identity has push/triage, not admin. No unsupported deletion/settings mutation or ref bypass. +- #285 Pages: repository reports has_pages=false. Deployment configuration interlock remains untouched. BUILT is not DEPLOYED or LIVE_VERIFIED; missing administration blocks activation only. +- #284 licensing: do not infer a code/data license or publish externally before the rights gate is resolved. +- Local checkout: Git DNS failed; exact source blobs were recovered and verified for actual local tests. Hosted full-repository CI supplies complete integrated execution evidence, not the limited local closure. +- Browser HTTP navigation returned ERR_BLOCKED_BY_ADMINISTRATOR. Current visual changes are VISUAL_NOT_VERIFIED; structural tests are not live browser acceptance. Continue other executable work without bypassing the restriction. diff --git a/pyproject.toml b/pyproject.toml index a32b9f0d..85c09c9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scriptorium-reader" -version = "0.2.0a1" +version = "0.4.0a1" description = "Private literary-text analysis and evidence-labelled reader reports" readme = "README.md" requires-python = ">=3.13" diff --git a/scriptorium/cli.py b/scriptorium/cli.py index bc604c5a..a836f95a 100644 --- a/scriptorium/cli.py +++ b/scriptorium/cli.py @@ -8,7 +8,7 @@ from pathlib import Path import sys -COMMANDS = {"analyze": "scriptorium.analyze", "site": "scriptorium.site_renderer"} +COMMANDS = {"analyze": "scriptorium.analyze", "compare": "scriptorium.compare", "site": "scriptorium.site_renderer"} MAX_DIAGNOSTIC_BYTES = 8 * 1024 * 1024 @@ -49,8 +49,9 @@ def main(argv: list[str] | None = None) -> int: return 0 if not args or args[0] in {"-h", "--help"}: print("Scriptorium Reader — private text analysis and evidence-labelled literary reports") - print("Usage: scriptorium [options]") + print("Usage: scriptorium [options]") print(" analyze UTF-8 manuscript -> JSON, HTML or CSV (29 metrics)") + print(" compare Two private analysis JSON files -> HTML, JSON or CSV; no uploads") print(" diagnostic Curated historical diagnostic -> offline HTML") print(" site Build the canonical allow-listed site from a repository checkout") print("Run scriptorium --help for command options.") diff --git a/scriptorium/compare.py b/scriptorium/compare.py new file mode 100644 index 00000000..11381f1f --- /dev/null +++ b/scriptorium/compare.py @@ -0,0 +1,467 @@ +"""Strict, local-only comparison of two Scriptorium Reader analysis JSON files. + +Numbers are compared as exact JSON decimals. This is not a FantLab parity test. +""" +from __future__ import annotations + +import argparse +import csv +from decimal import Decimal, InvalidOperation, localcontext +from html import escape +import io +import json +import math +from pathlib import Path +import re + +from .analyze import ( + LOCAL_ANALYSIS_SCHEMA, MAX_ANALYSIS_BYTES, REPRESENTATIVE_CHARACTERS, + LocalInputError, _read_bounded_regular_file, _write_new, +) +from .csv_safety import safe_csv_text +from .metric_labels import METRIC_LABELS +from .metrics import analyze_deterministic_metrics + +SCHEMA = "scriptorium-private-comparison-v1" +MAX_REPORT_BYTES = 8 * 1024 * 1024 +_HEX = re.compile(r"[0-9a-f]{64}\Z") +_DICTIONARY_METRICS = frozenset(( + "fantlab.vocabulary.active_dictionary", "fantlab.vocabulary.active_nondictionary", + "fantlab.vocabulary.uasz_3000", "fantlab.vocabulary.uasz_10000", + "fantlab.vocabulary.uasz_100000", +)) + + +class CompareError(ValueError): + """A controlled, path-private problem with local derived evidence.""" + + +def _unique_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise CompareError("duplicate JSON object key") + result[key] = value + return result + + +def _decimal_token(token: str) -> Decimal: + # Do not expand unreasonable scientific exponents or unbounded digit strings. + if len(token) > 72: + raise CompareError("numeric literal exceeds comparison bounds") + try: + value = Decimal(token) + except InvalidOperation as exc: + raise CompareError("invalid numeric literal") from exc + if not value.is_finite() or abs(value.as_tuple().exponent) > 40: + raise CompareError("numeric literal exceeds comparison bounds") + return value + + +def _integer_token(token: str) -> int: + if len(token) > 16: + raise CompareError("integer literal exceeds comparison bounds") + return int(token) + + +def _reject_constant(_: str): + raise CompareError("nonfinite numeric literal") + + +def load_report(path: Path) -> dict[str, object]: + raw = _read_bounded_regular_file(path, MAX_REPORT_BYTES, "analysis JSON") + try: + result = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_pairs, + parse_int=_integer_token, parse_float=_decimal_token, + parse_constant=_reject_constant) + except (UnicodeError, ValueError, RecursionError) as exc: + raise CompareError("invalid or unsupported analysis JSON") from exc + validate_report(result) + return result + + +def _obj(value: object, keys: set[str], label: str) -> dict[str, object]: + if type(value) is not dict or set(value) != keys: + raise CompareError(f"invalid {label} contract") + return value + + +def _text(value: object, label: str, max_length: int = 160) -> str: + if type(value) is not str or not value.strip() or len(value) > max_length: + raise CompareError(f"invalid {label} identity") + return value + + +def _sha(value: object, label: str) -> str: + if type(value) is not str or _HEX.fullmatch(value) is None: + raise CompareError(f"invalid {label} digest") + return value + + +def _nonnegative_int(value: object, label: str, limit: int = 1_000_000_000) -> int: + if type(value) is not int or not 0 <= value <= limit: + raise CompareError(f"invalid {label} count") + return value + + +def _number(value: object, label: str) -> Decimal | None: + if value is None: + return None + if type(value) is int: + _nonnegative_int(value, label) + return Decimal(value) + if type(value) is float: + if not math.isfinite(value): + raise CompareError(f"invalid {label} numeric value") + value = Decimal(repr(value)) + if type(value) is not Decimal or not value.is_finite(): + raise CompareError(f"invalid {label} numeric value") + if value < 0 or value.adjusted() > 12 or value.adjusted() < -40: + raise CompareError(f"invalid {label} numeric domain") + return value + + +def _snapshot() -> dict[str, object]: + # The actual analyzer owns the 29-ID unit/evidence registry, not this viewer. + return analyze_deterministic_metrics("") + + +def validate_report(report: object) -> None: + doc = _obj(report, {"schema_version", "metric_contract_id", "source", "analyzer", + "representativeness", "metrics"}, "top-level") + canonical = _snapshot() + if doc["schema_version"] != LOCAL_ANALYSIS_SCHEMA or doc["metric_contract_id"] != canonical["metric_contract_id"]: + raise CompareError("unsupported local-analysis metric contract") + + source = _obj(doc["source"], {"raw_sha256", "byte_count", "normalized_sha256", + "normalized_character_count", "source_text_embedded"}, "source") + if source["source_text_embedded"] is not False: + raise CompareError("source text must not be embedded") + _sha(source["raw_sha256"], "raw source") + _sha(source["normalized_sha256"], "normalized source") + byte_count = _nonnegative_int(source["byte_count"], "source bytes", MAX_ANALYSIS_BYTES) + length = _nonnegative_int(source["normalized_character_count"], "normalized characters", MAX_ANALYSIS_BYTES) + if length > byte_count: + raise CompareError("inconsistent source size") + + analyzer = _obj(doc["analyzer"], {"scriptorium_revision", "profiles", "dependencies"}, "analyzer") + if analyzer["scriptorium_revision"] is not None: + _text(analyzer["scriptorium_revision"], "revision") + profiles = _obj(analyzer["profiles"], set(canonical["profiles"]), "analyzer profiles") + for key, value in profiles.items(): + _text(value, f"{key} profile") + dependencies = _obj(analyzer["dependencies"], {"vocabulary_dictionary"}, "dependencies") + dictionary = dependencies["vocabulary_dictionary"] + if dictionary is not None: + dictionary = _obj(dictionary, {"profile", "normalized_lexemes_sha256", "lexeme_count"}, "dictionary") + _text(dictionary["profile"], "dictionary profile") + _sha(dictionary["normalized_lexemes_sha256"], "dictionary") + _nonnegative_int(dictionary["lexeme_count"], "dictionary lexemes") + + representative = _obj(doc["representativeness"], {"calibration_threshold_characters", + "meets_calibration_threshold", "note"}, "representativeness") + if representative["calibration_threshold_characters"] != REPRESENTATIVE_CHARACTERS or type(representative["meets_calibration_threshold"]) is not bool or representative["meets_calibration_threshold"] != (length >= REPRESENTATIVE_CHARACTERS): + raise CompareError("invalid representativeness status") + _text(representative["note"], "representativeness note", 1000) + + rows = _obj(doc["metrics"], set(METRIC_LABELS), "29-metric collection") + baseline = canonical["metrics"] + if set(baseline) != set(METRIC_LABELS): + raise CompareError("local metric registry drift") + for metric_id, standard in baseline.items(): + row = _obj(rows[metric_id], set(standard), "metric row") + for name in ("unit", "compatibility_status", "definition_evidence"): + if row[name] != standard[name]: + raise CompareError("metric unit or evidence contract mismatch") + value = _number(row["value"], "metric") + if value is not None: + if (standard["unit"] == "percent" and value > 100) or (standard["unit"] in ("characters", "words", "sentences") and type(row["value"]) is not int): + raise CompareError("invalid metric value domain") + if "raw_count" in row: + _nonnegative_int(row["raw_count"], "punctuation raw count") + if rows["fantlab.general.characters"]["value"] != length: + raise CompareError("character metric contradicts source identity") + # These measures are emitted unconditionally as exact counts by the + # canonical deterministic engine. Treating an imported null as an + # unavailable *rate* is valid, but treating an obligatory count as null + # used to reach comparisons below and raise an uncontrolled TypeError. + required_counts = ( + "fantlab.general.words", + "scriptorium.general.sentences", + "fantlab.vocabulary.unique_words", + ) + for metric_id in required_counts: + _nonnegative_int(rows[metric_id]["value"], metric_id) + words = rows["fantlab.general.words"]["value"] + sentences = rows["scriptorium.general.sentences"]["value"] + unique = rows["fantlab.vocabulary.unique_words"]["value"] + if words > length or unique > words or sentences > length: + raise CompareError("inconsistent source/word/sentence statistics") + # The canonical deterministic engine computes these means from nonempty, + # non-overlapping word/sentence spans. An imported null with a positive + # denominator, or an impossible character budget, is not an unavailable + # metric: it is a contradictory derived report. Allow one source character + # of headroom for binary64 division rounding on very large manuscripts. + for metric_id, denominator in ( + ("fantlab.general.mean_word_length_chars", words), + ("fantlab.general.mean_sentence_length_chars", sentences), + ): + mean = _number(rows[metric_id]["value"], metric_id) + if denominator == 0: + if mean is not None: + raise CompareError("mean exists without its denominator") + elif (mean is None or mean < 1 + or mean * Decimal(denominator) > Decimal(length + 1)): + raise CompareError("mean contradicts word/sentence source budget") + # Punctuation rates are exact deterministic derivatives of the same + # exported raw event counts and word denominator (metrics-v4). A valid + # JSON schema/hash declaration alone does not prove this relationship. + # Recompute the same binary64 expression used by metrics._per_1000 and + # compare its shortest round-trip decimal, not an ad-hoc tolerance. + for metric_id, row in rows.items(): + if not metric_id.startswith("fantlab.punctuation."): + continue + raw_count = row["raw_count"] + if raw_count > length: + raise CompareError("punctuation count exceeds normalized source") + expected = None if words == 0 else Decimal(repr(raw_count * 1000.0 / words)) + if _number(row["value"], metric_id) != expected: + raise CompareError("punctuation rate contradicts raw count or word denominator") + + if dictionary is None: + if any(rows[mid]["value"] is not None for mid in _DICTIONARY_METRICS): + raise CompareError("dictionary-dependent value without dictionary") + else: + active_dictionary = _nonnegative_int( + rows["fantlab.vocabulary.active_dictionary"]["value"], "active dictionary" + ) + active_nondictionary = _nonnegative_int( + rows["fantlab.vocabulary.active_nondictionary"]["value"], "active nondictionary" + ) + if active_dictionary > dictionary["lexeme_count"] or active_dictionary + active_nondictionary != unique: + raise CompareError("dictionary partition contradicts distinct-word counts") + for window_size in (3000, 10000, 100000): + metric_id = f"fantlab.vocabulary.uasz_{window_size}" + derived = _number(rows[metric_id]["value"], metric_id) + if (derived is None) != (words < window_size): + raise CompareError("dictionary diversity window contradicts word count") + if derived is not None and derived > min(window_size, dictionary["lexeme_count"]): + raise CompareError("dictionary diversity exceeds its maximum possible size") + + +def _digest_metadata(report: dict[str, object]) -> dict[str, object]: + source, analyzer = report["source"], report["analyzer"] + return { + "raw_sha256": source["raw_sha256"], + "normalized_sha256": source["normalized_sha256"], + "bytes": source["byte_count"], + "normalized_characters": source["normalized_character_count"], + "meets_calibration_threshold": report["representativeness"]["meets_calibration_threshold"], + "scriptorium_revision": analyzer["scriptorium_revision"], + "profiles": dict(analyzer["profiles"]), + "dictionary": None if analyzer["dependencies"]["vocabulary_dictionary"] is None else dict(analyzer["dependencies"]["vocabulary_dictionary"]), + } + + +def _decimal_string(value: object) -> str | None: + number = _number(value, "metric") + return None if number is None else str(number) + + +def _exact_difference(left: Decimal, right: Decimal) -> str: + """Subtract accepted decimal observations without context-precision rounding. + + JSON admits up to 72-character decimal literals and exponents up to 40. + The Python Decimal default 28-digit context can silently round their + difference. Derive sufficient precision from the two exact operands; + the result is still a decimal string, never converted through float. + """ + lowest_exponent = min(left.as_tuple().exponent, right.as_tuple().exponent) + highest_adjusted = max(left.adjusted(), right.adjusted()) + required_digits = max(1, highest_adjusted - lowest_exponent + 1) + with localcontext() as ctx: + ctx.prec = max(28, required_digits) + return str(right - left) + + +def _validate_cross_report_identity(left: dict[str, object], right: dict[str, object]) -> None: + """Reject contradictions provable from digest and version identities alone. + + The reports are unauthenticated declarations; these checks do *not* prove + either digest is truthful. They prevent an imported pair from asserting + contradictory lengths/normalizations or deterministic values for the + same declared normalized bytes and exact algorithm/dependency identity. + """ + a_source, b_source = left["source"], right["source"] + a_engine, b_engine = left["analyzer"], right["analyzer"] + same_raw = a_source["raw_sha256"] == b_source["raw_sha256"] + same_normalized = a_source["normalized_sha256"] == b_source["normalized_sha256"] + if same_raw: + if a_source["byte_count"] != b_source["byte_count"]: + raise CompareError("same raw digest has inconsistent byte counts") + if (a_engine["profiles"]["normalization"] == b_engine["profiles"]["normalization"] + and not same_normalized): + raise CompareError("same raw digest and normalization profile have different normalized digests") + if same_normalized and a_source["normalized_character_count"] != b_source["normalized_character_count"]: + raise CompareError("same normalized digest has inconsistent character counts") + # Declared algorithm profiles and dictionary content determine method + # identity. A changed commit/revision label alone is not a declared new + # metric algorithm: the same normalized input must still be deterministic. + # Otherwise the comparison would silently report a nonzero difference for + # one text despite claiming its two declared measurement methods match. + if same_normalized and a_engine["profiles"] == b_engine["profiles"]: + # Dictionary identity is a dependency of only five vocabulary metrics. + # Changing it must not excuse drift in word, sentence, punctuation or + # dialogue values for the very same normalized bytes and algorithm. + dictionary_equal = a_engine["dependencies"] == b_engine["dependencies"] + for metric_id in METRIC_LABELS: + if metric_id in _DICTIONARY_METRICS and not dictionary_equal: + continue + a_row, b_row = left["metrics"][metric_id], right["metrics"][metric_id] + if (_number(a_row["value"], metric_id) != _number(b_row["value"], metric_id) + or a_row.get("raw_count") != b_row.get("raw_count")): + raise CompareError("same normalized input and method have contradictory measurements") + + +def build_comparison(left: dict[str, object], right: dict[str, object]) -> dict[str, object]: + validate_report(left) + validate_report(right) + _validate_cross_report_identity(left, right) + rows = [] + profiles_equal = left["analyzer"]["profiles"] == right["analyzer"]["profiles"] + dictionaries_equal = left["analyzer"]["dependencies"] == right["analyzer"]["dependencies"] + canonical_rows = _snapshot()["metrics"] + for metric_id in sorted(METRIC_LABELS): + unit = canonical_rows[metric_id]["unit"] + a, b = left["metrics"][metric_id]["value"], right["metrics"][metric_id]["value"] + a_value, b_value = _number(a, "metric"), _number(b, "metric") + if not profiles_equal: + status, reason = "incomparable", "Algorithm profiles differ" + elif metric_id in _DICTIONARY_METRICS and not dictionaries_equal: + status, reason = "incomparable", "Dictionary identities differ" + elif a_value is None or b_value is None: + status, reason = "unavailable", "At least one measurement is not run or undefined" + else: + status, reason = "comparable", "Both derived values use the same declared method" + rows.append({ + "metric_id": metric_id, + "label": METRIC_LABELS[metric_id], + "unit": unit, + "definition_evidence": canonical_rows[metric_id]["definition_evidence"], + "compatibility_status": canonical_rows[metric_id]["compatibility_status"], + "left": _decimal_string(a), + "right": _decimal_string(b), + "difference": _exact_difference(a_value, b_value) if status == "comparable" else None, + "difference_unit": "percentage_points" if status == "comparable" and unit == "percent" else unit if status == "comparable" else None, + "comparison_status": status, + "reason": reason, + }) + return { + "schema_version": SCHEMA, + "metric_contract_id": left["metric_contract_id"], + "direction": "right_minus_left", + "scientific_boundary": "Inferred/extension local measurements only; not verified FantLab parity (M2 0/5).", + "left_source": _digest_metadata(left), + "right_source": _digest_metadata(right), + "metrics": rows, + } + + +def render_json(comparison: dict[str, object]) -> str: + return json.dumps(comparison, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False) + "\n" + + +def render_csv(comparison: dict[str, object]) -> str: + out = io.StringIO(newline="") + writer = csv.writer(out, lineterminator="\n") + fields = ("metric_id", "label", "unit", "compatibility_status", "definition_evidence", + "left", "right", "difference", "difference_unit", "comparison_status", "reason", + "left_sha256", "right_sha256") + writer.writerow(fields) + for row in comparison["metrics"]: + values = [row.get(k) for k in fields[:11]] + [comparison["left_source"]["raw_sha256"], comparison["right_source"]["raw_sha256"]] + writer.writerow([("" if v is None else str(v) if index in (5, 6, 7) else safe_csv_text(str(v))) + for index, v in enumerate(values)]) + return out.getvalue() + + +def render_html(comparison: dict[str, object], *, title: str = "Two private analyses") -> str: + name = escape(title) + rows = [] + for row in comparison["metrics"]: + def cell(key: str) -> str: + value = row[key] + return escape(str(value)) if value is not None else "Not available" + rows.append( + "" + escape(row["label"]) + "" + escape(row["metric_id"]) + + "" + cell("left") + "" + cell("right") + + "" + cell("unit") + "" + cell("difference") + "" + cell("difference_unit") + + "" + cell("comparison_status") + "" + cell("reason") + "" + ) + def source_card(label: str, source: dict[str, object]) -> str: + return ("

" + escape(label) + "

Normalized characters: " + + str(source["normalized_characters"]) + "

Source SHA-256: " + + escape(source["raw_sha256"]) + "

Reproducibility details" + + "

Normalized SHA-256: " + escape(source["normalized_sha256"]) + "

" + + "

Declared revision: " + escape(str(source["scriptorium_revision"] or "not recorded")) + "

" + + "

Dictionary identity: " + escape(json.dumps(source["dictionary"], ensure_ascii=False, sort_keys=True)) + "

" + + "

Length meets calibration floor: " + str(source["meets_calibration_threshold"]) + "

" + + "
" + "".join("
" + escape(k) + "
" + escape(v) + "
" + for k, v in sorted(source["profiles"].items())) + "
") + css = """*{box-sizing:border-box}body{margin:0;background:#f7f3ea;color:#243b34;font:16px/1.6 system-ui,sans-serif} +main{max-width:75rem;margin:auto;padding:clamp(1rem,4vw,3rem)}h1,h2{font-family:Georgia,serif;line-height:1.16} +h1{font-size:clamp(2.2rem,5vw,4rem)}.eyebrow{font-weight:700;letter-spacing:.12em;text-transform:uppercase;color:#23615b} +.lead{max-width:69ch;font-size:1.12rem}.notice{background:#e3ece7;padding:1rem 1.25rem;border-left:4px solid #35766e} +.cards{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem;margin:2rem 0}.card{background:#fffdfa;padding:1.25rem;border:1px solid #d8d0c0;border-radius:.6rem;min-width:0} +code{overflow-wrap:anywhere}.scroll{overflow:auto;border:1px solid #d8d0c0;background:#fffdfa;border-radius:.5rem} +table{border-collapse:collapse;width:100%;min-width:50rem;font-variant-numeric:tabular-nums}th,td{text-align:left;border-bottom:1px solid #d8d0c0;padding:.75rem;vertical-align:top}caption{text-align:left;padding:1rem;font-weight:600} +small{display:block;font-size:.75rem;font-weight:400;color:#4f635a}details summary{cursor:pointer;padding:.75rem 0;min-height:44px} +.skip{position:absolute;left:-9999px}.skip:focus{left:1rem;top:1rem;background:white;padding:1rem} +:focus-visible{outline:3px solid #a15b1e;outline-offset:3px}@media(max-width:680px){.cards{grid-template-columns:1fr}} +@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important}}""" + return ("" + + "" + name + " · Scriptorium" + + "

Scriptorium / Private comparison

" + + "

" + name + "

Compare two local reports without uploading the books. " + + "Values are exact JSON decimal observations; difference is right minus left.

" + + escape(comparison["scientific_boundary"]) + " Unavailable or method-incompatible values have no difference. " + + "Percent differences are percentage points, never relative percent changes.

" + + "

Imported JSON values and hashes are unauthenticated declarations. Rerun the analyzer on your original manuscript to verify them.

" + + source_card("Left report", comparison["left_source"]) + source_card("Right report", comparison["right_source"]) + + "

Measurements

" + + "
" + + "" + + "" + + "" + + "" + "".join(rows) + "
29 source-free metric comparisons, values and units
MetricLeftRightUnitDifferenceDifference unitStatus
" + + "

No tracking, server, manuscript text or FantLab parity claim.

\n") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Compare two locally generated Scriptorium Reader JSON reports.") + parser.add_argument("left", type=Path) + parser.add_argument("right", type=Path) + parser.add_argument("--format", choices=("json", "html", "csv"), default="html") + parser.add_argument("--title", default="Two private analyses") + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + try: + comparison = build_comparison(load_report(args.left), load_report(args.right)) + rendered = {"html": lambda: render_html(comparison, title=args.title), + "json": lambda: render_json(comparison), + "csv": lambda: render_csv(comparison)}[args.format]() + if args.output is None: + print(rendered, end="") + else: + _write_new(args.output, rendered) + except FileExistsError: + parser.error("comparison output already exists; choose another destination") + except (LocalInputError, CompareError): + parser.error("invalid or unsupported local analysis JSON; check the two generated reports") + except (OSError, UnicodeError, ValueError, TypeError, RecursionError): + parser.error("unable to compare local analyses or create the report") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scriptorium/full_work_gallery.py b/scriptorium/full_work_gallery.py index e3d94d8c..b0f6b81c 100644 --- a/scriptorium/full_work_gallery.py +++ b/scriptorium/full_work_gallery.py @@ -31,6 +31,20 @@ ("fantlab.punctuation.question.per_1000_words", "Question marks", "per 1,000 words", True), ) +# Source JSON uses stable scientific units, whereas the human Reader uses labels. +# This is a unit conversion (not a second registry of metric IDs); METRICS remains +# the single declaration of the ten displayed measurements. +_SOURCE_UNIT_BY_DISPLAY_UNIT = { + "characters": "characters", + "words": "words", + "characters/word": "characters/word", + "characters/sentence": "characters/sentence", + "%": "percent", + "types": "words", + "per 1,000 words": "occurrences/1000_words", +} + + class GalleryError(ValueError): """Unverified or unsafe source-free evidence.""" @@ -87,6 +101,8 @@ def _load(root: Path, index: int): if not isinstance(metric_rows, list) or len(metric_rows) > 500: raise GalleryError("invalid metric collection") values = {} + # Grin diagnostics have no source-unit field. Do not invent one for them. + source_units = {} if index == 1 else None for row in metric_rows: if not isinstance(row, dict) or not isinstance(row.get("metric_id"), str): raise GalleryError("invalid metric row") @@ -103,10 +119,16 @@ def _load(root: Path, index: int): if status == "not_run_dependency_unbound" and number is not None: raise GalleryError("not-run cannot have a value") values[mid] = number + if source_units is not None: + source_units[mid] = row.get("unit") for key, _, unit, normalized in METRICS: value = values.get(key) if value is None or value < 0 or (unit == "%" and value > 100): raise GalleryError("missing or invalid shared measurement") + # A newly approved, correctly repinned derived JSON may still be invalid + # for presentation (e.g. ratio silently displayed as a percent). + if source_units is not None and source_units.get(key) != _SOURCE_UNIT_BY_DISPLAY_UNIT[unit]: + raise GalleryError("incompatible source unit for displayed metric") # Cardinal observations are countable events; JSON 10.0 is not an integer. for metric_id in ("fantlab.general.characters", "fantlab.general.words", "fantlab.vocabulary.unique_words"): @@ -169,7 +191,7 @@ def render_html(records): cssclass + "' style='width:" + width + "%'>" + format(value, ".2f") + "") charts.append("

  • " + esc(label) + " " + esc(unit) + "

    " + "".join(bars) + "
    • ") - css = """*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:#f6f1e5;color:#20312b;font:16px/1.55 system-ui,sans-serif}.wrap{max-width:1120px;margin:auto;padding:clamp(1rem,4vw,3rem)}h1,h2,h3{font-family:Georgia,serif;font-weight:400;line-height:1.15}h1{font-size:clamp(2.8rem,6vw,5.5rem)}.lede{font-size:1.15rem;max-width:65ch}.eyebrow{font-size:.8rem;color:#20665f;text-transform:uppercase;letter-spacing:.09em;font-weight:700}.skip{position:absolute;top:-5rem;left:1rem;background:white;padding:1rem}.skip:focus{top:1rem}a{color:#14615a}a:focus-visible,summary:focus-visible,.scroll:focus-visible{outline:3px solid #a24f07;outline-offset:4px}.notice{padding:1rem 1.4rem;background:#e1ece6;border-left:5px solid #266b64}.cards,.charts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem}.card,.charts li{min-width:0;background:#fffdf8;border:1px solid #d7cebd;border-radius:.7rem;padding:1.4rem}.big{display:block;font:3rem Georgia,serif;margin:1rem 0}.card code{overflow-wrap:anywhere}.charts{padding:0;list-style:none}.charts h3{font-size:1.2rem}.charts small,th small{font:.8rem system-ui,sans-serif;color:#52635b}.bar{display:grid;grid-template-columns:3rem minmax(0,1fr) 4rem;gap:.6rem;align-items:center;font-size:.85rem;margin:.7rem 0}.bar strong{text-align:right}.track{background:#e2e0d5;height:.9rem;border-radius:3px;overflow:hidden}.fill{height:100%;border-radius:3px}.teal{background:#277069}.ochre{background:#ae7134}section{padding-top:2.5rem}.scroll{overflow:auto;background:#fffdf8;border:1px solid #d7cebd;border-radius:.5rem}table{border-collapse:collapse;width:100%;min-width:570px}caption{text-align:left;padding:1rem}th,td{text-align:left;padding:.8rem 1rem;border-bottom:1px solid #ded4c5;font-variant-numeric:tabular-nums}summary{cursor:pointer;min-height:44px;padding:.8rem 0;font-weight:700}footer{margin-top:3rem;border-top:1px solid #d7cebd;padding:1.5rem 0}@media(max-width:700px){.cards,.charts{grid-template-columns:1fr}}@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}}""" + css = """*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:#f6f1e5;color:#20312b;font:16px/1.55 system-ui,sans-serif}.wrap{max-width:1120px;margin:auto;padding:clamp(1rem,4vw,3rem)}h1,h2,h3{font-family:Georgia,serif;font-weight:400;line-height:1.15}h1{font-size:clamp(2.8rem,6vw,5.5rem)}.lede{font-size:1.15rem;max-width:65ch}.eyebrow{font-size:.8rem;color:#20665f;text-transform:uppercase;letter-spacing:.09em;font-weight:700}.skip{position:absolute;top:-5rem;left:1rem;background:white;padding:1rem}.skip:focus{top:1rem}a{color:#14615a}a:focus-visible,summary:focus-visible,.scroll:focus-visible{outline:3px solid #a24f07;outline-offset:4px}.notice{padding:1rem 1.4rem;background:#e1ece6;border-left:5px solid #266b64}.cards,.charts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem}.card,.charts li{min-width:0;background:#fffdf8;border:1px solid #d7cebd;border-radius:.7rem;padding:1.4rem}.big{display:block;font:3rem Georgia,serif;margin:1rem 0}.card code{overflow-wrap:anywhere}.charts{padding:0;list-style:none}.charts h3{font-size:1.2rem}.charts small,th small{font:.8rem system-ui,sans-serif;color:#52635b}.bar{display:grid;grid-template-columns:3rem minmax(0,1fr) 4rem;gap:.6rem;align-items:center;font-size:.85rem;margin:.7rem 0}.bar strong{text-align:right}.track{background:#e2e0d5;height:.9rem;border-radius:3px;overflow:hidden}.fill{height:100%;border-radius:3px}.teal{background:#277069}.ochre{background:#ae7134}section{padding-top:2.5rem}.scroll{overflow:auto;background:#fffdf8;border:1px solid #d7cebd;border-radius:.5rem}table{border-collapse:collapse;width:100%;min-width:570px}caption{text-align:left;padding:1rem}th,td{text-align:left;padding:.8rem 1rem;border-bottom:1px solid #ded4c5;font-variant-numeric:tabular-nums}summary{cursor:pointer;min-height:44px;padding:.8rem 0;font-weight:700}footer{margin-top:3rem;border-top:1px solid #d7cebd;padding:1.5rem 0}nav a,.actions a{display:inline-flex;align-items:center;min-height:44px;padding:.65rem .85rem}nav{display:flex;flex-wrap:wrap;gap:.5rem}.actions{display:flex;flex-wrap:wrap;align-items:center;gap:.75rem}.actions a{border:1px solid #266b64;border-radius:.35rem;background:#e1ece6}#main:focus-visible,#comparison:focus-visible{outline:3px solid #a24f07;outline-offset:4px}@media(max-width:700px){.cards,.charts{grid-template-columns:1fr}}@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}}""" return ("\n" + "Scriptorium — Two analyzed works" + "

    Scriptorium / Literary measurement

    " + diff --git a/tests/test_compare.py b/tests/test_compare.py new file mode 100644 index 00000000..624c7a1f --- /dev/null +++ b/tests/test_compare.py @@ -0,0 +1,428 @@ +"""Private Reader comparison contract and CLI end-to-end regressions.""" +from __future__ import annotations + +import contextlib +import copy +import csv +from decimal import Decimal +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from scriptorium import cli +from scriptorium.analyze import build_local_analysis +from scriptorium import compare + + +LEFT = "Кот спит. Кот и пёс гуляют!\n— Да? — сказал пёс." +RIGHT = "Кот, кот, кот!\n— А? — ответил он." + + +class LocalComparisonTests(unittest.TestCase): + def setUp(self): + self.a = build_local_analysis(LEFT.encode("utf-8")) + self.b = build_local_analysis(RIGHT.encode("utf-8")) + + def test_complete_29_metric_contract_and_decimal_differences(self): + value = compare.build_comparison(self.a, self.b) + self.assertEqual(value["schema_version"], "scriptorium-private-comparison-v1") + self.assertEqual(len(value["metrics"]), 29) + self.assertEqual({r["metric_id"] for r in value["metrics"]}, set(self.a["metrics"])) + rows = {r["metric_id"]: r for r in value["metrics"]} + count = rows["fantlab.general.words"] + self.assertEqual(count["difference"], str(self.b["metrics"]["fantlab.general.words"]["value"] - self.a["metrics"]["fantlab.general.words"]["value"])) + self.assertEqual(count["difference_unit"], "words") + share = rows["fantlab.dialogue.share_percent"] + self.assertEqual(share["difference_unit"], "percentage_points") + self.assertEqual(share["difference"], str(Decimal(repr(self.b["metrics"]["fantlab.dialogue.share_percent"]["value"])) - Decimal(repr(self.a["metrics"]["fantlab.dialogue.share_percent"]["value"])))) + self.assertTrue(any(r["comparison_status"] == "unavailable" for r in value["metrics"])) + self.assertEqual(value["scientific_boundary"].count("M2 0/5"), 1) + + def test_float_lexemes_produce_exact_decimal_difference(self): + self.a["metrics"]["fantlab.general.mean_word_length_chars"]["value"] = Decimal("1.0000000000000003") + self.b["metrics"]["fantlab.general.mean_word_length_chars"]["value"] = Decimal("1.0000000000000004") + value = compare.build_comparison(self.a, self.b) + row = next(r for r in value["metrics"] if r["metric_id"] == "fantlab.general.mean_word_length_chars") + self.assertEqual(row["difference"], "1E-16") + self.assertEqual(row["left"], "1.0000000000000003") + + def test_long_json_decimal_lexemes_do_not_round_difference(self): + # Both are accepted by the 72-character JSON literal gate. A normal + # Decimal subtraction at default context precision silently loses + # the 29th+ significant digit, violating exact decimal deltas. + cases = ( + ("2.123456789012345678901234567890", "2.111111111111111111111111111111", + "-0.012345677901234567790123456779"), + ("4.0000000000000000000000000000000000000001", + "3.0000000000000000000000000000000000000002", + "-0.9999999999999999999999999999999999999999"), + ) + for first, second, expected in cases: + with self.subTest(first=first, second=second): + left = copy.deepcopy(self.a) + right = copy.deepcopy(self.b) + metric = "fantlab.general.mean_word_length_chars" + left["metrics"][metric]["value"] = Decimal(first) + right["metrics"][metric]["value"] = Decimal(second) + row = next(r for r in compare.build_comparison(left, right)["metrics"] + if r["metric_id"] == metric) + self.assertEqual(row["difference"], expected) + self.assertEqual(row["difference_unit"], "characters/word") + + def test_imported_mean_length_and_presence_match_source_budget(self): + # The actual engine produces mandatory means when a word/sentence + # denominator is nonzero. Their character budget cannot exceed the + # normalized source. Zero-length inputs keep the mean unavailable. + accepted = self.a + compare.validate_report(accepted) + for metric_id in ( + "fantlab.general.mean_word_length_chars", + "fantlab.general.mean_sentence_length_chars", + ): + with self.subTest(metric_id=metric_id): + for bad_value in (None, 0, self.a["source"]["normalized_character_count"] + 1000): + invalid = copy.deepcopy(self.a) + invalid["metrics"][metric_id]["value"] = bad_value + with self.assertRaises(compare.CompareError): + compare.validate_report(invalid) + empty = build_local_analysis(b"") + for metric_id in ( + "fantlab.general.mean_word_length_chars", + "fantlab.general.mean_sentence_length_chars", + ): + self.assertIsNone(empty["metrics"][metric_id]["value"]) + compare.validate_report(empty) + invalid = copy.deepcopy(empty) + invalid["metrics"][metric_id]["value"] = 0 + with self.assertRaises(compare.CompareError): + compare.validate_report(invalid) + + def test_changed_dictionary_cannot_hide_same_text_dialogue_drift(self): + # Both are genuine Reader artifacts from identical bytes. Their only + # declared analysis dependency change is the optional lexicon. + left = self.a + right = build_local_analysis( + LEFT.encode("utf-8"), dictionary_words=["кот"], + dictionary_profile="test-lexicon-v1", + ) + self.assertEqual(left["source"]["normalized_sha256"], right["source"]["normalized_sha256"]) + self.assertEqual(left["analyzer"]["profiles"], right["analyzer"]["profiles"]) + compare.validate_report(right) + rows = {r["metric_id"]: r for r in compare.build_comparison(left, right)["metrics"]} + self.assertEqual(rows["fantlab.vocabulary.active_dictionary"]["comparison_status"], "incomparable") + self.assertEqual(rows["fantlab.general.words"]["difference"], "0") + right["metrics"]["fantlab.dialogue.share_percent"]["value"] += 1 + compare.validate_report(right) + with self.assertRaises(compare.CompareError): + compare.build_comparison(left, right) + + def test_changed_dictionary_cannot_hide_same_text_punctuation_drift(self): + left = self.a + right = build_local_analysis( + LEFT.encode("utf-8"), dictionary_words=["кот"], + dictionary_profile="test-lexicon-v1", + ) + row = right["metrics"]["fantlab.punctuation.dash.per_1000_words"] + row["raw_count"] += 1 + words = right["metrics"]["fantlab.general.words"]["value"] + row["value"] = row["raw_count"] * 1000.0 / words + compare.validate_report(right) + with self.assertRaises(compare.CompareError): + compare.build_comparison(left, right) + + def test_mismatched_algorithm_prevents_all_deltas(self): + self.b["analyzer"]["profiles"]["metrics"] = "future-metrics-v99" + value = compare.build_comparison(self.a, self.b) + self.assertTrue(all(r["comparison_status"] == "incomparable" and r["difference"] is None for r in value["metrics"])) + + def test_dictionary_mismatch_only_blocks_dependent_rows(self): + left = build_local_analysis(LEFT.encode(), dictionary_words=["кот"], dictionary_profile="same-dictionary-label") + right = build_local_analysis(RIGHT.encode(), dictionary_words=["пёс"], dictionary_profile="same-dictionary-label") + rows = {r["metric_id"]: r for r in compare.build_comparison(left, right)["metrics"]} + self.assertEqual(rows["fantlab.vocabulary.active_dictionary"]["comparison_status"], "incomparable") + self.assertIsNone(rows["fantlab.vocabulary.active_dictionary"]["difference"]) + self.assertEqual(rows["fantlab.general.words"]["comparison_status"], "comparable") + self.assertEqual(rows["fantlab.vocabulary.unique_words"]["comparison_status"], "comparable") + + def test_identical_normalized_input_must_have_deterministic_metrics(self): + left = build_local_analysis(LEFT.encode("utf-8")) + same = copy.deepcopy(left) + compare.build_comparison(left, same) + same["metrics"]["fantlab.dialogue.share_percent"]["value"] += 1.0 + compare.validate_report(same) + with self.assertRaisesRegex(compare.CompareError, "contradictory measurements"): + compare.build_comparison(left, same) + + def test_changed_revision_label_cannot_hide_same_method_contradiction(self): + # Revision is provenance, not a substitute for a versioned algorithm + # or dictionary profile. With identical normalized source and declared + # method a different commit label must not produce a fabricated delta. + left = build_local_analysis(LEFT.encode("utf-8"), scriptorium_revision="first-commit") + right = copy.deepcopy(left) + right["analyzer"]["scriptorium_revision"] = "later-commit" + compare.build_comparison(left, right) # different label; same values OK + right["metrics"]["fantlab.dialogue.share_percent"]["value"] += 1.0 + compare.validate_report(right) # a single report cannot prove this violation + with self.assertRaisesRegex(compare.CompareError, "contradictory measurements"): + compare.build_comparison(left, right) + + def test_identical_normalized_input_cannot_have_different_punctuation_counts(self): + left = build_local_analysis(LEFT.encode("utf-8")) + same = copy.deepcopy(left) + row = same["metrics"]["fantlab.punctuation.comma.per_1000_words"] + row["raw_count"] += 1 + row["value"] = row["raw_count"] * 1000.0 / same["metrics"]["fantlab.general.words"]["value"] + compare.validate_report(same) + with self.assertRaises(compare.CompareError): + compare.build_comparison(left, same) + + def test_equal_normalized_text_with_different_raw_newlines_is_allowed(self): + lf = build_local_analysis(LEFT.encode("utf-8")) + crlf = build_local_analysis(LEFT.replace("\n", "\r\n").encode("utf-8")) + self.assertNotEqual(lf["source"]["raw_sha256"], crlf["source"]["raw_sha256"]) + self.assertEqual(lf["source"]["normalized_sha256"], crlf["source"]["normalized_sha256"]) + compare.build_comparison(lf, crlf) + + def test_equal_raw_digest_cannot_have_inconsistent_source_metadata(self): + left = build_local_analysis(LEFT.encode("utf-8")) + changed_bytes = copy.deepcopy(left) + changed_bytes["source"]["byte_count"] += 1 + compare.validate_report(changed_bytes) + with self.assertRaises(compare.CompareError): + compare.build_comparison(left, changed_bytes) + changed_normalized = copy.deepcopy(left) + changed_normalized["source"]["normalized_sha256"] = "0" * 64 + compare.validate_report(changed_normalized) + with self.assertRaises(compare.CompareError): + compare.build_comparison(left, changed_normalized) + + def test_changed_method_does_not_claim_same_deterministic_measures(self): + left = build_local_analysis(LEFT.encode("utf-8")) + different = copy.deepcopy(left) + different["analyzer"]["profiles"]["punctuation"] = "candidate-v99" + different["metrics"]["fantlab.dialogue.share_percent"]["value"] += 1.0 + result = compare.build_comparison(left, different) + self.assertTrue(all(row["comparison_status"] == "incomparable" for row in result["metrics"])) + + def test_schema_unit_source_and_numeric_mutations_rejected(self): + edits = ( + lambda d: d["metrics"]["fantlab.dialogue.share_percent"].update(unit="ratio"), + lambda d: d["metrics"]["fantlab.general.words"].update(value=True), + lambda d: d["metrics"]["fantlab.general.words"].update(value=1.0), + lambda d: d["metrics"]["fantlab.dialogue.share_percent"].update(value=101), + lambda d: d["metrics"].pop("fantlab.general.words"), + lambda d: d["source"].update(source_text_embedded=True), + lambda d: d["source"].update(normalized_sha256="missing"), + lambda d: d["source"].update(normalized_character_count=1), + lambda d: d.update(manuscript=LEFT), + lambda d: d["analyzer"]["profiles"].update(dialogue=None), + lambda d: d["representativeness"].update(meets_calibration_threshold=True), + lambda d: d["metrics"]["fantlab.punctuation.comma.per_1000_words"].update(raw_count=-1), + ) + for edit in edits: + with self.subTest(edit=str(edit)): + artifact = copy.deepcopy(self.a) + edit(artifact) + with self.assertRaises(compare.CompareError): + compare.build_comparison(artifact, self.b) + + def test_mandatory_count_contract_rejects_null_and_impossible_counts(self): + # Imported reports are untrusted declarations. The actual analyzer + # *always* emits these counts, even for the empty input. None is only + # allowed for genuinely undefined optional derived statistics. + required = ( + "fantlab.general.words", + "scriptorium.general.sentences", + "fantlab.vocabulary.unique_words", + ) + for metric_id in required: + with self.subTest(metric_id=metric_id): + poisoned = copy.deepcopy(self.a) + poisoned["metrics"][metric_id]["value"] = None + with self.assertRaises(compare.CompareError): + compare.build_comparison(poisoned, self.b) + for metric_id in ("scriptorium.general.sentences", "fantlab.general.words"): + with self.subTest(impossible_count=metric_id): + poisoned = copy.deepcopy(self.a) + poisoned["metrics"][metric_id]["value"] = ( + poisoned["source"]["normalized_character_count"] + 1 + ) + with self.assertRaises(compare.CompareError): + compare.build_comparison(poisoned, self.b) + # Genuine undefined metrics remain representable and comparable as + # unavailable, rather than being silently synthesized as zero. + result = compare.build_comparison(self.a, self.b) + self.assertTrue(any(r["comparison_status"] == "unavailable" for r in result["metrics"])) + + def test_derived_punctuation_rate_integrity_is_enforced(self): + # The canonical metrics-v4 engine derives all 14 rates using + # raw_count * 1000.0 / words. A schema-valid JSON report with a + # different claimed denominator/rate cannot produce a trustworthy + # comparison, even if its source digest fields look well formed. + key = "fantlab.punctuation.comma.per_1000_words" + baseline = self.a["metrics"][key] + self.assertIsNotNone(baseline["value"]) + altered = copy.deepcopy(self.a) + altered["metrics"][key]["value"] = baseline["value"] + 1.0 + with self.assertRaises(compare.CompareError): + compare.build_comparison(altered, self.b) + + too_many = copy.deepcopy(self.a) + count = too_many["source"]["normalized_character_count"] + 1 + too_many["metrics"][key]["raw_count"] = count + words = too_many["metrics"]["fantlab.general.words"]["value"] + too_many["metrics"][key]["value"] = count * 1000.0 / words + with self.assertRaises(compare.CompareError): + compare.build_comparison(too_many, self.b) + + missing = copy.deepcopy(self.a) + missing["metrics"][key]["value"] = None + with self.assertRaises(compare.CompareError): + compare.build_comparison(missing, self.b) + + def test_dictionary_partition_and_window_integrity(self): + valid = build_local_analysis( + LEFT.encode("utf-8"), dictionary_words=["кот", "пёс"], + dictionary_profile="tested-lexemes-v1", + ) + compare.validate_report(valid) + dictionary_key = "fantlab.vocabulary.active_dictionary" + no_count = copy.deepcopy(valid) + no_count["metrics"][dictionary_key]["value"] = None + with self.assertRaises(compare.CompareError): + compare.validate_report(no_count) + + wrong_partition = copy.deepcopy(valid) + wrong_partition["metrics"][dictionary_key]["value"] += 1 + with self.assertRaises(compare.CompareError): + compare.validate_report(wrong_partition) + + unexpected_window = copy.deepcopy(valid) + unexpected_window["metrics"]["fantlab.vocabulary.uasz_3000"]["value"] = 0.0 + with self.assertRaises(compare.CompareError): + compare.validate_report(unexpected_window) + + enough_words = build_local_analysis( + ("Кот " * 3000).encode("utf-8"), dictionary_words=["кот"], + dictionary_profile="tested-lexemes-v1", + ) + self.assertIsNotNone(enough_words["metrics"]["fantlab.vocabulary.uasz_3000"]["value"]) + compare.validate_report(enough_words) + contradictory = copy.deepcopy(enough_words) + contradictory["metrics"]["fantlab.vocabulary.uasz_3000"]["value"] = None + with self.assertRaises(compare.CompareError): + compare.validate_report(contradictory) + + def test_punctuation_rates_are_unavailable_without_words(self): + punctuation_only = build_local_analysis("?! ... —".encode("utf-8")) + self.assertEqual(punctuation_only["metrics"]["fantlab.general.words"]["value"], 0) + compare.validate_report(punctuation_only) + with self.assertRaises(compare.CompareError): + bad = copy.deepcopy(punctuation_only) + bad["metrics"]["fantlab.punctuation.question.per_1000_words"]["value"] = 0.0 + compare.validate_report(bad) + + def test_private_html_csv_and_json_are_deterministic(self): + result = compare.build_comparison(self.a, self.b) + html = compare.render_html(result, title="My Books") + self.assertIn("My <a> Books", html) + self.assertEqual(html.count(""), 29) + self.assertIn("", html) + self.assertIn("prefers-reduced-motion", html) + self.assertNotIn(LEFT, html) + self.assertNotIn("

    "), 7) + self.assertTrue(all("unit" not in row for row in self.docs[0]["metrics"])) + + def test_canonical_frozen_records_retain_exact_metrics_and_units(self): + root = Path(__file__).resolve().parents[1] + records = gallery.load_gallery(root) + self.assertEqual([record["body"] for record in records], [363819, 1095467]) + self.assertEqual(len(gallery.METRICS), 10) + self.assertEqual(len(gallery.render_csv(records).splitlines()), 21) + page = gallery.render_html(records) + self.assertEqual(page.count("

  • "), 7) + self.assertIn("nav a,.actions a{display:inline-flex;align-items:center;min-height:44px", page) + self.assertIn("summary:focus-visible", page) + self.assertIn("prefers-reduced-motion", page) + self.assertIn("M2 0/5", page) + + def test_percent_to_ratio_repin_is_rejected(self): + self._edit_beketova_unit("%", "ratio") + with self.assertRaisesRegex(gallery.GalleryError, "source unit"): + self._load() + + def test_per_thousand_rate_changed_to_words_repin_is_rejected(self): + self._edit_beketova_unit("per 1,000 words", "words") + with self.assertRaisesRegex(gallery.GalleryError, "source unit"): + self._load() + + def test_cardinal_unit_changed_to_percent_repin_is_rejected(self): + self._edit_beketova_unit("characters", "percent") + with self.assertRaisesRegex(gallery.GalleryError, "source unit"): + self._load() + + def test_unique_word_forms_use_source_words_not_display_types(self): + self._edit_beketova_unit("types", "types") + with self.assertRaisesRegex(gallery.GalleryError, "source unit"): + self._load() + + def test_missing_null_and_nonnumeric_unit_fail_closed(self): + for invalid in (None, 100, [], {}): + with self.subTest(invalid=invalid): + self._edit_beketova_unit("characters/word", invalid) + with self.assertRaisesRegex(gallery.GalleryError, "source unit"): + self._load() + + def test_repin_does_not_silently_upgrade_scientific_status(self): + self.docs[1]["gates"]["m2_weight"] = 1 + with self.assertRaisesRegex(gallery.GalleryError, "scientific gate"): + self._load() + + def test_pins_are_still_mandatory(self): + specs = self._refresh_pins() + target = self.root / "corpus/candidates/diagnostics" / (specs[1][0] + ".json") + target.write_bytes(target.read_bytes() + b" ") + with patch.object(gallery, "SOURCES", specs), self.assertRaisesRegex(gallery.GalleryError, "checksum"): + gallery.load_gallery(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_imported_mean_integrity.py b/tests/test_imported_mean_integrity.py new file mode 100644 index 00000000..e4f3bc4e --- /dev/null +++ b/tests/test_imported_mean_integrity.py @@ -0,0 +1,75 @@ +"""Imported-mean regressions using the actual canonical Reader, without stubs.""" +from __future__ import annotations +import copy +from pathlib import Path +import tempfile +import json +import unittest +from scriptorium import compare +from scriptorium.analyze import build_local_analysis + +class MeanIntegrityTests(unittest.TestCase): + def setUp(self): + self.left = build_local_analysis('Вечер. — Мы читаем книгу, — сказал он.\nТихо светит лампа.'.encode()) + self.right = build_local_analysis('Утро пришло. Открой окно!'.encode()) + + def test_impossible_legacy_fixture_is_rejected(self): + forged = copy.deepcopy(self.right) + forged['metrics']['fantlab.general.mean_word_length_chars']['value'] = 100000 + with self.assertRaises(compare.CompareError): + compare.validate_report(forged) + + def test_real_analyzer_reports_pass(self): + compare.validate_report(self.left) + compare.validate_report(self.right) + self.assertEqual(len(compare.build_comparison(self.left, self.right)['metrics']), 29) + + def test_positive_word_mean_must_be_available(self): + self.assertGreater(self.left['metrics']['fantlab.general.words']['value'], 0) + self.left['metrics']['fantlab.general.mean_word_length_chars']['value'] = None + with self.assertRaises(compare.CompareError): + compare.validate_report(self.left) + + def test_impossible_word_length_is_rejected(self): + self.left['metrics']['fantlab.general.mean_word_length_chars']['value'] = 100000 + with self.assertRaises(compare.CompareError): + compare.validate_report(self.left) + + def test_nonempty_mean_cannot_be_zero(self): + for key in ('fantlab.general.mean_word_length_chars', 'fantlab.general.mean_sentence_length_chars'): + with self.subTest(key=key): + bad = copy.deepcopy(self.left) + bad['metrics'][key]['value'] = 0 + with self.assertRaises(compare.CompareError): + compare.validate_report(bad) + + def test_positive_sentence_mean_must_be_available(self): + self.assertGreater(self.left['metrics']['scriptorium.general.sentences']['value'], 0) + self.left['metrics']['fantlab.general.mean_sentence_length_chars']['value'] = None + with self.assertRaises(compare.CompareError): + compare.validate_report(self.left) + + def test_impossible_sentence_length_is_rejected(self): + self.left['metrics']['fantlab.general.mean_sentence_length_chars']['value'] = 100000 + with self.assertRaises(compare.CompareError): + compare.validate_report(self.left) + + def test_imported_json_rejects_forged_mean(self): + self.left['metrics']['fantlab.general.mean_word_length_chars']['value'] = 100000 + with tempfile.TemporaryDirectory() as td: + path = Path(td) / 'analysis.json' + path.write_text(json.dumps(self.left, ensure_ascii=False), encoding='utf-8') + with self.assertRaises(compare.CompareError): + compare.load_report(path) + + def test_zero_denominator_means_undefined(self): + empty = build_local_analysis(b'') + compare.validate_report(empty) + for name in ('fantlab.general.mean_word_length_chars', 'fantlab.general.mean_sentence_length_chars'): + bad = copy.deepcopy(empty) + bad['metrics'][name]['value'] = 0 + with self.assertRaises(compare.CompareError): + compare.validate_report(bad) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/private_compare_smoke.py b/tools/private_compare_smoke.py new file mode 100644 index 00000000..4f441bf9 --- /dev/null +++ b/tools/private_compare_smoke.py @@ -0,0 +1,77 @@ +"""Exercise an installed Reader outside its checkout, with real analyzer reports.""" +from __future__ import annotations + +import csv +from decimal import Decimal +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile + + +def run(*args: str, ok: bool = True) -> subprocess.CompletedProcess[str]: + process = subprocess.run([sys.executable, '-I', '-m', 'scriptorium', *args], + capture_output=True, text=True, timeout=30) + if ok and process.returncode: + raise AssertionError('installed Reader command failed: ' + process.stderr) + if not ok and not process.returncode: + raise AssertionError('invalid report was accepted') + return process + + +def main() -> None: + import scriptorium + from scriptorium.compare import build_comparison, load_report + + checkout = Path(sys.argv[1]).resolve() if len(sys.argv) == 2 else None + location = Path(scriptorium.__file__).resolve() + if checkout and location.is_relative_to(checkout): + raise AssertionError('smoke must import the installed wheel, not the checkout') + with tempfile.TemporaryDirectory() as td: + root = Path(td) + texts = ('Кот спит. Кот и пёс гуляют!\n— Да? — сказал пёс.', + 'Кот, кот, кот!\n— А? — ответил он.') + paths = [] + for index, text in enumerate(texts): + source, report = root / f'private-{index}.txt', root / f'analysis-{index}.json' + source.write_text(text, encoding='utf-8') + run('analyze', str(source), '--output', str(report)) + paths.append(report) + a, b = map(load_report, paths) + expected = build_comparison(a, b) + values = {} + for fmt in ('json', 'html', 'csv'): + output = root / f'comparison.{fmt}' + run('compare', *map(str, paths), '--format', fmt, '--output', str(output)) + payload = output.read_text(encoding='utf-8') + values[fmt] = payload + assert not any(text in payload for text in texts) + assert payload == run('compare', *map(str, paths), '--format', fmt).stdout + failure = run('compare', *map(str, paths), '--format', fmt, '--output', str(output), ok=False) + assert output.read_text(encoding='utf-8') == payload + assert str(root) not in failure.stderr + result = json.loads(values['json']) + assert result == expected and len(result['metrics']) == 29 + csv_rows = list(csv.DictReader(io.StringIO(values['csv']))) + assert len(csv_rows) == 29 and values['html'].count("scope='row'") == 29 + for row, exported in zip(result['metrics'], csv_rows): + assert row['metric_id'] == exported['metric_id'] + assert (row['difference'] or '') == exported['difference'] + if row['comparison_status'] == 'comparable': + assert Decimal(row['difference']) == Decimal(row['right']) - Decimal(row['left']) + assert 'unauthenticated declarations' in values['html'] + assert "Unit" in values['html'] + forged = json.loads(paths[0].read_text()) + forged['metrics']['fantlab.general.mean_word_length_chars']['value'] = 100000 + bad = root / 'private-invalid-report.json' + bad.write_text(json.dumps(forged)) + failed_output = root / 'should-not-exist.json' + failure = run('compare', str(bad), str(paths[1]), '--output', str(failed_output), ok=False) + assert not failed_output.exists() and str(root) not in failure.stderr + print('Installed private comparison PASS: real analyzer, 29 metrics, HTML/JSON/CSV, exact deltas, deterministic create-only exports and invalid-report rejection') + + +if __name__ == '__main__': + main() diff --git a/tools/reader_package_smoke.py b/tools/reader_package_smoke.py index 63915db1..0dd1ad1a 100644 --- a/tools/reader_package_smoke.py +++ b/tools/reader_package_smoke.py @@ -3,10 +3,12 @@ import hashlib import io import json +from importlib.metadata import version from pathlib import Path import subprocess import sys import tempfile +import tomllib def main(): @@ -17,9 +19,11 @@ def main(): for name in ("analyze_deterministic_metrics", "analyze_pos_metrics", "normalize_text", "word_tokens"): assert callable(getattr(scriptorium, name)), f"public API missing: {name}" def run(*args): - return subprocess.run([sys.executable, "-m", "scriptorium", *map(str, args)], + return subprocess.run([sys.executable, "-I", "-m", "scriptorium", *map(str, args)], check=True, capture_output=True, text=True, timeout=60) - assert "0.2.0a1" in run("--version").stdout + declared_version = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] + assert version("scriptorium-reader") == declared_version, "installed wheel disagrees with project version" + assert run("--version").stdout == f"Scriptorium Reader {declared_version}; inferred/extension preview, not FantLab parity\n" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) source = root / "private.txt"