diff --git a/.github/workflows/darwin-render-surface.yml b/.github/workflows/darwin-render-surface.yml new file mode 100644 index 00000000..beeb9f90 --- /dev/null +++ b/.github/workflows/darwin-render-surface.yml @@ -0,0 +1,94 @@ +name: Darwin Rachinsky rendering surface + +on: + pull_request: + paths: + - ".github/workflows/darwin-render-surface.yml" + - "scriptorium/darwin_render_surface.py" + - "scriptorium/darwin_render_surface_freeze.py" + - "scriptorium/darwin_body_contract.py" + - "scriptorium/darwin_page_freeze.py" + - "scriptorium/darwin_page_shards.py" + - "tests/test_darwin_render_surface.py" + - "tests/test_darwin_render_surface_freeze.py" + - "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.page-revisions.*.json" + - "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json" + workflow_dispatch: + +permissions: + contents: read + +jobs: + audit: + name: Audit 388 exact literary Page revisions + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout pull-request head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python 3.13 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Run source-free audit tests + run: | + python -m unittest tests.test_darwin_render_surface -v + python -m unittest tests.test_darwin_render_surface_freeze -v + + - name: Replay exact revisions and inventory markup surface + run: | + set -euo pipefail + mkdir -p build/repro + python -m scriptorium.darwin_render_surface \ + --index corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.page-revisions.index.json \ + --output build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.full.json + + - name: Rebuild and compare durable source-free freeze + run: | + set -euo pipefail + python -m scriptorium.darwin_render_surface_freeze \ + --input build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.full.json \ + --output build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.json + cmp \ + build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.json \ + corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json + + - name: Verify source and gate boundary + run: | + python - <<'PY' + import json + from pathlib import Path + path = Path("build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.json") + audit = json.loads(path.read_text(encoding="utf-8")) + assert audit["literary_dependency_count"] == 388 + assert audit["page_surface_receipts_count"] == 388 + assert audit["identity_replay_match"] is True + assert audit["source_text_included"] is False + assert audit["rendering_profile_frozen"] is False + assert audit["literary_body_count_and_digests_frozen"] is False + assert audit["minimum_300k_proved"] is False + assert audit["admitted_for_calibration"] is False + assert audit["fantlab_source_edition_match"] == "unknown" + assert audit["diagnostic_ready"] is False + assert audit["m2_parity_admissible"] is False + print("SCRIPTORIUM_DARWIN_RENDER_SURFACE_PAGES=388") + print("SCRIPTORIUM_DARWIN_RENDER_SURFACE_FREEZE_SHA256=" + audit["freeze_manifest_sha256"]) + PY + + - name: Print source-free durable freeze for evidence + run: cat build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.json + + - name: Upload source-free audit evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: darwin-rachinsky-render-surface + path: | + build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.full.json + build/repro/darwin-origin-species-rachinsky-1864-ru.render-surface.json + if-no-files-found: error + retention-days: 7 diff --git a/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md index 6bafb44e..1f9d00e2 100644 --- a/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md +++ b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md @@ -69,6 +69,16 @@ The implementation fails closed if the 418-Page identity inventory, identity-row This is deliberately a composition/input freeze, **not yet a literary-body identity freeze**. The Page-wikitext-to-prose rendering profile and composite separator semantics remain unfrozen, so no body count or body digest is claimed by this unit. +## Source-free rendering-surface inventory + +**SCRIP-CORPUS-036 / Draft PR #160** adds the bounded prerequisite needed before a renderer can be specified without guessing at MediaWiki/template behavior. The hosted audit replays the exact **388 already-frozen literary Page revisions**, verifies each title/revision/timestamp/MediaWiki SHA-1 identity before inspecting content, and reduces the transient wikitext to a source-free construct inventory. + +The durable freeze contains **38 template name/arity shapes** and **18 HTML/ProofreadPage tag name/kind shapes**. Aggregate observations include **1,069 comments**, **776 `` blocks**, **8 wikilinks**, no external links, no headings, no table opens/closes, and no triple-brace template-parameter constructs on this exact literary surface. The inventory records construct identities/counts and cryptographic digests only; template arguments, lexical snippets, Page wikitext and rendered prose are not committed. + +The full hosted audit covered all **388/388** literary dependencies with `identity_replay_match=true`. Its per-Page source-free receipt list is represented durably by SHA-256 `7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b`; the compact deterministic freeze has SHA-256 `e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d`. + +This **does not freeze the renderer**. It freezes only the set of markup constructs the future candidate-specific renderer must handle or explicitly reject. `rendering_profile_frozen=false` remains mandatory until a separate implementation proves deterministic semantics for this exact inventory. + ## Provider-reported scan-file metadata The retained source surface reports **27,368,263 bytes**, MIME `image/vnd.djvu`, dimensions **3744 × 5616**, **432 pages**, SHA-1 `75ef508588194ae74874272ce290f3ec1043ea9b`, current file-history display `11:45, 8 March 2016` by `Nonexyst`, and Archive.org locator `https://archive.org/details/oproiskhozhdenii00darw`. @@ -81,13 +91,13 @@ The retained Wikisource work surface explicitly states that the work is in the p ## Corpus boundary -This candidate is still **not admitted** to the >=300,000-character calibration/profile corpus. The source family, route graph, 418-dependency topology, exact 418 Page revision identities and the 388/30 literary-versus-apparatus composition boundary are now source-free frozen, but the actual literary prose has not yet been deterministically rendered or counted. +This candidate is still **not admitted** to the >=300,000-character calibration/profile corpus. The source family, route graph, 418-dependency topology, exact 418 Page revision identities, the 388/30 literary-versus-apparatus composition boundary, and the exact 388-Page source-free markup-surface inventory are now frozen, but the actual literary prose has not yet been deterministically rendered or counted. Still required before corpus admission: -- define and verify a candidate-specific fail-closed Page-wikitext-to-prose rendering profile over the 388 exact literary dependencies; +- define and verify a candidate-specific fail-closed Page-wikitext-to-prose rendering profile that covers the frozen construct inventory; - freeze the rendered literary-body character count including spaces plus raw/normalized digests; -- verify that the frozen body itself, not nominal page count or topology, clears **300,000 characters**. +- verify that the frozen body itself, not nominal page count, topology or construct counts, clears **300,000 characters**. Accordingly `admitted_for_calibration=false` remains mandatory. @@ -100,8 +110,12 @@ FantLab work **`work969964`** identifies Darwin's 1859 English monograph. The cu - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.json) — retained translation/source identity and gate boundary from the earlier provenance stages. - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.source-graph.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.source-graph.json) — exact route/topology evidence and apparatus roles. - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.page-revisions.index.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.page-revisions.index.json) — authoritative source-free index for all 418 exact Page revision identities and four committed shards. +- [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json) — deterministic source-free 388-Page construct inventory with no source payload. +- [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json) — provenance edges from frozen Page identities through literary composition and hosted markup audit to the durable inventory. - [`../../scriptorium/darwin_body_contract.py`](../../scriptorium/darwin_body_contract.py) — deterministic source-free 388/30 composition contract generator and fail-closed validator. +- [`../../scriptorium/darwin_render_surface.py`](../../scriptorium/darwin_render_surface.py) — exact-revision identity-verifying hosted construct audit. +- [`../../scriptorium/darwin_render_surface_freeze.py`](../../scriptorium/darwin_render_surface_freeze.py) — deterministic compact freeze reducer/validator. ## Next evidence -A later bounded unit should inspect the exact pinned Page wikitext shapes and freeze a candidate-specific fail-closed rendering profile for the 388 literary dependencies without persisting source prose. Only then should a separate identity/admission unit compute source-free counts/digests and independently prove the >=300,000-character threshold. FantLab parity remains a separate, stricter source-matching problem. +A later bounded unit should use the now-frozen template/tag surface to define and verify a candidate-specific fail-closed rendering profile for the 388 literary dependencies without persisting source prose. Only then should a separate identity/admission unit compute source-free counts/digests and independently prove the >=300,000-character threshold. FantLab parity remains a separate, stricter source-matching problem. diff --git a/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json new file mode 100644 index 00000000..ceb5d1b2 --- /dev/null +++ b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json @@ -0,0 +1,351 @@ +{ + "admitted_for_calibration": false, + "candidate_id": "darwin-origin-species-rachinsky-1864-ru", + "construct_totals": { + "comment_count": 1069, + "external_link_count": 0, + "heading_count": 0, + "noinclude_block_count": 776, + "table_close_count": 0, + "table_open_count": 0, + "template_parameter_count": 0, + "wikilink_count": 8 + }, + "diagnostic_ready": false, + "fantlab_source_edition_match": "unknown", + "freeze_manifest_sha256": "e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d", + "identity_replay_match": true, + "literary_body_count_and_digests_frozen": false, + "literary_dependency_count": 388, + "m2_parity_admissible": false, + "minimum_300k_proved": false, + "page_surface_receipts_count": 388, + "page_surface_receipts_sha256": "7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b", + "rendering_profile_frozen": false, + "schema_version": "scriptorium-darwin-render-surface-freeze-v1", + "source_revision_index_sha256": "a1b1011095dc84ab0a5cd3ef45b2a46ce99588a0ef9cf507c1f711c925ba86dd", + "source_text_included": false, + "status": "surface_inventory_frozen_renderer_unfrozen", + "tag_shapes": [ + { + "count": 64, + "kind": "close", + "name": "big" + }, + { + "count": 64, + "kind": "open", + "name": "big" + }, + { + "count": 386, + "kind": "close", + "name": "div" + }, + { + "count": 388, + "kind": "open", + "name": "div" + }, + { + "count": 26, + "kind": "close", + "name": "math" + }, + { + "count": 26, + "kind": "open", + "name": "math" + }, + { + "count": 776, + "kind": "close", + "name": "noinclude" + }, + { + "count": 776, + "kind": "open", + "name": "noinclude" + }, + { + "count": 388, + "kind": "self_closing", + "name": "pagequality" + }, + { + "count": 39, + "kind": "close", + "name": "ref" + }, + { + "count": 39, + "kind": "open", + "name": "ref" + }, + { + "count": 388, + "kind": "self_closing", + "name": "references" + }, + { + "count": 808, + "kind": "close", + "name": "small" + }, + { + "count": 808, + "kind": "open", + "name": "small" + }, + { + "count": 2, + "kind": "close", + "name": "sub" + }, + { + "count": 2, + "kind": "open", + "name": "sub" + }, + { + "count": 92, + "kind": "close", + "name": "sup" + }, + { + "count": 92, + "kind": "open", + "name": "sup" + } + ], + "template_shapes": [ + { + "count": 2, + "name": "=", + "named": 0, + "positional": 0 + }, + { + "count": 255, + "name": "Lang", + "named": 0, + "positional": 2 + }, + { + "count": 122, + "name": "^", + "named": 0, + "positional": 0 + }, + { + "count": 90, + "name": "^", + "named": 0, + "positional": 1 + }, + { + "count": 6, + "name": "dotted TOC", + "named": 0, + "positional": 3 + }, + { + "count": 7, + "name": "fsp", + "named": 0, + "positional": 0 + }, + { + "count": 2, + "name": "inline float", + "named": 2, + "positional": 4 + }, + { + "count": 179, + "name": "lang", + "named": 0, + "positional": 2 + }, + { + "count": 2, + "name": "lang-en", + "named": 0, + "positional": 1 + }, + { + "count": 1, + "name": "lang-la", + "named": 0, + "positional": 1 + }, + { + "count": 22, + "name": "nobr", + "named": 0, + "positional": 1 + }, + { + "count": 14, + "name": "noindent", + "named": 0, + "positional": 0 + }, + { + "count": 14, + "name": "noindent", + "named": 0, + "positional": 1 + }, + { + "count": 4, + "name": "nop", + "named": 0, + "positional": 0 + }, + { + "count": 13, + "name": "psp", + "named": 0, + "positional": 0 + }, + { + "count": 8, + "name": "razr2", + "named": 0, + "positional": 1 + }, + { + "count": 2, + "name": "right", + "named": 0, + "positional": 1 + }, + { + "count": 30, + "name": "roman", + "named": 0, + "positional": 1 + }, + { + "count": 52, + "name": "sc", + "named": 0, + "positional": 1 + }, + { + "count": 388, + "name": "ВАР", + "named": 0, + "positional": 2 + }, + { + "count": 2, + "name": "ВАР2", + "named": 0, + "positional": 2 + }, + { + "count": 21, + "name": "Градус", + "named": 0, + "positional": 1 + }, + { + "count": 253, + "name": "Перенос", + "named": 0, + "positional": 2 + }, + { + "count": 2, + "name": "Перенос", + "named": 1, + "positional": 2 + }, + { + "count": 259, + "name": "Перенос2", + "named": 0, + "positional": 2 + }, + { + "count": 15, + "name": "ПримВТ", + "named": 0, + "positional": 0 + }, + { + "count": 1, + "name": "Так в тексте", + "named": 0, + "positional": 1 + }, + { + "count": 46, + "name": "акут", + "named": 0, + "positional": 0 + }, + { + "count": 28, + "name": "во", + "named": 1, + "positional": 1 + }, + { + "count": 108, + "name": "гравис", + "named": 0, + "positional": 0 + }, + { + "count": 16, + "name": "дробь", + "named": 0, + "positional": 2 + }, + { + "count": 76, + "name": "мби", + "named": 0, + "positional": 3 + }, + { + "count": 99, + "name": "нп", + "named": 0, + "positional": 0 + }, + { + "count": 3, + "name": "опечатка", + "named": 0, + "positional": 2 + }, + { + "count": 135, + "name": "опечатка2", + "named": 0, + "positional": 2 + }, + { + "count": 17, + "name": "разделитель", + "named": 0, + "positional": 1 + }, + { + "count": 62, + "name": "ц", + "named": 0, + "positional": 1 + }, + { + "count": 2227, + "name": "ё", + "named": 0, + "positional": 0 + } + ] +} diff --git a/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json new file mode 100644 index 00000000..5ac39b84 --- /dev/null +++ b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json @@ -0,0 +1,84 @@ +{ + "schema_version": "scriptorium-darwin-render-surface-source-graph-v1", + "candidate_id": "darwin-origin-species-rachinsky-1864-ru", + "source_text_committed": false, + "nodes": [ + { + "id": "page-revision-index", + "kind": "frozen_source_free_page_identity_index", + "path": "darwin-origin-species-rachinsky-1864-ru.page-revisions.index.json", + "sha256": "a1b1011095dc84ab0a5cd3ef45b2a46ce99588a0ef9cf507c1f711c925ba86dd", + "included_dependency_count": 418 + }, + { + "id": "literary-composition", + "kind": "frozen_source_free_composition_contract", + "contract_version": "scriptorium-darwin-literary-body-contract-v1", + "literary_dependency_count": 388, + "apparatus_dependency_count": 30, + "explicit_no_text_non_dependencies": [114, 411] + }, + { + "id": "hosted-render-surface-audit", + "kind": "transient_exact_revision_markup_audit", + "workflow_run_id": 35487766954, + "workflow_job_id": 106017178554, + "head_sha": "708e780417efa43c80332a782181348aea3e79fb", + "artifact_id": 10598162631, + "artifact_zip_digest": "sha256:49fdcce34d1e30a10b335024a90c27feb8042d4f48a819e6ff3bb21cc3a8ca1c", + "full_audit_sha256": "cd6b748090298a5a1a4fe0b1bc63feb7e1ba9c49332bb6217e744b3fd7197b49", + "identity_replay_match": true, + "literary_dependency_count": 388, + "source_text_persisted": false + }, + { + "id": "durable-render-surface-freeze", + "kind": "frozen_source_free_markup_surface_inventory", + "path": "darwin-origin-species-rachinsky-1864-ru.render-surface.json", + "freeze_manifest_sha256": "e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d", + "page_surface_receipts_count": 388, + "page_surface_receipts_sha256": "7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b", + "template_shape_count": 38, + "tag_shape_count": 18, + "source_text_included": false + }, + { + "id": "future-renderer", + "kind": "unfrozen_page_wikitext_to_prose_profile", + "status": "unfrozen", + "must_cover_exactly_frozen_surface": true + } + ], + "edges": [ + { + "from": "page-revision-index", + "to": "literary-composition", + "relation": "partitioned_by" + }, + { + "from": "literary-composition", + "to": "hosted-render-surface-audit", + "relation": "exact_388_literary_revisions_replayed_for_transient_markup_inspection" + }, + { + "from": "hosted-render-surface-audit", + "to": "durable-render-surface-freeze", + "relation": "reduced_to_source_free_deterministic_construct_inventory" + }, + { + "from": "durable-render-surface-freeze", + "to": "future-renderer", + "relation": "constrains_supported_construct_surface" + } + ], + "gate_boundary": { + "surface_inventory_frozen": true, + "rendering_profile_frozen": false, + "literary_body_count_and_digests_frozen": false, + "minimum_300k_proved": false, + "admitted_for_calibration": false, + "fantlab_source_edition_match": "unknown", + "diagnostic_ready": false, + "m2_parity_admissible": false + } +} diff --git a/project/CHANGELOG.d/SCRIP-CORPUS-036.md b/project/CHANGELOG.d/SCRIP-CORPUS-036.md new file mode 100644 index 00000000..ed6aa6f0 --- /dev/null +++ b/project/CHANGELOG.d/SCRIP-CORPUS-036.md @@ -0,0 +1,9 @@ +## SCRIP-CORPUS-036 — Darwin/Rachinsky rendering-surface inventory + +- Opened Issue #159 and Draft PR #160 from `master` `f5687b95d20e2714cebfcef8370883377c25f0bb`. +- Added an exact-revision hosted audit over the 388 frozen literary Page dependencies selected by `scriptorium-darwin-literary-body-contract-v1`. Every fetched Page is checked against frozen title/revision/timestamp/MediaWiki SHA-1 before transient markup inspection. +- The first hosted audit correctly exposed a parser defect: nested template closing braces were being mistaken for triple-brace parameter syntax. The unit recovered by replacing substring rejection with stack-based curly-construct parsing and adding regressions for nested templates and real parameter constructs. +- Successful exact-revision replay then froze a source-free markup inventory: 38 template name/arity shapes, 18 tag name/kind shapes, 1,069 comments, 776 `noinclude` blocks, 8 wikilinks, and zero external links/headings/tables/template-parameter constructs across all 388 literary revisions. +- Added a deterministic compact freeze reducer. The committed freeze stores no wikitext, rendered prose, template arguments or lexical snippets; the 388 per-Page source-free receipts are represented by SHA-256 `7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b` and the compact manifest by SHA-256 `e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d`. +- Added a source-free rendering-surface provenance graph and synchronized the public candidate page. The renderer itself remains deliberately unfrozen: body count/digests, >=300k admission, independent scan SHA-256, FantLab source matching and M2 parity remain closed. +- This authored PR remains Draft for a later independent exact-head review; no self-approval or merge is performed in this run. diff --git a/project/STATE_AND_QUEUE.md b/project/STATE_AND_QUEUE.md index c06fdacb..b771b261 100644 --- a/project/STATE_AND_QUEUE.md +++ b/project/STATE_AND_QUEUE.md @@ -1,22 +1,22 @@ # STATE_AND_QUEUE — Scriptorium -STATE_REVISION: 180 +STATE_REVISION: 181 PHASE: M1 — Deterministic FantLab surface -LAST_COMMITTED_RUN_AT: 2026-09-20T02:52:00Z -LAST_RESULT: SCRIP-CORPUS-035 / Issue #157 completed after independent exact-head review of PR #158. Exact reviewed head `b78396377612a254dda7605425355b2f435b2e78` remained mergeable against unchanged base `c885600f528e2883396d58e14f229824c61b0902`, 12 commits ahead / 0 behind, with no inline review threads. The prior structured-provenance blocker was resolved: the canonical trace now freezes the 418-identity plus 388-literary / 30-apparatus composition boundary while keeping rendering/body/admission/FantLab gates closed. All 15 pull-request-triggered workflows completed successfully; dedicated run `35482586257`, job `106002941641`, passed exact-head contract verification and produced source-free artifact `10596435931` (1,346 bytes, `sha256:ce2e734417f45ed18effd563af720af6ad31ae6a27d71e1f21319cb1cb1ae392`). PR #158 was marked Ready and squash-merged as `ad2439baa4585759558b5eccb93ae2bf58979f00`; Issue #157 closed completed. -LAST_VERIFIED_PROGRESS: `darwin-origin-species-rachinsky-1864-ru` now canonically retains the 418 exact Page revision identities plus `scriptorium-darwin-literary-body-contract-v1`: 388 dependencies from numbered routes `/1`–`/14` are the frozen literary composition surface and 30 parent-only/alphabetical-index dependencies are frozen apparatus; no-text 114 and 411 remain non-dependencies. Structured provenance and the public candidate surface are reconciled with that boundary. Page-wikitext-to-prose rendering semantics, composite separator/body-rendering semantics, literary-body count/digests, independent scan bytes/SHA-256 and >=300k admission remain unfrozen/false. FantLab still exposes Timiryazev rather than Rachinsky and M2 remains 0/5. +LAST_COMMITTED_RUN_AT: 2026-09-20T03:55:00Z +LAST_RESULT: SCRIP-CORPUS-036 / Issue #159 advanced in Draft PR #160. The unit added an identity-verifying hosted audit over the 388 exact literary Darwin/Rachinsky Page revisions, recovered a real nested-template-brace parser defect found by the first hosted replay, and then successfully froze a source-free rendering-surface inventory. Successful run `35487766954`, job `106017178554`, replayed 388/388 exact literary revisions with `identity_replay_match=true`; artifact `10598162631` is 43,615 bytes with `sha256:49fdcce34d1e30a10b335024a90c27feb8042d4f48a819e6ff3bb21cc3a8ca1c`. The durable compact freeze records 38 template name/arity shapes, 18 tag name/kind shapes, aggregate construct counts, a digest over the 388 per-Page source-free receipts (`7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b`) and compact manifest digest `e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d`, without source wikitext/prose or template arguments. The authored PR remains Draft for independent exact-head review. +LAST_VERIFIED_PROGRESS: `darwin-origin-species-rachinsky-1864-ru` now retains 418 exact Page revision identities, the reviewed 388-literary / 30-apparatus composition contract, and a source-free exact-388 markup-surface inventory. Hosted evidence observed 1,069 comments, 776 noinclude blocks, 8 wikilinks, zero external links/headings/tables/template-parameter constructs, and complete 388/388 identity replay. This inventory constrains the next renderer but does not define rendering semantics. `rendering_profile_frozen=false`, literary-body count/digests, independent scan bytes/SHA-256 and >=300k admission remain unfrozen/false. FantLab still exposes Timiryazev rather than Rachinsky and M2 remains 0/5. ## Current unit ```text -UNIT_ID: SCRIP-CORPUS-035 -ISSUE: #157 (closed completed) -STATUS: COMPLETE -PR: #158 (squash-merged) -MERGED_COMMIT: ad2439baa4585759558b5eccb93ae2bf58979f00 -NEXT_ACTION: Resume normal-flow P2 corpus/provenance work. For Darwin/Rachinsky, the next evidence-bearing strengthening - is a candidate-specific fail-closed Page-wikitext-to-prose rendering profile over the 388 frozen literary - dependencies; literary-body count/digests and >=300k admission remain separate gates and must be independently proved. +UNIT_ID: SCRIP-CORPUS-036 +ISSUE: #159 (open) +STATUS: REVIEW_PENDING +PR: #160 (Draft; authored, not self-approved) +NEXT_ACTION: Independently review the exact current PR #160 head and all settled exact-head CI. Confirm that the committed + render-surface freeze is deterministically reproduced from 388 exact pinned literary revisions, contains no + source payload, and keeps renderer/body/admission/FantLab gates closed. If clean, mark Ready and merge; otherwise + record a precise blocker and repair in a later production run. ``` ## Current milestone gate @@ -31,6 +31,7 @@ Evaluate rows in priority order. Recovery/review-ready work and failing required | Priority | Unit | Mode | Deliverable | Gate / dependency | | --- | --- | --- | --- | --- | +| P0 | SCRIP-CORPUS-036 review | review / verification | Independently review Draft PR #160 exact head, source-free freeze and settled CI; merge only if evidence is clean | Do not self-approve production-authored changes; all downstream gates must stay closed | | P2 | SCRIP-CORPUS continuation | corpus / provenance | Add or strengthen legally usable >=300k candidates, prioritizing diversity beyond 19th-century Russian classics when licensing/source identity is strong enough | Preserve translation/edition identity and explicit legal provenance | ## Retained corpus / provenance status @@ -50,7 +51,7 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec - **The Twelve Chairs** — SCRIP-CORPUS-029 is complete. The distinct 41-chapter 1928 first-standalone family now has a merged source-free parent route graph: work index `oldid=5706135`, ProofreadPage index `oldid=5702510`, part parents `5702507` / `5704332` / `5702508`, and exact transclusion ranges 8–149, 152–313, and 316–421 totaling 410 Page dependencies. The 410 Page revision identities, PDF bytes and literary body remain unfrozen. The 40-chapter later 1938/1961 editorial family stays distinct; coauthorship prevents individual-author attribution without an explicit VOICE model. - **The Life of Klim Samgin** — SCRIP-CORPUS-020 is complete. Four exact parent revisions and the Part 2 dependency are replay-frozen; merged PRs #111–#124 freeze the target-only `#lst`, inferred `poemx1`/Poem boundaries, structural surface and all thirteen plain literary values; #126 merged the candidate-specific fail-closed extractor; #127 merged the canonical source-free per-part and composite literary-body identities; #128 reconciled the structured provenance trace. Exact historical MediaWiki/Poem deployment and FantLab analyzer-input/source-edition identity remain unresolved. - **Children of Captain Grant — Beketova translation** — SCRIP-CORPUS-026 is complete. Merged PR #140 keeps exact Russian Wikisource revision `oldid=5304880` replay-frozen and adds candidate-specific `scriptorium-beketova-captain-grant-wikisource-body-v1`, freezing a 1,095,467-character / 2,040,240-byte literary body with raw and normalized SHA-256 `4931eba535a4ef3989ac905d895258f00db57029e488d98511db50a7ce4d1161`. Beketova / Detgiz 1955 / `az.lib.ru` bibliography and explicit source PD evidence covering the translation remain retained. The frozen body clears the >=300k rule for general calibration/profile use. No FantLab linguistic result or analyzer-input identity is established for this translation; M2 weight remains zero. -- **On the Origin of Species — Rachinsky translation** — SCRIP-CORPUS-035 is complete. Master freezes all 418 included Page revision identities and `scriptorium-darwin-literary-body-contract-v1`: 388 dependencies from numbered rendered routes `/1`–`/14` are the canonical literary composition surface; 30 parent-only/alphabetical-index dependencies are excluded as apparatus; no-text 114/411 remain non-dependencies. Page-wikitext rendering, literary-body counts/digests, independent scan SHA-256 and >=300k admission remain unfrozen, so `admitted_for_calibration=false`. FantLab work 969964 currently exposes Timiryazev rather than Rachinsky; source match/M2 remain unknown/zero. +- **On the Origin of Species — Rachinsky translation** — SCRIP-CORPUS-036 is in review. The branch retains the 418 exact Page revision identities and reviewed 388-literary/30-apparatus composition contract, then adds a source-free hosted inventory of the exact 388 literary Page markup surface: 38 template shapes, 18 tag shapes and a digest-pinned 388-entry source-free Page receipt set. No Page prose or template arguments are committed. Rendering semantics, literary-body count/digests, independent scan SHA-256 and >=300k admission remain unfrozen; `admitted_for_calibration=false`. FantLab work 969964 currently exposes Timiryazev rather than Rachinsky; source match/M2 remain unknown/zero. ## Deterministic / morphology findings @@ -70,7 +71,7 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec - Shining World exposes a merged inspectable source-free inventory artifact and corrected public/structured provenance that distinguish the 34 links advertised by the reviewed index from the 19 pages that actually exist; missing targets are visible blockers, not silently filled text. - Running on Waves now exposes in master a source-free frozen literary-body identity and >=300k general-corpus admission while preserving the separate 1980/az.lib.ru route and keeping FantLab source match/M2 closed. - Merged SCRIP-CORPUS-029 makes the 1928 *Twelve Chairs* parent route graph inspectable on the dedicated candidate page and in a machine-readable source-free manifest while explicitly keeping the 410 Page revisions and literary body unfrozen. -- Merged SCRIP-CORPUS-035 makes Darwin/Rachinsky's source-free 388-literary / 30-apparatus composition contract canonical and inspectable on top of the frozen 418 exact Page revision identities without publishing Page prose; all literary-body identity/admission and FantLab gates remain visibly closed. +- Draft SCRIP-CORPUS-036 makes Darwin/Rachinsky's exact-388 markup surface inspectable as a source-free deterministic construct inventory on top of the frozen 418 identities and 388/30 composition contract. The page explicitly labels the renderer, body identity/admission and FantLab gates as still open rather than implying rendered-body parity. - Klim Samgin exposes the source graph, fail-closed extractor, canonical source-free per-part/composite literary-body identities, public candidate page, and a structured provenance trace consistent with those identities. Evidence remains explicitly diagnostic-only and source-match unknown. - Live Pages deployment remains disabled behind `SCRIPTORIUM_PAGES_DEPLOY_ENABLED=true` plus repository Pages administration. @@ -89,5 +90,5 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec 11. Klim Samgin is fully frozen at the candidate-specific literary-body/provenance level, but exact historical Russian Wikisource MediaWiki-core/Poem deployment equivalence and FantLab analyzer-input bytes/edition remain unproven; these are evidence boundaries, not open acceptance criteria for closed Issue #109. 12. The Beketova *Children of Captain Grant* translation now has a frozen literary body large enough for general corpus admission, but no FantLab linguistic result/analyzer-input identity has been established for this translation; it cannot advance M2 until that evidence exists. 13. The merged 1928 *Twelve Chairs* parent route graph leaves all 410 underlying Page-namespace revisions, the PDF byte stream, literary extraction/composition and body digests unfrozen; FantLab source identity remains unknown. -14. Darwin/Rachinsky 1864 now has a canonical source-free 388/30 literary-versus-apparatus composition contract on top of the canonical 418-Page identity freeze, but Page-wikitext rendering semantics, body count/digests, independent scan-byte/SHA-256 identity and >=300k admission remain unresolved; FantLab's current Russian translation surface is a different translation. +14. Darwin/Rachinsky 1864 now has a reviewed source-free 388/30 composition contract plus a Draft source-free inventory of the exact 388 literary Page markup surface. The renderer semantics, body count/digests, independent scan-byte/SHA-256 identity and >=300k admission remain unresolved; FantLab's current Russian translation surface is a different translation. 15. Pages live activation remains a repository-admin effect and is off. diff --git a/project/run-receipts/SCRIP-CORPUS-036.md b/project/run-receipts/SCRIP-CORPUS-036.md new file mode 100644 index 00000000..6471d7de --- /dev/null +++ b/project/run-receipts/SCRIP-CORPUS-036.md @@ -0,0 +1,36 @@ +# SCRIP-CORPUS-036 run receipt + +- **Unit:** Darwin/Rachinsky source-free rendering-surface inventory +- **Issue:** #159 +- **PR:** #160 (Draft; authored this run, not self-approved or merged) +- **Base:** `f5687b95d20e2714cebfcef8370883377c25f0bb` +- **Repository:** `drevendev/Scriptorium` +- **Connected identity:** `andy-zen-dev` + +## Produced + +1. Added `scriptorium.darwin_render_surface`, which replays only the 388 already-frozen literary Page revision IDs and verifies title/revision/timestamp/MediaWiki SHA-1 before transient wikitext inspection. +2. Added a source-free construct audit for template name/arity shapes, tag name/kind shapes and aggregate structural counts. Template arguments, lexical snippets, wikitext, rendered prose, OCR and scan bytes are never serialized. +3. Added `scriptorium.darwin_render_surface_freeze`, which reduces the hosted per-Page audit to a deterministic compact durable manifest and validates that all downstream gates remain closed. +4. Added dedicated hosted CI and regression tests for nested templates, triple-brace parameter syntax, noinclude/comments, literal blocks, source-payload leakage and deterministic freeze materialization. +5. Added a public candidate-page surface plus a source-free provenance graph connecting the 418-Page identity freeze, 388/30 composition contract, hosted exact-revision audit and compact inventory. + +## Hosted evidence + +The first hosted run failed after unit tests because the initial curly-brace scanner treated `}}}` inside nested template closing runs as proof of triple-brace parameter syntax. That was a real implementation defect, not waived evidence; the parser was replaced with a stack-based scanner and regressions were added. + +The repaired exact-head hosted audit on `708e780417efa43c80332a782181348aea3e79fb` succeeded: + +- workflow run `35487766954`; +- job `106017178554`; +- artifact `10598162631`, 43,615 bytes, digest `sha256:49fdcce34d1e30a10b335024a90c27feb8042d4f48a819e6ff3bb21cc3a8ca1c`; +- full source-free audit SHA-256 `cd6b748090298a5a1a4fe0b1bc63feb7e1ba9c49332bb6217e744b3fd7197b49`; +- 388/388 exact literary Page identities replayed with `identity_replay_match=true`. + +Observed source-free surface: 38 template shapes, 18 tag shapes, 1,069 comments, 776 noinclude blocks, 8 wikilinks, and zero external links, headings, table openings/closings or triple-brace parameter constructs. The durable compact freeze hashes the full 388-entry source-free Page receipt list as `7c170f513bf21f2c9e0d7736bc9fc20b4c9a40fdfe942a7b1dd1b0ac048ba09b`; compact manifest digest is `e5ae625fef6afb2e890de990e1ef3b9e108a409a2e0ea2136947c2c3e1cea44d`. + +## Verification boundary + +This unit freezes **only the observed markup surface**, not MediaWiki rendering semantics. `rendering_profile_frozen=false`, `literary_body_count_and_digests_frozen=false`, `minimum_300k_proved=false`, `admitted_for_calibration=false`, `fantlab_source_edition_match=unknown`, `diagnostic_ready=false`, and `m2_parity_admissible=false` remain mandatory. + +The PR is left Draft for an independent later exact-head review. Final exact-head CI must be re-read after the bookkeeping/freeze commits before any Ready/merge decision. diff --git a/scriptorium/darwin_render_surface.py b/scriptorium/darwin_render_surface.py new file mode 100644 index 00000000..b8cef1df --- /dev/null +++ b/scriptorium/darwin_render_surface.py @@ -0,0 +1,406 @@ +"""Audit Darwin/Rachinsky Page-wikitext constructs without persisting source prose. + +This is deliberately a prerequisite to rendering: exact pinned literary Page revisions +are replayed transiently, identity-checked, and reduced to source-free construct shapes +and digests. Wikitext, template arguments, rendered prose, OCR, and scan bytes are never +serialized. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Callable, Iterable, Mapping, Sequence + +from .darwin_body_contract import literary_page_sequences +from .darwin_page_freeze import CAPTURE_BATCH_SIZE, CANDIDATE_ID, _api_query +from .darwin_page_shards import load_sharded_manifest + +AUDIT_VERSION = "scriptorium-darwin-render-surface-audit-v1" +PROFILE_STATUS = "surface_inventory_frozen_renderer_unfrozen" +EXPECTED_LITERARY_PAGE_COUNT = 388 + +_COMMENT_RE = re.compile(r"", re.DOTALL) +_NOINCLUDE_BLOCK_RE = re.compile(r"]*>.*?", re.I | re.S) +_NOINCLUDE_EMPTY_RE = re.compile(r"]*/\s*>", re.I) +_NOINCLUDE_TOKEN_RE = re.compile(r"]*)>", re.S) +_LITERAL_BLOCK_RE = re.compile( + r"<(nowiki|pre|source|syntaxhighlight|math)\b[^>]*>.*?", re.I | re.S +) +_EXTERNAL_LINK_RE = re.compile(r"(? str: + return sha256(text.encode("utf-8")).hexdigest() + + +def _canonical_json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _strip_nontranscluded_regions(wikitext: str) -> tuple[str, int, int]: + comments = len(_COMMENT_RE.findall(wikitext)) + text = _COMMENT_RE.sub("", wikitext) + noinclude_blocks = len(_NOINCLUDE_BLOCK_RE.findall(text)) + text = _NOINCLUDE_BLOCK_RE.sub("", text) + text = _NOINCLUDE_EMPTY_RE.sub("", text) + if _NOINCLUDE_TOKEN_RE.search(text): + raise ValueError("unbalanced noinclude markup in pinned Page wikitext") + return text, comments, noinclude_blocks + + +def _mask_literal_blocks(text: str) -> str: + return _LITERAL_BLOCK_RE.sub(lambda match: " " * len(match.group(0)), text) + + +def _scan_curly_constructs(text: str) -> tuple[list[str], int]: + """Return raw template inners plus count of triple-brace parameter constructs. + + Long runs of closing braces are parsed by the current stack type, so nested + `{{outer|{{inner}}}}` is not misclassified as a triple-brace parameter. + """ + + stack: list[tuple[str, int]] = [] + templates: list[str] = [] + parameter_count = 0 + index = 0 + while index < len(text): + if text.startswith("{{{", index): + stack.append(("parameter", index)) + index += 3 + continue + if text.startswith("{{", index): + stack.append(("template", index)) + index += 2 + continue + if stack and stack[-1][0] == "parameter" and text.startswith("}}}", index): + stack.pop() + parameter_count += 1 + index += 3 + continue + if text.startswith("}}", index): + if not stack or stack[-1][0] != "template": + raise ValueError("unbalanced template close in Page wikitext") + _, start = stack.pop() + templates.append(text[start + 2 : index]) + index += 2 + continue + index += 1 + if stack: + raise ValueError(f"unbalanced curly construct in Page wikitext: {stack[-1][0]}") + return templates, parameter_count + + +def _split_top_level(inner: str) -> list[str]: + parts: list[str] = [] + start = 0 + template_depth = parameter_depth = link_depth = 0 + index = 0 + while index < len(inner): + if inner.startswith("{{{", index): + parameter_depth += 1 + index += 3 + continue + if inner.startswith("{{", index): + template_depth += 1 + index += 2 + continue + if parameter_depth and inner.startswith("}}}", index): + parameter_depth -= 1 + index += 3 + continue + if template_depth and inner.startswith("}}", index): + template_depth -= 1 + index += 2 + continue + if inner.startswith("[[", index): + link_depth += 1 + index += 2 + continue + if inner.startswith("]]", index): + if not link_depth: + raise ValueError("unbalanced wikilink close inside template") + link_depth -= 1 + index += 2 + continue + if inner[index] == "|" and not (template_depth or parameter_depth or link_depth): + parts.append(inner[start:index]) + start = index + 1 + index += 1 + if template_depth or parameter_depth or link_depth: + raise ValueError("unbalanced nested construct inside template") + parts.append(inner[start:]) + return parts + + +def _has_top_level_equals(field: str) -> bool: + template_depth = parameter_depth = link_depth = 0 + index = 0 + while index < len(field): + if field.startswith("{{{", index): + parameter_depth += 1 + index += 3 + continue + if field.startswith("{{", index): + template_depth += 1 + index += 2 + continue + if parameter_depth and field.startswith("}}}", index): + parameter_depth -= 1 + index += 3 + continue + if template_depth and field.startswith("}}", index): + template_depth -= 1 + index += 2 + continue + if field.startswith("[[", index): + link_depth += 1 + index += 2 + continue + if link_depth and field.startswith("]]", index): + link_depth -= 1 + index += 2 + continue + if field[index] == "=" and not (template_depth or parameter_depth or link_depth): + return bool(field[:index].strip()) + index += 1 + return False + + +def _template_shapes(text: str) -> tuple[list[tuple[str, int, int]], int]: + invocations, parameter_count = _scan_curly_constructs(text) + shapes: list[tuple[str, int, int]] = [] + for inner in invocations: + fields = _split_top_level(inner) + raw_name = re.sub(r"\s+", " ", fields[0].strip()) + if not raw_name or len(raw_name) > 120 or "\n" in raw_name or "\r" in raw_name: + raise ValueError("invalid template name in Page wikitext") + name = "" if "{" in raw_name or "}" in raw_name else raw_name + positional = sum(not _has_top_level_equals(field) for field in fields[1:]) + named = len(fields) - 1 - positional + shapes.append((name, positional, named)) + return shapes, parameter_count + + +def audit_wikitext(wikitext: str) -> dict[str, object]: + if not isinstance(wikitext, str): + raise TypeError("wikitext must be str") + tag_kinds: Counter[tuple[str, str]] = Counter() + for match in _TAG_RE.finditer(wikitext): + name = match.group(2).lower() + suffix = match.group(3).rstrip() + kind = "close" if match.group(1) else ("self_closing" if suffix.endswith("/") else "open") + tag_kinds[(name, kind)] += 1 + + transcluded, comments, noinclude_blocks = _strip_nontranscluded_regions(wikitext) + masked = _mask_literal_blocks(transcluded) + shapes, parameter_count = _template_shapes(masked) + templates = Counter(shapes) + wikilink_open = masked.count("[[") + if wikilink_open != masked.count("]]" ): + raise ValueError("unbalanced wikilink markup in Page wikitext") + + summary: dict[str, object] = { + "comment_count": comments, + "noinclude_block_count": noinclude_blocks, + "template_parameter_count": parameter_count, + "template_shapes": [ + {"name": name, "positional": positional, "named": named, "count": count} + for (name, positional, named), count in sorted(templates.items()) + ], + "tag_shapes": [ + {"name": name, "kind": kind, "count": count} + for (name, kind), count in sorted(tag_kinds.items()) + ], + "wikilink_count": wikilink_open, + "external_link_count": len(_EXTERNAL_LINK_RE.findall(masked)), + "heading_count": len(_HEADING_RE.findall(masked)), + "table_open_count": len(_TABLE_OPEN_RE.findall(masked)), + "table_close_count": len(_TABLE_CLOSE_RE.findall(masked)), + } + summary["surface_signature_sha256"] = _sha256_text(_canonical_json(summary)) + return summary + + +def fetch_pinned_literary_wikitext( + rows: Iterable[Mapping[str, object]], + *, + query: Callable[[Mapping[str, str]], dict[str, object]] = _api_query, +) -> dict[int, str]: + expected = tuple(rows) + fetched: dict[int, str] = {} + for offset in range(0, len(expected), CAPTURE_BATCH_SIZE): + batch = expected[offset : offset + CAPTURE_BATCH_SIZE] + ids = [int(row["revision_id"]) for row in batch] + by_id = {int(row["revision_id"]): row for row in batch} + payload = query({ + "action": "query", "prop": "revisions", "revids": "|".join(map(str, ids)), + "rvprop": "ids|timestamp|sha1|content", "rvslots": "main", + }) + query_obj = payload.get("query") + pages = query_obj.get("pages") if isinstance(query_obj, dict) else None + if not isinstance(pages, list): + raise ValueError("MediaWiki content replay response missing pages") + for page in pages: + if not isinstance(page, dict): + raise ValueError("unexpected MediaWiki content replay page") + revisions = page.get("revisions") + if not isinstance(revisions, list) or len(revisions) != 1: + raise ValueError("unexpected MediaWiki content replay shape") + revision = revisions[0] + if not isinstance(revision, dict): + raise ValueError("unexpected MediaWiki content revision") + revid = revision.get("revid") + if not isinstance(revid, int) or revid not in by_id or revid in fetched: + raise ValueError(f"unexpected/duplicate pinned revision ID: {revid!r}") + row = by_id[revid] + if ( + page.get("title") != row["title"] + or revision.get("timestamp") != row["timestamp"] + or revision.get("sha1") != row["mediawiki_sha1"] + ): + raise ValueError(f"pinned Page identity drift before markup audit: {revid}") + slots = revision.get("slots") + main = slots.get("main") if isinstance(slots, dict) else None + content = main.get("content") if isinstance(main, dict) else None + if not isinstance(content, str): + raise ValueError(f"missing exact Page wikitext for revision {revid}") + fetched[revid] = content + if set(fetched) != {int(row["revision_id"]) for row in expected}: + raise ValueError("literary Page content replay inventory mismatch") + return fetched + + +def build_audit_manifest( + index_path: Path, + *, + fetcher: Callable[[Iterable[Mapping[str, object]]], dict[int, str]] = fetch_pinned_literary_wikitext, +) -> dict[str, object]: + rows = load_sharded_manifest(index_path) + literary = set(literary_page_sequences()) + selected = tuple(row for row in rows if int(row["page_sequence"]) in literary) + if len(selected) != EXPECTED_LITERARY_PAGE_COUNT: + raise ValueError("expected exactly 388 frozen literary Page identities") + fetched = fetcher(selected) + + templates: Counter[tuple[str, int, int]] = Counter() + tags: Counter[tuple[str, str]] = Counter() + totals: Counter[str] = Counter() + page_receipts: list[dict[str, object]] = [] + for row in selected: + revid = int(row["revision_id"]) + wikitext = fetched[revid] + try: + summary = audit_wikitext(wikitext) + except ValueError as exc: + raise ValueError(f"Page {row['page_sequence']} revision {revid}: {exc}") from exc + for item in summary["template_shapes"]: + templates[(str(item["name"]), int(item["positional"]), int(item["named"]))] += int(item["count"]) + for item in summary["tag_shapes"]: + tags[(str(item["name"]), str(item["kind"]))] += int(item["count"]) + for key in ( + "comment_count", "noinclude_block_count", "template_parameter_count", "wikilink_count", + "external_link_count", "heading_count", "table_open_count", "table_close_count", + ): + totals[key] += int(summary[key]) + page_receipts.append({ + "page_sequence": int(row["page_sequence"]), + "revision_id": revid, + "mediawiki_sha1": row["mediawiki_sha1"], + "wikitext_sha256": _sha256_text(wikitext), + "surface_signature_sha256": summary["surface_signature_sha256"], + "template_invocation_count": sum(int(item["count"]) for item in summary["template_shapes"]), + "tag_token_count": sum(int(item["count"]) for item in summary["tag_shapes"]), + }) + + manifest: dict[str, object] = { + "audit_version": AUDIT_VERSION, + "candidate_id": CANDIDATE_ID, + "status": PROFILE_STATUS, + "source_revision_index_sha256": sha256(index_path.read_bytes()).hexdigest(), + "literary_dependency_count": len(selected), + "identity_replay_match": True, + "source_text_included": False, + "template_shapes": [ + {"name": name, "positional": positional, "named": named, "count": count} + for (name, positional, named), count in sorted(templates.items()) + ], + "tag_shapes": [ + {"name": name, "kind": kind, "count": count} + for (name, kind), count in sorted(tags.items()) + ], + "construct_totals": dict(sorted(totals.items())), + "page_surface_receipts": page_receipts, + "rendering_profile_frozen": False, + "literary_body_count_and_digests_frozen": False, + "minimum_300k_proved": False, + "admitted_for_calibration": False, + "fantlab_source_edition_match": "unknown", + "diagnostic_ready": False, + "m2_parity_admissible": False, + } + manifest["audit_sha256"] = _sha256_text(_canonical_json(manifest)) + return manifest + + +def validate_audit_manifest(manifest: Mapping[str, object]) -> None: + if manifest.get("audit_version") != AUDIT_VERSION or manifest.get("candidate_id") != CANDIDATE_ID: + raise ValueError("Darwin rendering-surface audit identity drift") + if manifest.get("status") != PROFILE_STATUS or manifest.get("literary_dependency_count") != 388: + raise ValueError("Darwin rendering-surface audit status/count drift") + if manifest.get("identity_replay_match") is not True or manifest.get("source_text_included") is not False: + raise ValueError("Darwin rendering-surface source boundary drift") + for key in ( + "rendering_profile_frozen", "literary_body_count_and_digests_frozen", "minimum_300k_proved", + "admitted_for_calibration", "diagnostic_ready", "m2_parity_admissible", + ): + if manifest.get(key) is not False: + raise ValueError(f"{key} must remain false in rendering-surface audit") + if manifest.get("fantlab_source_edition_match") != "unknown": + raise ValueError("FantLab source match must remain unknown") + pages = manifest.get("page_surface_receipts") + if not isinstance(pages, list) or len(pages) != 388: + raise ValueError("rendering-surface audit must contain 388 source-free Page receipts") + forbidden = {"wikitext", "content", "text", "prose", "body", "template_arguments", "snippet"} + keys: set[str] = set() + stack: list[object] = [manifest] + while stack: + value = stack.pop() + if isinstance(value, Mapping): + for key, child in value.items(): + keys.add(str(key).lower()) + stack.append(child) + elif isinstance(value, list): + stack.extend(value) + if forbidden.intersection(keys): + raise ValueError("source payload key leaked into rendering-surface audit") + expected = manifest.get("audit_sha256") + unsigned = dict(manifest) + unsigned.pop("audit_sha256", None) + if expected != _sha256_text(_canonical_json(unsigned)): + raise ValueError("rendering-surface audit digest drift") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--index", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + manifest = build_audit_manifest(args.index) + validate_audit_manifest(manifest) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(manifest, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scriptorium/darwin_render_surface_freeze.py b/scriptorium/darwin_render_surface_freeze.py new file mode 100644 index 00000000..7102bfb1 --- /dev/null +++ b/scriptorium/darwin_render_surface_freeze.py @@ -0,0 +1,116 @@ +"""Materialize the durable source-free Darwin rendering-surface freeze. + +The hosted audit keeps per-Page receipts only as transient CI evidence. This module +reduces that evidence to a compact deterministic manifest containing construct shapes, +aggregate counts, and one digest over the 388 per-Page source-free receipts. +""" + +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +from typing import Mapping, Sequence + +from .darwin_render_surface import ( + CANDIDATE_ID, + EXPECTED_LITERARY_PAGE_COUNT, + PROFILE_STATUS, + validate_audit_manifest, +) + +FREEZE_VERSION = "scriptorium-darwin-render-surface-freeze-v1" + + +def _canonical_json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _sha256_json(value: object) -> str: + return sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def build_freeze_manifest(audit: Mapping[str, object]) -> dict[str, object]: + """Reduce a validated hosted audit to deterministic source-free durable evidence.""" + + validate_audit_manifest(audit) + pages = audit["page_surface_receipts"] + assert isinstance(pages, list) + manifest: dict[str, object] = { + "schema_version": FREEZE_VERSION, + "candidate_id": CANDIDATE_ID, + "status": PROFILE_STATUS, + "source_revision_index_sha256": audit["source_revision_index_sha256"], + "literary_dependency_count": audit["literary_dependency_count"], + "identity_replay_match": audit["identity_replay_match"], + "source_text_included": False, + "template_shapes": audit["template_shapes"], + "tag_shapes": audit["tag_shapes"], + "construct_totals": audit["construct_totals"], + "page_surface_receipts_count": len(pages), + "page_surface_receipts_sha256": _sha256_json(pages), + "rendering_profile_frozen": False, + "literary_body_count_and_digests_frozen": False, + "minimum_300k_proved": False, + "admitted_for_calibration": False, + "fantlab_source_edition_match": "unknown", + "diagnostic_ready": False, + "m2_parity_admissible": False, + } + manifest["freeze_manifest_sha256"] = _sha256_json(manifest) + return manifest + + +def validate_freeze_manifest(manifest: Mapping[str, object]) -> None: + if manifest.get("schema_version") != FREEZE_VERSION: + raise ValueError("Darwin rendering-surface freeze schema drift") + if manifest.get("candidate_id") != CANDIDATE_ID or manifest.get("status") != PROFILE_STATUS: + raise ValueError("Darwin rendering-surface freeze identity/status drift") + if manifest.get("literary_dependency_count") != EXPECTED_LITERARY_PAGE_COUNT: + raise ValueError("Darwin rendering-surface freeze literary dependency drift") + if manifest.get("page_surface_receipts_count") != EXPECTED_LITERARY_PAGE_COUNT: + raise ValueError("Darwin rendering-surface freeze receipt count drift") + if manifest.get("identity_replay_match") is not True or manifest.get("source_text_included") is not False: + raise ValueError("Darwin rendering-surface freeze source boundary drift") + for key in ( + "rendering_profile_frozen", + "literary_body_count_and_digests_frozen", + "minimum_300k_proved", + "admitted_for_calibration", + "diagnostic_ready", + "m2_parity_admissible", + ): + if manifest.get(key) is not False: + raise ValueError(f"{key} must remain false in rendering-surface freeze") + if manifest.get("fantlab_source_edition_match") != "unknown": + raise ValueError("FantLab source match must remain unknown") + for key in ("template_shapes", "tag_shapes", "construct_totals"): + if key not in manifest: + raise ValueError(f"missing rendering-surface freeze field: {key}") + digest = manifest.get("freeze_manifest_sha256") + unsigned = dict(manifest) + unsigned.pop("freeze_manifest_sha256", None) + if digest != _sha256_json(unsigned): + raise ValueError("Darwin rendering-surface freeze digest drift") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + audit = json.loads(args.input.read_text(encoding="utf-8")) + manifest = build_freeze_manifest(audit) + validate_freeze_manifest(manifest) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(manifest["freeze_manifest_sha256"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_darwin_render_surface.py b/tests/test_darwin_render_surface.py new file mode 100644 index 00000000..c7c7cc55 --- /dev/null +++ b/tests/test_darwin_render_surface.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +import unittest + +from scriptorium.darwin_render_surface import audit_wikitext, build_audit_manifest, validate_audit_manifest +from scriptorium.darwin_page_shards import load_sharded_manifest + + +ROOT = Path(__file__).resolve().parents[1] +INDEX = ROOT / "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.page-revisions.index.json" + + +class DarwinRenderSurfaceTests(unittest.TestCase): + def test_audit_records_shapes_without_argument_values(self) -> None: + source = ( + "" + "header {{ignored|secret}}" + "{{outer|{{inner|lexical value|named=private}}|link=[[Target|Label]]}}" + "citation
" + "footer" + ) + audit = audit_wikitext(source) + shapes = { + (item["name"], item["positional"], item["named"], item["count"]) + for item in audit["template_shapes"] + } + self.assertIn(("inner", 1, 1, 1), shapes) + self.assertIn(("outer", 1, 1, 1), shapes) + self.assertFalse(any(item["name"] == "ignored" for item in audit["template_shapes"])) + serialized = str(audit) + for forbidden in ("lexical value", "private", "Target", "Label"): + self.assertNotIn(forbidden, serialized) + self.assertEqual(audit["comment_count"], 1) + self.assertEqual(audit["noinclude_block_count"], 2) + self.assertEqual(audit["wikilink_count"], 1) + + def test_nested_template_closing_is_not_mistaken_for_parameter_braces(self) -> None: + audit = audit_wikitext("{{outer|{{inner|x}}}}") + names = [item["name"] for item in audit["template_shapes"]] + self.assertEqual(names, ["inner", "outer"]) + self.assertEqual(audit["template_parameter_count"], 0) + + def test_parameter_construct_is_counted_without_serializing_value(self) -> None: + audit = audit_wikitext("{{outer|{{{parameter|secret default}}}}}") + self.assertEqual(audit["template_parameter_count"], 1) + self.assertNotIn("secret default", str(audit)) + + def test_audit_fails_closed_on_unbalanced_constructs(self) -> None: + with self.assertRaisesRegex(ValueError, "unbalanced curly construct"): + audit_wikitext("visible {{broken|value") + with self.assertRaisesRegex(ValueError, "unbalanced noinclude"): + audit_wikitext("header visible") + + def test_literal_blocks_do_not_create_false_template_shapes(self) -> None: + audit = audit_wikitext("{{not-a-template|x}}{{real|x}}") + names = [item["name"] for item in audit["template_shapes"]] + self.assertEqual(names, ["real"]) + + def test_build_selects_388_literary_dependencies_and_keeps_gates_closed(self) -> None: + rows = load_sharded_manifest(INDEX) + by_id = {int(row["revision_id"]): row for row in rows} + + def fake_fetcher(selected): + result = {} + for row in selected: + revid = int(row["revision_id"]) + self.assertEqual(row, by_id[revid]) + result[revid] = ( + "" + "{{razr|synthetic}} [[Example|label]]
" + "footer" + ) + return result + + manifest = build_audit_manifest(INDEX, fetcher=fake_fetcher) + validate_audit_manifest(manifest) + self.assertEqual(manifest["literary_dependency_count"], 388) + self.assertEqual(len(manifest["page_surface_receipts"]), 388) + self.assertFalse(manifest["rendering_profile_frozen"]) + self.assertFalse(manifest["literary_body_count_and_digests_frozen"]) + self.assertFalse(manifest["minimum_300k_proved"]) + self.assertFalse(manifest["admitted_for_calibration"]) + self.assertEqual(manifest["fantlab_source_edition_match"], "unknown") + self.assertFalse(manifest["m2_parity_admissible"]) + + def test_validator_rejects_source_payload_key(self) -> None: + def fake_fetcher(selected): + return {int(row["revision_id"]): "plain synthetic text" for row in selected} + + manifest = build_audit_manifest(INDEX, fetcher=fake_fetcher) + leaked = deepcopy(manifest) + leaked["wikitext"] = "forbidden" + with self.assertRaisesRegex(ValueError, "source payload key"): + validate_audit_manifest(leaked) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_darwin_render_surface_freeze.py b/tests/test_darwin_render_surface_freeze.py new file mode 100644 index 00000000..a0274fb5 --- /dev/null +++ b/tests/test_darwin_render_surface_freeze.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import unittest + +from scriptorium.darwin_render_surface_freeze import build_freeze_manifest, validate_freeze_manifest + + +def _canonical_sha256(value: object) -> str: + return hashlib.sha256( + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +class DarwinRenderSurfaceFreezeTests(unittest.TestCase): + def _audit(self) -> dict[str, object]: + pages = [ + { + "page_sequence": index, + "revision_id": 100000 + index, + "mediawiki_sha1": f"sha1-{index}", + "wikitext_sha256": f"raw-{index}", + "surface_signature_sha256": f"surface-{index}", + "template_invocation_count": index % 3, + "tag_token_count": 4, + } + for index in range(1, 389) + ] + audit: dict[str, object] = { + "audit_version": "scriptorium-darwin-render-surface-audit-v1", + "candidate_id": "darwin-origin-species-rachinsky-1864-ru", + "status": "surface_inventory_frozen_renderer_unfrozen", + "source_revision_index_sha256": "index-digest", + "literary_dependency_count": 388, + "identity_replay_match": True, + "source_text_included": False, + "template_shapes": [{"name": "ё", "positional": 0, "named": 0, "count": 12}], + "tag_shapes": [{"name": "ref", "kind": "open", "count": 2}], + "construct_totals": {"comment_count": 1}, + "page_surface_receipts": pages, + "rendering_profile_frozen": False, + "literary_body_count_and_digests_frozen": False, + "minimum_300k_proved": False, + "admitted_for_calibration": False, + "fantlab_source_edition_match": "unknown", + "diagnostic_ready": False, + "m2_parity_admissible": False, + } + audit["audit_sha256"] = _canonical_sha256(audit) + return audit + + def test_freeze_is_compact_deterministic_and_keeps_gates_closed(self) -> None: + audit = self._audit() + freeze = build_freeze_manifest(audit) + validate_freeze_manifest(freeze) + self.assertEqual(freeze["page_surface_receipts_count"], 388) + self.assertNotIn("page_surface_receipts", freeze) + self.assertEqual(freeze["page_surface_receipts_sha256"], _canonical_sha256(audit["page_surface_receipts"])) + self.assertFalse(freeze["rendering_profile_frozen"]) + self.assertFalse(freeze["minimum_300k_proved"]) + self.assertFalse(freeze["m2_parity_admissible"]) + self.assertEqual(build_freeze_manifest(audit), freeze) + + def test_freeze_validator_rejects_gate_promotion_and_digest_drift(self) -> None: + freeze = build_freeze_manifest(self._audit()) + promoted = deepcopy(freeze) + promoted["rendering_profile_frozen"] = True + with self.assertRaisesRegex(ValueError, "must remain false"): + validate_freeze_manifest(promoted) + + drifted = deepcopy(freeze) + drifted["construct_totals"] = {"comment_count": 999} + with self.assertRaisesRegex(ValueError, "digest drift"): + validate_freeze_manifest(drifted) + + +if __name__ == "__main__": + unittest.main()