diff --git a/.claude/skills/curate-yaml-record/SKILL.md b/.claude/skills/curate-yaml-record/SKILL.md index 5c47db092..938621b68 100644 --- a/.claude/skills/curate-yaml-record/SKILL.md +++ b/.claude/skills/curate-yaml-record/SKILL.md @@ -6,21 +6,181 @@ metadata: category: curation requires_database: false requires_internet: true - version: 1.0.0 + version: 2.0.0 --- # Curate one CommunityMech YAML record +## Structured Review Output + +For every new review or audit, follow +[docs/record-reviews.md](../../../docs/record-reviews.md) and +[the local profile](../../../docs/record-review-profile.md). +Capture exact targets and input hashes before judging, preserve this skill's +native rubric, rule IDs, scores and evidence requirements, then author the +structured assessment and run: + +```bash +uv run python scripts/record_review.py inspect --targets /tmp/review-targets.yaml +uv run python scripts/record_review.py validate /tmp/completed-review.yaml +uv run python scripts/record_review.py save --content /tmp/completed-review.yaml +``` + +Choose session-unique temporary paths. Save authoritative YAML and derived +Markdown under `reviews/structured/-/`; link both in the +final response. This output contract supersedes prose-only report examples. +A single record uses `kind: record`; batches declare exact selection, +population, reviewed targets and limits. Categories also state boundary decisions. +Every reviewed target must have an assessment. Keep P1-P4 and other native +severity/rule information with a justified common severity, and metric definitions, +scales and denominators. Do not infer scientific approval from a native score. + +Raw provider drafts and deterministic validator/scan reports are diagnostic +inputs, not completed scientific reviews. Use `scientific_review: false` +for deterministic-only or provenance-only assessments; mark required unavailable +checks and incomplete coverage explicitly. A valid bundle does not change native +status, clear release holds, authorize edits, or append curation history. +For audit-only requests, stop after assessment and persistence; any application +steps below require curation intent. + Produce a defensible community or taxon record and an explicit account of what is supported, corrected, missing, and genuinely unknown. Search results and raw research reports are leads; only inspected sources can support a claim. +## Shared Contract + + +Produce a defensible record and an explicit account of four things: what is +**supported**, what was **corrected**, what is **still unresolved**, and what is +**genuinely unknown**. The last two are different — a gap you searched for and +could not close is a finding; a gap you did not look at is not. + +**One target.** Resolve exactly one record before touching anything. If a label +matches several, or a request names a family rather than a member, stop and +disambiguate. Silently substituting a similar record is the error that no later +check catches, because everything downstream is then correct about the wrong +thing. + +**Audit preserves scientific inputs. Curation authorises edits to the named +record only.** A review or audit request changes no scientific record, status, +or curation history. It does save a new timestamped structured review through +`docs/record-reviews.md` and the native rubric in `docs/record-review-profile.md`. +A curate, improve, complete, correct or add-evidence request authorises local edits to that record and the smallest +maintained path its provenance requires — not to neighbours, not to whatever +else looked wrong on the way. + +**Search results are leads. Only an inspected source supports a claim.** A +search hit, a deep-research report, a rendered page, and a generated artifact are +each somewhere to look, and none is evidence. Evidence is text you read in the +source, attached to the narrowest assertion it actually supports. + + +## Shared Boundaries + + +- **A generated artifact is never the fix.** Pages, merged products, exports and + derived indexes are outputs. Correct the input or rule that owns the value and + regenerate; patching the output makes it look right once and diverge on the + next build. +- **No outbound action without explicit authorisation for that action.** Do not + launch paid research, contact an author, or create or edit a GitHub issue, PR + or comment because curation seemed to call for it. Authorisation to curate a + record is not authorisation to spend or to speak. +- **Absence is not evidence of falsity, and coverage is not a goal.** Never + infer that an unstated optional property is false. Never fill an optional + slot to make the record look more complete. An empty field the source does + not address is correct. +- **Search before declaring anything absent — and search past `.gitignore`.** + Before treating a record, evidence source, decision row or overlay as missing, + search for its identifier, label and slug with an ignore-independent tool + (`rg --no-ignore --hidden`, `grep -r`, or `find`). Ordinary search skips + ignored files, so an ordinary miss is a search over a subset, not a result. +- **Preserve unrelated work.** Use a branch, and a separate worktree when the + checkout is dirty or occupied by something else. + + +## Shared Evidence Standard + + +- Each claim is its own object. A definition, an example, a relation and a + mechanism edge are separate assertions; attach a source to the narrowest one + it supports, never to the record as a whole. +- Resolve every DOI, PMID and CURIE, and read enough of the source to establish + support for the *exact* claim and scope. A matching string from the wrong + paper, or an unrelated sentence from the right one, is not support. +- A snippet is short verbatim text from the source. Interpretation belongs in a + notes field, never inside the quotation. +- **The kind of source is part of the citation.** A database assertion, a + primary experiment, a review, a prediction and a search snippet are different + strengths of support. Cite each as what it is; never present a database row + or a review as if it were the primary study. +- Association and prediction do not establish mechanism or causality. Do not + let a co-occurrence or a computed score become a mechanism edge. +- **A near-miss is not a match.** Never ground to a CURIE or canonical label + because it looks plausible, and never use a broader or related term as an + exact identity. Unresolved stays unresolved, recorded as such, until a source + resolves it. +- **Evidence about one thing supports a claim about that thing.** Do not + generalise one organism, strain, protein instance, construct or experiment + into a family-wide, universal or "optimal" claim. Scope inflation is the most + common way a true observation becomes a false record. +- Keep conflicts. When sources disagree, record both and the disagreement; do + not resolve it by omission. +- A bounded search that found nothing is a result. Report it as "not found, + searched X" rather than leaving the field silently empty. + + +## Shared Write Contract + + +**Write only through the path that preserves formatting and records history.** +Never hand-edit a curated YAML with a text editor or a generic dump: canonical +key order, quoting and derived metadata are what make the corpus diffable, and a +dump destroys them in one save. + +Repositories in this fleet do this in three different, equally correct ways, and +which one applies is a property of the corpus: + +- **Direct guarded write** — a narrowly scoped mutator loads the record, asserts + its identity, changes only the reviewed nodes, appends a curation event, and + writes through the repository's validated writer. +- **Registered editor** — no generic writer exists on purpose; in-place changes + use text-preserving operations through an editor that is registered and + behaviourally tested, and the writer audit rejects anything else. +- **Regenerate from inputs** — the record is a build product. The fix goes into + the decision row, term request, overlay or source inventory that owns the + value, and the record is regenerated; the YAML is never edited directly. + +The section below says which one this repository uses and names the exact +functions or files. Do not guess from a sibling. + +Inspect the diff before committing. Whole-file presentation churn — reordered +keys, requoted strings, a hundred lines changed to alter one value — means the +write path was bypassed; abandon and repair rather than commit it. + + +## Shared History And Attribution + + +- Use `curator="claude"` when no curator identity was supplied. **Never + attribute an agent's judgement to the user.** The history entry is a record + of who decided, and it will be read when the decision is questioned. +- Mark LLM assistance where the schema records it. +- **Do not append a history event when nothing substantive changed.** A + no-op event is noise that makes real events harder to find. +- The history entry describes the actual diff. If the corpus derives status or + history from inputs, never set them directly — change the input. +- A REVIEWED status means a human reviewed it. Do not invent one, and do not + promote to it on the strength of an agent pass. + + ## Boundaries - Resolve one target under `kb/communities/`, `data/isolates/`, or `kb/taxa/`. If a name matches several communities, experiments, or taxon records, stop and disambiguate before editing. -- An audit/review request is read-only. Curate, improve, complete, correct, or +- Audit/review preserves scientific inputs and saves a new structured review. + Curate, improve, complete, correct, or add-evidence requests authorize local edits to the named record and the smallest necessary history/generated-product paths. - Do not generalize a strain pair, enrichment, synthetic consortium, or diff --git a/.claude/skills/evidence-curation/SKILL.md b/.claude/skills/evidence-curation/SKILL.md index 94ee9ca91..dca79dbf5 100644 --- a/.claude/skills/evidence-curation/SKILL.md +++ b/.claude/skills/evidence-curation/SKILL.md @@ -10,6 +10,37 @@ tags: [evidence, snippets, literature, pmid, doi, references, repair, curation] # Evidence Curation Skill +## Assessed Review Handoff + +For an evidence review or audit, delegate the final assessment to +[review-communities](../review-communities/SKILL.md), following +[docs/record-reviews.md](../../../docs/record-reviews.md) and +[the local profile](../../../docs/record-review-profile.md). +Use the registered review-communities route as the saved `skill`; preserve +per-record P1-P4 findings, both native scores, and assertion-level evidence. +For each interaction or causal edge keep participants, direction, mechanism, +experimental context, exact source locator and inspected verbatim snippet. +Distinguish abstract, full text and supplementary evidence. + +Extraction, network checks and repair suggestions are diagnostic inputs, not +completed scientific review or snippet approval. A deterministic-only +assessment uses `scientific_review: false`. No producer draft may be promoted +without actual source assessment. Missing evidence remains a scoped gap. +Save final assessed YAML plus derived Markdown under +`reviews/structured/-/` and link both: + +```bash +uv run python scripts/record_review.py inspect --targets /tmp/evidence-targets.yaml +uv run python scripts/record_review.py validate /tmp/completed-evidence-review.yaml +uv run python scripts/record_review.py save --content /tmp/completed-evidence-review.yaml +``` + +Use session-unique temporary paths. Audit-only requests run no repair/apply +commands and change no scientific records, status or history. The repair +workflows below require explicit curation intent and the native guarded writer. +Paid/LLM research or repair additionally requires explicit authorization; +the assessment handoff does not invoke or recreate a research producer. + ## Overview CommunityMech community records require evidence-backed claims: each interaction, @@ -22,7 +53,7 @@ This skill covers the full evidence lifecycle: - **Review** — manual literature review and snippet approval - **Validate** — confirm evidence passes schema and reference checks -**Run from `CommunityMech/CommunityMech/` directory.** +**Run from the repository root.** --- @@ -65,7 +96,7 @@ python scripts/apply_pmc_conversions.py # 5. Normalize reference formats (PMID: prefix, DOI capitalization) python scripts/fix_reference_formats.py -# 6. Run review-communities to check remaining issues +# 6. Network diagnostics only; then hand off to review-communities above communitymech audit-network ``` @@ -77,13 +108,8 @@ communitymech audit-network # Extract snippets from a PDF python scripts/curate_evidence_with_pdfs.py --pdf path/to/paper.pdf -# Quick literature review for a specific community -python scripts/quick_literature_review.py --community CommunityMech:000042 - -# Full literature review with scoring -python scripts/review_literature.py --community CommunityMech:000042 - -# Apply suggested fixes after review +# Assess extracted evidence through review-communities and the common saver. +# Apply suggested fixes only after separately authorized curation review. python scripts/apply_suggested_snippets.py python scripts/apply_suggested_fixes.py ``` @@ -102,8 +128,6 @@ python scripts/apply_suggested_fixes.py | `scripts/apply_pmc_conversions.py` | PMC ID → PMID conversion | | `scripts/handle_special_references.py` | Edge cases (preprints, books, datasets) | | `scripts/curate_evidence_with_pdfs.py` | PDF-backed snippet extraction | -| `scripts/quick_literature_review.py` | Fast per-community literature scan | -| `scripts/review_literature.py` | Full literature review with scoring | | `scripts/analyze_literature_report.py` | Analyze review results | | `scripts/analyze_review_cases.py` | Categorize review case types | | `scripts/apply_suggested_snippets.py` | Apply auto-suggested snippet text | @@ -141,11 +165,13 @@ All API calls respect rate limits; no API key required for PubMed. Always run validation after evidence changes: ```bash -# Full schema + reference validation +# Network diagnostics; this is not full schema + reference validation communitymech audit-network -# Or use review-communities skill -# (checks evidence references as part of full QA) +# Separate schema and reference checks for the actual reviewed record: +just validate-strict +just validate-references-explained +# Then use review-communities for the assessed review and shared persistence. ``` --- @@ -163,8 +189,9 @@ communitymech audit-network - May be preprint DOI — use `handle_special_references.py` **"Missing evidence entirely":** -- Run `quick_literature_review.py` to find candidate papers -- Or use LLM: `communitymech repair-network --community CommunityMech:XXXXXX` +- Hand off to `review-communities` to assess inspected sources or record a gap. +- Do not run paid/LLM producers without explicit authorization. Suggestions + remain unassessed leads until the source and exact assertion are checked. --- diff --git a/.claude/skills/review-communities/SKILL.md b/.claude/skills/review-communities/SKILL.md index 7a648d85b..8e926cc32 100644 --- a/.claude/skills/review-communities/SKILL.md +++ b/.claude/skills/review-communities/SKILL.md @@ -9,6 +9,38 @@ created: 2026-03-16 # Review Communities Skill +## Structured Review Output + +For every new review or audit, follow +[docs/record-reviews.md](../../../docs/record-reviews.md) and +[the local profile](../../../docs/record-review-profile.md). +Capture exact targets and input hashes before judging, preserve this skill's +native rubric, rule IDs, scores and evidence requirements, then author the +structured assessment and run: + +```bash +uv run python scripts/record_review.py inspect --targets /tmp/review-targets.yaml +uv run python scripts/record_review.py validate /tmp/completed-review.yaml +uv run python scripts/record_review.py save --content /tmp/completed-review.yaml +``` + +Choose session-unique temporary paths. Save authoritative YAML and derived +Markdown under `reviews/structured/-/`; link both in the +final response. This output contract supersedes prose-only report examples. +A single record uses `kind: record`; batches declare exact selection, +population, reviewed targets and limits. Categories also state boundary decisions. +Every reviewed target must have an assessment. Keep P1-P4 and other native +severity/rule information with a justified common severity, and metric definitions, +scales and denominators. Do not infer scientific approval from a native score. + +Raw provider drafts and deterministic validator/scan reports are diagnostic +inputs, not completed scientific reviews. Use `scientific_review: false` +for deterministic-only or provenance-only assessments; mark required unavailable +checks and incomplete coverage explicitly. A valid bundle does not change native +status, clear release holds, authorize edits, or append curation history. +For audit-only requests, stop after assessment and persistence; any application +steps below require curation intent. + ## Overview The **Review Communities** skill provides quality assurance for ontology-mapped microbial @@ -101,7 +133,8 @@ source attribution (CultureMech vs community_curated). ### 6. Full Quality Control ```bash -just qc # validate-all + validate-terms-all + validate-references-all + lint + test +just qc # native schema/cross-field, taxonomy, terms, lint and test gates +just qc-references # separate reference/evidence check; may need network access ``` ### Claude Code-Assisted Review @@ -154,7 +187,7 @@ Issues are graded by priority. Full definitions (checks, impact, fixes) are in The skill drives existing `just` recipes: ``` -qc: validate-all validate-terms-all validate-references-all lint test +qc / qc-references # distinct gates validate[-all] / validate-terms[-all] / validate-references[-all] # validation audit-network / check-network-quality # network integrity link-media-dry / link-media / link-media-report # growth media diff --git a/.claude/skills/review-communities/reference/execution-protocol.md b/.claude/skills/review-communities/reference/execution-protocol.md index c4c20d024..029aa628d 100644 --- a/.claude/skills/review-communities/reference/execution-protocol.md +++ b/.claude/skills/review-communities/reference/execution-protocol.md @@ -99,6 +99,13 @@ for line in validation_output: ### Step 4: Report Issues to User +Save the assessed observation using [docs/record-reviews.md](../../../../docs/record-reviews.md) +and `uv run python scripts/record_review.py save --content /tmp/completed-review.yaml` +after input inspection and validation. Use session-unique temporary paths. +The template below is a diagnostic presentation example, not the final output +contract. A review-only invocation ends after saving its structured bundle; +the apply/history steps require separately authorized curation. + ```markdown ## Validation Report: {community_name} diff --git a/.claude/skills/review-communities/reference/metrics-and-reports.md b/.claude/skills/review-communities/reference/metrics-and-reports.md index f1999ffc9..d3a6086ce 100644 --- a/.claude/skills/review-communities/reference/metrics-and-reports.md +++ b/.claude/skills/review-communities/reference/metrics-and-reports.md @@ -92,6 +92,13 @@ def calculate_validation_score(validation_summary): ## Validation Reports +The examples below illustrate native metrics and diagnostic summaries. +New final reviews follow [docs/record-reviews.md](../../../../docs/record-reviews.md) +and use the shared saver for YAML plus derived Markdown. Preserve both score +definitions above in assessment metrics; keep findings and evidence per record +and per interaction/causal edge. A high score or a clean network scan is not +scientific approval. + ### Text Report Format ```markdown diff --git a/.claude/skills/review-yaml-category/SKILL.md b/.claude/skills/review-yaml-category/SKILL.md index 1570d268a..6749cb44d 100644 --- a/.claude/skills/review-yaml-category/SKILL.md +++ b/.claude/skills/review-yaml-category/SKILL.md @@ -24,7 +24,7 @@ mix unrelated concepts and should be split, which patterns are well supported, and which future curation changes would make the category sound. Reviewing a category is not curation. A review request authorizes reads, -validation commands, Markdown reports under the review-report path named below, +validation commands, structured YAML/Markdown review bundles described below, and a concise final summary; it does not authorize editing records, regenerating products, spending provider credits, contacting anyone, or creating or mutating GitHub issues, pull requests, comments, labels, or @@ -43,23 +43,22 @@ lump them before reviewing so duplicates are judged together. ## Scope -Review only YAML records from this repository's curated record corpus. If a -record is generated from a maintained table, overlay, or source transform, -report the maintained upstream input that owns any future fix. Do not patch -generated artifacts, generated pages, cache files, reports, or cross-repository -outputs to make a reviewed category look correct, except for new review -reports this skill writes. - -Use the write boundaries, generated-output warnings, and curation ownership -rules from `.claude/skills/curate-yaml-record/SKILL.md` to decide where future -fixes would belong. For this skill, report those paths; do not make the -changes. - -Use `.claude/skills/review-yaml-record/SKILL.md` for the per-record review -rubric and `.claude/skills/curate-yaml-record/references/review-checklist.md` -for repository-specific field expectations. The category review adds a boundary -layer: membership, lumping, splitting, and systemic patterns across the member -records. +Review records from this repository's declared corpus, including an exact frozen +snapshot row when the local profile uses that surface. Read +`docs/record-review-profile.md` for the active native review entrypoints, local +rubrics, validators, and maintained input ownership. A curation skill is not a +prerequisite for reviewing a record. + +For a generated record, identify the maintained table, overlay, source transform +or seeder that owns a future fix. Do not patch records, generated artifacts, +pages, caches, or cross-repository outputs. Only the new review bundle is written. + +Preserve local claim types, priority rules, scoring definitions, and scientific +status/history gates. The shared shape standardizes observations; it does not +replace the Mech's field-by-field rubric or authorize status promotion. + +Use the local record-review rubric for individual members. Add explicit cohort +membership, lump/split/retain/defer decisions and systemic findings. ## Evidence Rules @@ -140,10 +139,8 @@ search, call the miss provisional. 1. Read the local guidance that names exact validators and write boundaries: - `CLAUDE.md`, `justfile`, - `.claude/skills/review-yaml-record/SKILL.md`, - `.claude/skills/curate-yaml-record/SKILL.md`, and - `.claude/skills/curate-yaml-record/references/review-checklist.md`. + `CLAUDE.md`, `justfile`, `docs/record-review-profile.md`, its local rubrics, + and `docs/record-reviews.md` for the shared output contract. 2. Resolve the requested category into one or more bounded cohorts under the curated record globs named above. Record the selection rule for each cohort, then enumerate its candidate members from the filesystem. Do not write a @@ -188,82 +185,38 @@ search, call the miss provisional. ## Output -After resolving at least one coherent target category and completing a review, -write one timestamped Markdown report per reviewed cohort before the final -response: - -- Name each report - `reports/yaml_category_review/-.md`. Use - `date -u +%Y%m%dT%H%M%SZ` for the UTC timestamp. Preserve the category slug - when it is already filename-safe; otherwise slugify it to lower-case ASCII - words joined with `-`. -- Create `reports/yaml_category_review/` if it does not exist. -- Do not overwrite or append to a prior review. If a filename already exists, - regenerate the timestamp. -- Keep this section order so review reports are easy to diff across the fleet: - -```markdown -# YAML Category Review: - -- Repository: -- Category: -- Selection Rule: -- Started UTC: -- Finished UTC: -- Verdict: - -## Target Category -## Selection and Membership -## Validation -## Lump and Split Review -## Identity and Grounding -## Evidence Patterns -## Completeness Patterns -## Findings -## Recommended Edits -## Follow-up Checks -## Additional Notes -``` - -Use tables, bullets, or prose inside those headings as the category demands. -Put repo-specific diagnostics, edge cases, and low-signal observations under -**Additional Notes** instead of inventing new top-level sections. - -The report must cover: - -- **Verdict**: pass, pass with minor issues, or needs curation. -- **Target Category**: the resolved cohort label, source request, selection - rule, member count, and whether coverage was full, split into subcohorts, or - sampled. -- **Selection and Membership**: every included record path and every excluded - near miss that explains the boundary. -- **Validation**: each command run and its result, including unavailable checks. -- **Lump and Split Review**: synonyms, duplicate identities, over-split - variants, over-broad groups, singleton smells, and legitimate siblings that - should remain separate. -- **Identity and Grounding**: whether the members agree on class, identifier - policy, label shape, source identity, ontology grounding, and internal - references. -- **Evidence Patterns**: supported category-level patterns, unsupported or - over-scoped source usage, and citation or snippet mismatches that recur. -- **Completeness Patterns**: consequential gaps, empty optional slots correctly - left empty, and bounded searches that found nothing. -- **Findings**: blocker, major, and minor findings, each with evidence, - affected paths, and a maintained owner path for any fix. -- **Recommended Edits**: concrete future curation actions, ordered by severity, - with the maintained path that owns each fix. -- **Follow-up Checks**: the narrowest validators or manual checks that would - prove each recommended edit. - -Use `None found` or `Not checked: ` when a section has no findings or a -check cannot run; do not delete required headings. If only a sample was read, -the verdict must say `sampled` and must not claim full-category coverage. - -Do not append a curation event, promote a review status, or write a history -entry from this read-only review. Those belong to a later curation change. -If the request needs disambiguation before a coherent category is resolved, ask -for it without creating a report. - -In the final response, link every report path and summarize only each verdict, -finding counts by severity, and any skipped validators or unresolved blockers. +Save one immutable structured review bundle per reviewed cohort +using the CLAW-governed contract in `docs/record-reviews.md` and +`schema/record_review.yaml`. Preserve the local rubric identified by +`docs/record-review-profile.md`. + +- Capture actual UTC start/finish, reviewer identity and independence, exact + target IDs/locators, Git base, input hashes, and generated-input owners. +- Retain every check and its real result, domain assessments, inspected evidence, + normalized findings, native rules/severity rationale, proposed actions with + acceptance checks, and explicit limitations. Do not equate a deterministic + check with scientific review. +- Use `kind: category`; enumerate reviewed members, population and selection, + and record explicit lump/split/retain/defer decisions with evidence. Sampled + coverage must retain its method and uninspected remainder. +- Invoke `uv run python scripts/record_review.py inspect --targets ` + before assessment, then `validate ` and + `save --content ` with the same script. Recheck changed + inputs instead of silently refreshing their hashes. +- The saver writes + `reviews/structured/-/review.yaml` plus `review.md`. + YAML is authoritative; do not hand-edit the rendered Markdown or overwrite an + earlier bundle. Run `uv run python scripts/record_review.py check` afterward. +- If required checks are unavailable after the target is resolved, save an honest + partial/blocked observation. If the shared saver itself cannot run, report + that persistence is blocked; session-only prose is not a saved review. +- Retain stable issue keys and exact `previous_occurrences` when reassessing a + finding. A later clean report does not close earlier unresolved findings. +- Do not append curation/history events or promote native scientific status. + Those require a separately authorized curation change and native gates. + +Do not create a report for an unresolved ambiguous target. In the final response, +link both saved files and summarize scope, verdict, findings by severity, and +unavailable checks. Existing ad hoc Markdown is historical, not the output format +for new reviews. diff --git a/.claude/skills/review-yaml-record/SKILL.md b/.claude/skills/review-yaml-record/SKILL.md index 58a206d04..28aa8dce3 100644 --- a/.claude/skills/review-yaml-record/SKILL.md +++ b/.claude/skills/review-yaml-record/SKILL.md @@ -23,7 +23,7 @@ or internally inconsistent, what is materially incomplete, and what bounded checks would resolve the remaining uncertainty. Reviewing is not curation. A review request authorizes reads, validation -commands, one new Markdown report under the review-report path named below, and +commands, one new structured YAML/Markdown review bundle described below, and a concise final summary; it does not authorize editing a record, regenerating products, spending provider credits, contacting anyone, or creating or mutating GitHub issues, pull requests, comments, labels, or settings. @@ -40,21 +40,19 @@ cannot support a record-level verdict on their own. ## Scope -Review only YAML records from this repository's curated record corpus. If a -record is generated from a maintained table, overlay, or source transform, -report the maintained upstream input that owns any future fix. Do not patch -generated artifacts, generated pages, cache files, reports, or cross-repository -outputs to make a reviewed record look correct, except for the new review -report this skill writes. - -Use the write boundaries, generated-output warnings, and curation ownership -rules from `.claude/skills/curate-yaml-record/SKILL.md` to decide where a future -fix would belong. For this skill, report that path; do not make the change. - -Use `.claude/skills/curate-yaml-record/references/review-checklist.md` as the -field-by-field rubric for this corpus. The checklist is deliberately local: the -shared review shape is the same across the fleet, but claim types and -completeness criteria are Mech-specific. +Review records from this repository's declared corpus, including an exact frozen +snapshot row when the local profile uses that surface. Read +`docs/record-review-profile.md` for the active native review entrypoints, local +rubrics, validators, and maintained input ownership. A curation skill is not a +prerequisite for reviewing a record. + +For a generated record, identify the maintained table, overlay, source transform +or seeder that owns a future fix. Do not patch records, generated artifacts, +pages, caches, or cross-repository outputs. Only the new review bundle is written. + +Preserve local claim types, priority rules, scoring definitions, and scientific +status/history gates. The shared shape standardizes observations; it does not +replace the Mech's field-by-field rubric or authorize status promotion. ## Evidence Rules @@ -130,10 +128,9 @@ search, call the miss provisional. 1. Read the local guidance that names exact validators and write boundaries: - `CLAUDE.md`, `justfile`, - `.claude/skills/curate-yaml-record/SKILL.md`, and - `.claude/skills/curate-yaml-record/references/review-checklist.md`. -2. Resolve one YAML file under the curated record globs named above. Confirm + `CLAUDE.md`, `justfile`, `docs/record-review-profile.md`, its local rubrics, + and `docs/record-reviews.md` for the shared output contract. +2. Resolve one record under the declared corpus or native frozen-row selector. Confirm its class, identifier, label, source provenance, grounding status, evidence entries, discussion or quality flags, generated status, and curation history shape. @@ -162,69 +159,36 @@ search, call the miss provisional. ## Output -After resolving exactly one target and completing a review, write exactly one -timestamped Markdown report before the final response: - -- Name it `reports/yaml_record_review/-.md`. - Use `date -u +%Y%m%dT%H%M%SZ` for the UTC timestamp. Preserve the target - file stem when it is already filename-safe; otherwise slugify the stem to - lower-case ASCII words joined with `-`. -- Create `reports/yaml_record_review/` if it does not exist. -- Do not overwrite or append to a prior review. If a filename already exists, - regenerate the timestamp. -- Keep this section order so review reports are easy to diff across the fleet: - -```markdown -# YAML Record Review: - -- Repository: -- Record: -- Started UTC: -- Finished UTC: -- Verdict: - -## Target -## Validation -## Identity and Grounding -## Evidence -## Completeness -## Findings -## Recommended Edits -## Follow-up Checks -## Additional Notes -``` - -Use tables, bullets, or prose inside those headings as the record demands. Put -repo-specific diagnostics, edge cases, and low-signal observations under -**Additional Notes** instead of inventing new top-level sections. - -The report must cover: - -- **Verdict**: pass, pass with minor issues, or needs curation. -- **Target**: the record reviewed, its path, class, ID, label, and generated or - maintained status. -- **Validation**: each command run and its result, including unavailable checks. -- **Identity and Grounding**: whether the record's ID, label, category, source - identity, and ontology grounding agree. -- **Evidence**: supported claims, unsupported or over-scoped claims, and any - citation or snippet mismatch. -- **Completeness**: consequential gaps, empty optional slots correctly left - empty, and bounded searches that found nothing. -- **Findings**: blocker, major, and minor findings, each with evidence and a - maintained owner path for any fix. -- **Recommended Edits**: concrete future curation actions, ordered by severity, - with the maintained path that owns each fix. -- **Follow-up Checks**: the narrowest validators or manual checks that would - prove each recommended edit. - -Use `None found` or `Not checked: ` when a section has no findings or a -check cannot run; do not delete required headings. - -Do not append a curation event, promote a review status, or write a history -entry from this read-only review. Those belong to a later curation change. -If the request needs disambiguation before one target is resolved, ask for it -without creating a report. - -In the final response, link the report path and summarize only the verdict, -finding counts by severity, and any skipped validators or unresolved blockers. +Save one immutable structured review bundle for the resolved record +using the CLAW-governed contract in `docs/record-reviews.md` and +`schema/record_review.yaml`. Preserve the local rubric identified by +`docs/record-review-profile.md`. + +- Capture actual UTC start/finish, reviewer identity and independence, exact + target IDs/locators, Git base, input hashes, and generated-input owners. +- Retain every check and its real result, domain assessments, inspected evidence, + normalized findings, native rules/severity rationale, proposed actions with + acceptance checks, and explicit limitations. Do not equate a deterministic + check with scientific review. +- Use `kind: record`; it identifies exactly one target. +- Invoke `uv run python scripts/record_review.py inspect --targets ` + before assessment, then `validate ` and + `save --content ` with the same script. Recheck changed + inputs instead of silently refreshing their hashes. +- The saver writes + `reviews/structured/-/review.yaml` plus `review.md`. + YAML is authoritative; do not hand-edit the rendered Markdown or overwrite an + earlier bundle. Run `uv run python scripts/record_review.py check` afterward. +- If required checks are unavailable after the target is resolved, save an honest + partial/blocked observation. If the shared saver itself cannot run, report + that persistence is blocked; session-only prose is not a saved review. +- Retain stable issue keys and exact `previous_occurrences` when reassessing a + finding. A later clean report does not close earlier unresolved findings. +- Do not append curation/history events or promote native scientific status. + Those require a separately authorized curation change and native gates. + +Do not create a report for an unresolved ambiguous target. In the final response, +link both saved files and summarize scope, verdict, findings by severity, and +unavailable checks. Existing ad hoc Markdown is historical, not the output format +for new reviews. diff --git a/.github/workflows/validate-strict.yaml b/.github/workflows/validate-strict.yaml index 6a5c08ab1..bff3fecf7 100644 --- a/.github/workflows/validate-strict.yaml +++ b/.github/workflows/validate-strict.yaml @@ -12,6 +12,10 @@ on: - kb/communities/** - prompts/** - conf/deep_research_provider.yaml + - conf/record_review.yaml + - schema/record_review.yaml + - reviews/structured/** + - docs/record-review*.md - data/isolates/** - kb/taxa/** - .claude/skills/** @@ -33,12 +37,15 @@ permissions: # Keep uv commands on the fleet runtime, including --no-project invocations. env: UV_PYTHON: "3.13" + RECORD_REVIEW_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || (github.event_name == 'workflow_dispatch' && 'HEAD') || 'INVALID-REVIEW-BASE' }} jobs: validate-strict: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: extractions/setup-just@v3 @@ -95,6 +102,9 @@ jobs: - name: Run audit-writers run: just audit-writers + - name: Validate structured record reviews + run: just test-record-reviews + - name: Run tests run: uv run pytest tests/ -q --no-cov diff --git a/.gitignore b/.gitignore index e326fa7cd..ca6bed232 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,7 @@ pages/style.css # linkml-term-validator / OAK label cache cache/ +# New structured review bundles are tracked; legacy report rules stay scoped. +!/reviews/ +!/reviews/structured/ +!/reviews/structured/** diff --git a/conf/record_review.yaml b/conf/record_review.yaml new file mode 100644 index 000000000..2fa4536ce --- /dev/null +++ b/conf/record_review.yaml @@ -0,0 +1,15 @@ +version: 1 +repository: CultureBotAI/CommunityMech +skills: + - .claude/skills/review-yaml-record/SKILL.md + - .claude/skills/review-yaml-category/SKILL.md + - .claude/skills/review-communities/SKILL.md + - .claude/skills/curate-yaml-record/SKILL.md + - .claude/skills/evidence-curation/SKILL.md +rubrics: + - CLAUDE.md + - .claude/skills/review-communities/reference/validation-rules.md + - .claude/skills/review-communities/reference/metrics-and-reports.md + - .claude/skills/review-communities/reference/execution-protocol.md + - .claude/skills/curate-yaml-record/references/review-checklist.md + - .claude/skills/evidence-curation/SKILL.md diff --git a/docs/record-review-profile.md b/docs/record-review-profile.md new file mode 100644 index 000000000..d3ea862ad --- /dev/null +++ b/docs/record-review-profile.md @@ -0,0 +1,100 @@ +# CommunityMech Record Review Profile + +Passing verdicts require positive evidence-linked assessments of the reviewed +targets. Terminal finding dispositions require those targets to have actually +been reviewed and assessed, with completion other than failed. Every +supersession must preserve all affected targets from the cited prior finding, +including multi-record findings; a narrower observation cannot silently drop +unassessed members or close their issues. Keep their remaining scope explicit. + +Use [Structured Record Reviews](record-reviews.md) and the route/rubric inventory +in `conf/record_review.yaml`. + +## Routes And Domain Evidence + +- `review-yaml-record`: one resolved community, isolate or reusable taxon. +- `review-yaml-category`: coherent ecological/taxonomic/experimental cohorts + with explicit membership, sampling and lump/split/retain/defer decisions. +- `review-communities`: per-record or explicit batch P1-P4 review. +- `curate-yaml-record`: the audit-only route uses the same saved output. +- `evidence-curation`: delegate final assessed evidence review to registered + `review-communities`, saved with that route as `skill`. Extraction, repair + suggestions and network diagnostics are not completed scientific review. + Preserve per-record P1-P4, both scores and edge-level source/participant/ + direction/context evidence. No stale literature producer is recreated; + paid/LLM calls need explicit authorization, and review-only work runs no + repair/apply commands or status/history transitions. + +Preserve the native validation rules, execution protocol, metrics and curation +checklist. Each reviewed record needs its own assessment and affected findings. +Record stable IDs, NCBITaxon/GTDB grounding, source strain, community scope, +experimental setting, medium and environmental context in evidence-linked +dimensions. A strain pair, enrichment or synthetic consortium does not establish +the same claim for a natural community. + +For each ecological interaction and downstream causal edge, retain source and +target participants, direction, mechanism, context, exact field path, PMID/DOI, +source locator and inspected verbatim snippet. Network integrity and lexical +snippet similarity do not prove causal support. Evidence belongs at the specific +assertion. Keep abstract, full-text and supplementary evidence boundaries. + +Retain `rule_id` and P1-P4 `native_severity` with explicit impact-based +`normalization_reason` (normally P1 blocker, P2 major, P3 minor, P4 informational). +Preserve the two distinct 0-100 native metrics from +`reference/metrics-and-reports.md`: completeness and validation quality. +Store each under assessment metrics with its formula/definition and scale; +record constituent issue counts and evidence-per-interaction denominator. +Neither score substitutes for scientific evidence or a release verdict. + +## Ownership + +Maintained records are `kb/communities/`, `data/isolates/` (MicrobialCommunity) +and `kb/taxa/` (CommonTaxon). Identify exact record paths and claim selectors. +The LinkML source schema owns data shape; generated Python, HTML and KGX are +derived. Trace generated defects to the record, template or exporter owner. +Hash all local schema/source/cache inputs actually used with `inspect --input`. +Do not patch generated Pages or append history during review-only work. + +## Real Commands And Persistence + +```bash +uv run python scripts/record_review.py inspect --targets /tmp/review-targets.yaml +just validate +just validate-strict +just validate-terms +just validate-references-explained +just audit-network +just check-network-quality +uv run python scripts/record_review.py validate /tmp/completed-review.yaml +uv run python scripts/record_review.py save --content /tmp/completed-review.yaml +just check-record-reviews +just test-record-reviews +``` + +For reusable taxa use `just validate-taxa` and `just validate-terms-taxa`. +Report their corpus scope honestly. Select GTDB/domain gates when relevant. +`just qc` retains lint/tests, schema/cross-field, GTDB, scalar and ontology +checks. Reference checking is separate: `just qc-references` may need network +access and is not silently counted as part of `qc`. + +Use session-unique temporary inputs. New final reviews persist as authoritative +YAML and derived Markdown under `reviews/structured/-/`. +A sampled batch states selection, population, reviewed IDs, exclusions and limits. +Link both saved files. Missing required checks mean partial/blocked review. + +`audit-network-json`, `audit-network-report`, linkage reports and validator +output are deterministic diagnostics. They may be evidence for an assessment, +but are not completed scientific review. A saved diagnostic-only assessment +uses `scientific_review: false`. Raw research output requires actual source and +edge-level assessment before saving a scientific review; no provider is invoked +by this contract. + +## Retained Gates + +There is no record-level REVIEWED flag. Keep guarded writes, claim evidence, +native schemas, network thresholds, reference checks and append-only history. +Saving a bundle neither authorizes curation nor records a status transition. +Native generated-docs checks remain required for changes to records/templates. +No historical reports or history entries are migrated. The shared schema, +helper, documentation and contract test remain byte-identical CLAW payloads; +local profiles, rubrics and scientific judgments are maintained here. diff --git a/docs/record-reviews.md b/docs/record-reviews.md new file mode 100644 index 000000000..326f10002 --- /dev/null +++ b/docs/record-reviews.md @@ -0,0 +1,176 @@ +# Structured Record Reviews + +CLAW owns `schema/record_review.yaml` and `scripts/record_review.py`. The schema +defines the common record; the helper enforces its semantic constraints and +saves an immutable pair: + +```text +reviews/structured/-/review.yaml +reviews/structured/-/review.md +``` + +YAML is authoritative. Markdown is rendered from it, not a second editable +verdict. New reviews use this format; old reports remain historical evidence +and are not automatically converted. A valid report does not certify scientific +truth or authorize curation, Git publication, paid research, or status promotion. + +Read `docs/record-review-profile.md` for this Mech's entrypoints, local rubrics, +input ownership, validators, and any native scientific-status gates. Preserve +those domain rules; this contract standardizes their output, not their science. + +## Capture And Save + +Use the repository's Python environment with LinkML, linkml-runtime, jsonschema, +and PyYAML installed. Missing dependencies are a blocked output step, not +permission to save unvalidated prose instead. From the repository root: + +```bash +uv run python scripts/record_review.py inspect --targets /tmp/review-targets.yaml +uv run python scripts/record_review.py validate /tmp/completed-review.yaml +uv run python scripts/record_review.py save --content /tmp/completed-review.yaml +uv run python scripts/record_review.py check +``` + +The temporary input locations are illustrative; choose session-unique paths. +`inspect` takes a mapping with a nonempty `targets` list. Each target uses the +schema's `ReviewTarget` fields: stable `target_id`, repository-relative `path`, +`label`, `kind`, and `owner_paths` (repository/path/role) or an explicit +`ownership_note`. Include an exact `selector` for a frozen snapshot row. Add +`--input ` for each schema, overlay, source table, or other local input +whose bytes informed the review. Inspect before judging; retain the resulting +Git base and SHA-256 input hashes in `source`. Inspection is not a saved review. + +Read the actual input bytes. For a generated record, identify the maintained +table, overlay, source transform, or seeder that owns a future fix. Include an +algorithm-labelled `semantic_digest` when the native Mech gate uses one; raw +file hashes and semantic hashes serve different purposes. + +Author a complete document against `RecordReview` in the schema: + +- Identity: `schema_version: 1.0.0`, timestamp-prefixed `review_id`, `kind`, + repository, title, actual UTC start/finish, reviewer identity and independence + basis, invoking skill, completion, verdict, scientific-review flag, summary. +- Scope: exact selected targets, selection rule, population denominator, + actually reviewed target IDs, full/sampled/partial coverage, exclusions. + Sampling requires its method and limitations; retain seed/strata where used. +- Checks: actual documented commands, actual exit codes, required status, + result, targets, and evidence. Do not execute commands copied from an old + report. Distinguish failed, skipped, unavailable, and not applicable. +- Evidence: stable local IDs, source reference, precise locator, actual access + timestamp, support/refutation/context, and what was inspected. Record bounded + search scope, including whether ignored files were searched, for absence + findings. Search snippets and expression-module membership are not proof of + experimental mechanism. +- Assessments: common area plus a domain-specific topic, outcome, targets, + evidence, summary, optional detailed prose and metrics. Every reviewed target + must be assessed. Scores need units, definitions, denominators where relevant, + and scales; do not equate scores across Mechs just because both are percentages. + Use evidence-linked `dimensions` (name, value, definition, evidence IDs) for + structured domain context such as strain, criticality authority/edition, + biomass state, source category, taxon rank or evidence tier. Prose details may + remain flexible without losing these important triage coordinates. +- Findings: local ID, stable repository-scoped `issue_key`, category, severity, + status, certainty, affected targets and field paths, evidence, owner paths, + and description. Preserve local rule IDs and native severities with an + explicit normalization rationale. Do not reuse an issue key for an unrelated + defect. Use `related_findings` for similarities across repositories. +- Actions: affected findings and targets, maintained owners, concrete acceptance + checks, optional generator, dependency IDs, and blocking finding references. + These are proposed work, not a record that the fix was already performed. +- Category boundaries: explicit lump/split/retain/defer decisions, evidence, + rationale and proposed membership. A split is a complete disjoint partition; + a lump requires at least two targets. Similar names alone do not prove identity. +- Explicit limitations and optional notes, tags, links and related reviews. + Empty `findings: []` means no findings in this scope, not a fleet-wide pass. + +Use `kind: record` for one record, `category` for a coherent cohort with a +boundary decision, `batch` for an explicit sample/batch, and `repository` for a +bounded repository-level inspection. Unresolved identity or an ambiguous target +needs clarification before a report is created. Once a target is resolved, +retain a partial/blocked review when required checks cannot run. A process crash +or provider failure without assessed content is not a completed review. + +Use current UTC times, not a historical filename's timestamp. On an ID collision, +choose a new suffix; never overwrite an earlier review. Saving verifies origin, +Git base, hashes, path safety and Git visibility. If inputs changed, inspect and +reassess the changed content; do not merely replace the old hashes. A deliberately +historical review can use `source.state: git_commit` with exact tracked bytes. +It must say it assesses that revision, not the current working tree. +On ingestion, the base commit must still be available. A `git_commit` review's +hashes are checked against that historical tree, not today's files. A +`working_tree` review preserves the reviewer's input-byte attestation; it is not +proof that those bytes were committed. Invalid or unverifiable provenance is +reported as an error and cannot retire a previous finding. + +The saver publishes YAML only after Markdown and input rechecks succeed. It +does not stage or commit either file. `check` includes hidden/ignored existing +bundles and fails on mismatched, malformed or incomplete pairs. No saved reviews +is reported as missing coverage; `check --require-reviews` makes that an error. + +`check --base ` also rejects deletion or byte changes to previously +committed bundles. The default is `RECORD_REVIEW_BASE` when configured, otherwise +`HEAD`. CI must set that variable to the trusted pull-request, merge-group or +push event base and fetch that revision; checking against an already changed +HEAD cannot detect a committed rewrite. Unknown bases fail rather than silently +skipping the check. The canonical consumer test uses the same base. Correct a +review by adding a new linked observation, never by rewriting its history. + +## Verdict And Severity + +Use `pass`, `pass_with_limitations`, `needs_curation`, `seed_only`, `blocked`, +or `not_assessed`. Preserve a Mech-specific verdict in `native_verdict` if needed. +`completion` is separate from the scientific verdict. Deterministic validation +and seed metadata use `scientific_review: false`. Provenance-only inspection +must explicitly say literature support was not reassessed. + +- Blocker: wrong identity, invalid representation, broken critical references, + or a defect that makes the record denote the wrong thing. +- Major: materially unsupported claims, wrong grounding, scope inflation, + missing required representation, or systemic source/generator errors. +- Minor: bounded, non-blocking wording, redundancy or provenance problems. +- Informational: a scoped observation that does not require a corrective edit. + +Passing verdicts cannot conceal unresolved major/blocker findings or failed +required checks. Unavailable required checks preclude completed status. Report +limitations instead of silently dropping checks or broadening sample results. +Every passing target needs a positive, evidence-linked assessment. An unknown +or concerning assessment precludes plain `pass`; a qualified pass must state +its limitations and still positively assess each actually reviewed target. +Do not promote native record status or append curation/history events merely +because the review document validates. + +## Triage And Follow-Up + +Each review is a new observation. To change a finding's disposition, retain its +`issue_key` and cite exact repository/review/finding identities in +`previous_occurrences`. Resolved, rejected and accepted-risk findings require +that history, inspected evidence, and a disposition reason. A newer clean +review does not close an older issue. Multiple conflicting heads stay visible +until a new observation explicitly reconciles all relevant predecessors. +Every successor, including an open or deferred update, must retain all affected +targets of each predecessor it supersedes. A failed review cannot make a +terminal disposition. A partial review can resolve a scoped finding only when +its affected targets were actually reviewed and assessed against evidence. + +From a configured CLAW checkout, aggregate without modifying Mechs: + +```bash +kg-microbe-reviews fleet --all --format markdown --output /tmp/review-triage.md +kg-microbe-reviews fleet --all --status open --severity major --format tsv +``` + +The default reads each local cached `origin/main` commit, not uncommitted reports, +and never fetches. Use the normal fleet refresh workflow separately when remote +freshness is required. `--working-tree` includes local reports, including ignored +ones. Every manifest target remains in the inventory, even if unavailable or +without structured output. JSON retains all evidence and action history; the +Markdown/TSV views are summaries. Filters never bypass validation or hide lineage +errors. Current input hashes indicate whether the observed source surface still +matches, changed, or could not be established, not whether the science is true. +Currentness remains visible for closed findings as well as open ones. Action +plans come from unresolved finding heads and their prerequisite closure; they +are proposed work with execution status not established, not completed tasks. + +Link the saved Markdown and YAML in the final response. Summarize the declared +scope, verdict, finding counts, and unavailable checks. Do not create GitHub +issues or curate records unless separately requested. diff --git a/justfile b/justfile index 08d085548..2af6bc55c 100644 --- a/justfile +++ b/justfile @@ -9,6 +9,13 @@ set dotenv-load := true # uses $1/$@, so enabling this changes nothing else. set positional-arguments := true +# Validate immutable structured review bundles and their adoption contract. +check-record-reviews: + uv run python scripts/record_review.py check + +test-record-reviews: + uv run --with pytest --with pytest-cov python -m pytest tests/test_record_review_contract.py --no-cov -q + # Shared tooling lives in the culturebotai-claw checkout. Override CLAW_SRC when # claw is not the default sibling directory — CI checks it out elsewhere. claw_src := env_var_or_default("CLAW_SRC", "../../culturebotai-claw/src") diff --git a/pyproject.toml b/pyproject.toml index a85080ae5..1822f228d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,9 @@ target-version = ["py310", "py311", "py312"] # byte-identical across the Mech repos and would break `vendored-sync` — the # directory form is excluded, the explicit form was not (#412 review). force-exclude = ''' -/tests/test_id_label_(empty_adapter|unknown_prefix|plausibility)\.py$ +/scripts/record_review\.py$ +|/tests/test_record_review_contract\.py$ +|/tests/test_id_label_(empty_adapter|unknown_prefix|plausibility)\.py$ |/scripts/(check_vendored_sync|validate_id_label_correspondence|chem_formula|auto_merge_ready_prs|verify_merge_integrity)\.py$ ''' @@ -128,6 +130,8 @@ target-version = "py310" # restyling (e.g. N802 test_NOT_empty→test_not_empty, E501 wraps) would break that # cross-repo pin, so they are excluded from both ruff and black here. extend-exclude = [ + "scripts/record_review.py", + "tests/test_record_review_contract.py", "src/communitymech/datamodel/communitymech.py", "tests/test_id_label_empty_adapter.py", "tests/test_id_label_unknown_prefix.py", diff --git a/schema/record_review.yaml b/schema/record_review.yaml new file mode 100644 index 000000000..630d95bda --- /dev/null +++ b/schema/record_review.yaml @@ -0,0 +1,347 @@ +id: https://w3id.org/culturebotai/claw/record-review +name: record_review +title: Mech Record Review +description: >- + An immutable, scoped review observation, not permission to curate or proof of + scientific truth. Shared findings support fleet triage while assessments retain + domain-specific questions, evidence, scores, uncertainty, and ownership. +version: 1.0.0 +prefixes: + review: https://w3id.org/culturebotai/claw/record-review/ + xsd: http://www.w3.org/2001/XMLSchema# +default_prefix: review +default_range: string +types: + string: + base: str + uri: xsd:string + integer: + base: int + uri: xsd:integer + boolean: + base: Bool + uri: xsd:boolean + float: + base: float + uri: xsd:double + NonBlank: + typeof: string + pattern: '\S' + Token: + typeof: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._-]{0,159}(?![\s\S])' + SHA256: + typeof: string + pattern: '^[0-9a-f]{64}(?![\s\S])' + GitRevision: + typeof: string + pattern: '^([0-9a-f]{40}|[0-9a-f]{64})(?![\s\S])' + UTCTimestamp: + typeof: string + pattern: '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z$' + Repository: + typeof: string + pattern: '^[A-Za-z0-9][A-Za-z0-9.-]*/[A-Za-z0-9_.-]+(?![\s\S])' +enums: + ReviewKind: + permissible_values: {record: {}, category: {}, batch: {}, repository: {}} + Completion: + permissible_values: {completed: {}, partial: {}, failed: {}} + Verdict: + permissible_values: + pass: {} + pass_with_limitations: {} + needs_curation: {} + seed_only: {} + blocked: {} + not_assessed: {} + Coverage: + permissible_values: {full: {}, sampled: {}, partial: {}} + SourceState: + permissible_values: {git_commit: {}, working_tree: {}} + TargetKind: + permissible_values: {maintained: {}, generated: {}, snapshot_row: {}, source: {}} + ReviewerKind: + permissible_values: {human: {}, agent: {}, deterministic: {}} + Independence: + permissible_values: {independent: {}, self_review: {}, not_applicable: {}, unknown: {}} + CheckStatus: + permissible_values: {passed: {}, failed: {}, skipped: {}, unavailable: {}, not_applicable: {}} + EvidenceKind: + permissible_values: + primary_source: {} + database: {} + authority: {} + record_content: {} + validation: {} + search: {} + prior_review: {} + Support: + permissible_values: {supports: {}, refutes: {}, partial: {}, context_only: {}, unknown: {}} + Area: + permissible_values: + identity: {} + grounding: {} + evidence: {} + completeness: {} + schema: {} + consistency: {} + provenance: {} + scope: {} + graph: {} + nomenclature: {} + quantity: {} + ownership: {} + representation: {} + other: {} + AssessmentOutcome: + permissible_values: {supported: {}, concern: {}, unknown: {}, not_applicable: {}} + Severity: + permissible_values: {blocker: {}, major: {}, minor: {}, informational: {}} + FindingStatus: + permissible_values: {open: {}, deferred: {}, resolved: {}, accepted_risk: {}, rejected: {}} + Certainty: + permissible_values: {confirmed: {}, provisional: {}} + BoundaryAction: + permissible_values: {lump: {}, split: {}, retain: {}, defer: {}} +classes: + RecordReview: + tree_root: true + attributes: + schema_version: {range: string, required: true, pattern: '^1\.0\.0(?![\s\S])'} + review_id: {range: Token, required: true} + kind: {range: ReviewKind, required: true} + repository: {range: Repository, required: true} + title: {range: NonBlank, required: true} + started_at: {range: UTCTimestamp, required: true} + finished_at: {range: UTCTimestamp, required: true} + reviewer: {range: Reviewer, required: true, inlined: true} + skill: {range: NonBlank, required: true} + completion: {range: Completion, required: true} + verdict: {range: Verdict, required: true} + native_verdict: {range: NonBlank} + scientific_review: {range: boolean, required: true} + summary: {range: NonBlank, required: true} + source: {range: ReviewSource, required: true, inlined: true} + scope: {range: ReviewScope, required: true, inlined: true} + targets: + range: ReviewTarget + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 1 + checks: + range: ReviewCheck + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 1 + evidence: + range: ReviewEvidence + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 0 + assessments: + range: ReviewAssessment + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 1 + findings: + range: ReviewFinding + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 0 + actions: + range: ReviewAction + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 0 + boundary_decisions: + range: BoundaryDecision + multivalued: true + inlined_as_list: true + limitations: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 0} + notes: {range: NonBlank, multivalued: true} + related_reviews: {range: ReviewReference, multivalued: true, inlined_as_list: true} + links: {range: NonBlank, multivalued: true} + tags: {range: Token, multivalued: true} + Reviewer: + attributes: + identity: {range: NonBlank, required: true} + kind: {range: ReviewerKind, required: true} + model: {range: NonBlank} + independence: {range: Independence, required: true} + independence_basis: {range: NonBlank, required: true} + ReviewSource: + attributes: + git_revision: {range: GitRevision, required: true} + state: {range: SourceState, required: true} + snapshot_id: {range: NonBlank} + inputs: + range: InputFile + required: true + multivalued: true + inlined_as_list: true + minimum_cardinality: 1 + InputFile: + attributes: + path: {range: NonBlank, required: true} + sha256: {range: SHA256, required: true} + role: {range: NonBlank, required: true} + ReviewTarget: + attributes: + target_id: {range: NonBlank, required: true} + path: {range: NonBlank, required: true} + selector: + range: NonBlank + description: Exact row or claim locator within path; never part of a filesystem path. + label: {range: NonBlank, required: true} + record_class: {range: NonBlank} + kind: {range: TargetKind, required: true} + semantic_digest: {range: ContentDigest, inlined: true} + owner_paths: {range: OwnedPath, multivalued: true, inlined_as_list: true} + ownership_note: {range: NonBlank} + ContentDigest: + attributes: + algorithm: {range: NonBlank, required: true} + sha256: {range: SHA256, required: true} + description: {range: NonBlank, required: true} + OwnedPath: + attributes: + repository: {range: Repository, required: true} + path: {range: NonBlank, required: true} + role: {range: NonBlank, required: true} + ReviewScope: + attributes: + description: {range: NonBlank, required: true} + selection: {range: NonBlank, required: true} + coverage: {range: Coverage, required: true} + population_size: {range: integer, required: true, minimum_value: 1} + reviewed_target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 0} + sampling_method: {range: NonBlank} + sampling_seed: {range: NonBlank} + exclusions: {range: ScopeExclusion, multivalued: true, inlined_as_list: true} + ScopeExclusion: + attributes: + target: {range: NonBlank, required: true} + reason: {range: NonBlank, required: true} + ReviewCheck: + attributes: + check_id: {range: Token, required: true} + name: {range: NonBlank, required: true} + status: {range: CheckStatus, required: true} + required: {range: boolean, required: true} + summary: {range: NonBlank, required: true} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + command: + range: NonBlank + description: Descriptive command actually run, never executed by a review reader. + exit_code: {range: integer} + expected_exit_code: {range: integer} + evidence_ids: {range: Token, multivalued: true} + scope_note: {range: NonBlank} + ReviewEvidence: + attributes: + evidence_id: {range: Token, required: true} + kind: {range: EvidenceKind, required: true} + reference: {range: NonBlank, required: true} + locator: {range: NonBlank} + accessed_at: {range: UTCTimestamp, required: true} + support: {range: Support, required: true} + summary: {range: NonBlank, required: true} + quote: {range: NonBlank} + snapshot_sha256: {range: SHA256} + search_scope: + range: NonBlank + description: Queries, sources, ignored-file inclusion, bounds, and access limits for a search. + ReviewAssessment: + attributes: + assessment_id: {range: Token, required: true} + area: {range: Area, required: true} + topic: {range: NonBlank, required: true} + outcome: {range: AssessmentOutcome, required: true} + summary: {range: NonBlank, required: true} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + evidence_ids: {range: Token, required: true, multivalued: true, minimum_cardinality: 0} + metrics: {range: ReviewMetric, multivalued: true, inlined_as_list: true} + dimensions: {range: ReviewDimension, multivalued: true, inlined_as_list: true} + details: {range: NonBlank} + ReviewDimension: + description: Named, evidence-linked domain context without unconstrained extension keys. + attributes: + name: {range: Token, required: true} + value: {range: NonBlank, required: true} + definition: {range: NonBlank, required: true} + evidence_ids: {range: Token, required: true, multivalued: true, minimum_cardinality: 0} + ReviewMetric: + attributes: + name: {range: NonBlank, required: true} + value: {range: float, required: true} + unit: {range: NonBlank, required: true} + definition: {range: NonBlank, required: true} + denominator: {range: integer, minimum_value: 1} + minimum: {range: float} + maximum: {range: float} + ReviewFinding: + attributes: + finding_id: {range: Token, required: true} + issue_key: + range: Token + required: true + description: Repository-scoped stable issue identity, deliberately reused only for the same issue. + category: {range: Area, required: true} + severity: {range: Severity, required: true} + status: {range: FindingStatus, required: true} + certainty: {range: Certainty, required: true} + title: {range: NonBlank, required: true} + description: {range: NonBlank, required: true} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + field_paths: {range: NonBlank, multivalued: true} + evidence_ids: {range: Token, required: true, multivalued: true, minimum_cardinality: 1} + rule_id: {range: NonBlank} + native_severity: {range: NonBlank} + normalization_reason: {range: NonBlank} + owner_paths: {range: OwnedPath, multivalued: true, inlined_as_list: true} + ownership_note: {range: NonBlank} + disposition_reason: {range: NonBlank} + previous_occurrences: {range: FindingReference, multivalued: true, inlined_as_list: true} + related_findings: {range: FindingReference, multivalued: true, inlined_as_list: true} + external_issues: {range: NonBlank, multivalued: true} + tags: {range: Token, multivalued: true} + ReviewAction: + attributes: + action_id: {range: Token, required: true} + description: {range: NonBlank, required: true} + finding_ids: {range: Token, required: true, multivalued: true, minimum_cardinality: 0} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + owner_paths: {range: OwnedPath, multivalued: true, inlined_as_list: true} + ownership_note: {range: NonBlank} + acceptance_checks: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + depends_on: {range: Token, multivalued: true} + blockers: {range: FindingReference, multivalued: true, inlined_as_list: true} + generator: {range: NonBlank} + BoundaryDecision: + attributes: + action: {range: BoundaryAction, required: true} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + rationale: {range: NonBlank, required: true} + evidence_ids: {range: Token, required: true, multivalued: true, minimum_cardinality: 0} + proposed_groups: {range: ProposedGroup, multivalued: true, inlined_as_list: true} + ProposedGroup: + attributes: + label: {range: NonBlank, required: true} + target_ids: {range: NonBlank, required: true, multivalued: true, minimum_cardinality: 1} + ReviewReference: + attributes: + repository: {range: Repository, required: true} + review_id: {range: Token, required: true} + relationship: {range: NonBlank, required: true} + FindingReference: + attributes: + repository: {range: Repository, required: true} + review_id: {range: Token, required: true} + finding_id: {range: Token, required: true} diff --git a/scripts/record_review.py b/scripts/record_review.py new file mode 100644 index 000000000..a7d0c2388 --- /dev/null +++ b/scripts/record_review.py @@ -0,0 +1,806 @@ +"""Validate immutable Mech reviews without executing their recorded commands. + +This module is the canonical, standalone payload used by CLAW and its Mechs. +The self-contained LinkML schema defines shape; the checks below enforce scope, +reference closure, provenance, and honest review/triage lifecycle semantics. +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import importlib +import json +import math +import os +import re +import secrets +import stat +import subprocess +import sys +import threading +from collections.abc import Sequence +from contextlib import contextmanager +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import urlparse + +import yaml +from jsonschema.validators import validator_for +from linkml.generators.jsonschemagen import JsonSchemaGenerator +from linkml_runtime.linkml_model.meta import SchemaDefinition + +VERSION = "1.0.0" +REPORT_ROOT = "reviews/structured" +SCHEMA_PATH = Path(__file__).resolve().parents[1] / "schema/record_review.yaml" +OPEN_STATUSES = frozenset({"open", "deferred"}) + + +class ReviewError(ValueError): + """A review cannot be trusted as a structured observation.""" + + +class _ReviewDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + + +def _dump_document(review: dict) -> str: + return yaml.dump(review, Dumper=_ReviewDumper, allow_unicode=True, sort_keys=False) + + +class _ReviewLoader(yaml.SafeLoader): + yaml_implicit_resolvers = { + key: [item for item in values if item[0] != "tag:yaml.org,2002:timestamp"] + for key, values in yaml.SafeLoader.yaml_implicit_resolvers.items() + } + yaml_path_resolvers: dict[Any, Any] = {} + + def compose_node(self, parent, index): + event = self.peek_event() + if (self.check_event(yaml.AliasEvent) or getattr(event, "anchor", None) + or getattr(event, "tag", None)): + raise ReviewError("review YAML must not contain aliases, anchors, or explicit tags") + return super().compose_node(parent, index) + + def construct_mapping(self, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, str) or key in result: + raise ReviewError(f"duplicate or non-string YAML key: {key!r}") + result[key] = self.construct_object(value_node, deep=deep) + return result + + +class _LinkMLLoader(yaml.SafeLoader): + """Keep LinkML parsing independent of C-loader import order and foreign resolvers.""" + + yaml_path_resolvers: dict[Any, Any] = {} + + def construct_mapping(self, node, deep=False): + self.flatten_mapping(node) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if key in mapping: + raise ReviewError(f"duplicate LinkML YAML key: {key!r}") + mapping[key] = self.construct_object(value_node, deep=deep) + return mapping + + +_linkml_yaml_loader = importlib.import_module("linkml_runtime.loaders.yaml_loader") +# Vendored copies and the research contract mutate the same process-global loader. +_LINKML_LOADER_LOCK = vars(_linkml_yaml_loader).setdefault( + "_claw_linkml_loader_lock", threading.RLock() +) + + +@contextmanager +def _pure_linkml_loader(): + loader = _linkml_yaml_loader + with _LINKML_LOADER_LOCK: + original = loader.DupCheckYamlLoader + loader.DupCheckYamlLoader = _LinkMLLoader + try: + yield + finally: + loader.DupCheckYamlLoader = original + + +def load_document(raw: str | bytes) -> dict: + """Read YAML/JSON with duplicate-key and non-finite-number rejection.""" + try: + data = yaml.load(raw, Loader=_ReviewLoader) + except (yaml.YAMLError, UnicodeError, RecursionError) as exc: + raise ReviewError(f"invalid review YAML: {exc}") from exc + if not isinstance(data, dict): + raise ReviewError("document must contain one mapping") + try: + json.dumps(data, allow_nan=False) + except (TypeError, ValueError, RecursionError) as exc: + raise ReviewError(f"document must contain finite JSON-compatible values: {exc}") from exc + return data + + +def relative_path(value: str) -> PurePosixPath: + """Accept only unambiguous repository-relative file paths, not locators.""" + if (not isinstance(value, str) or not value or value.startswith("/") + or "\\" in value or any(ord(c) < 32 for c in value) + or any(c in value for c in ":?#%") + or any(part in {"", ".", ".."} for part in value.split("/"))): + raise ReviewError(f"unsafe repository-relative path: {value!r}") + return PurePosixPath(value) + + +def timestamp(value: str) -> datetime: + if not isinstance(value, str) or not re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z", value + ): + raise ReviewError("timestamps must be explicit ISO 8601 UTC strings ending in Z") + try: + return datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise ReviewError(f"invalid UTC timestamp: {value}") from exc + + +@lru_cache(maxsize=8) +def _schema_validator(schema_text: str): + schema = load_document(schema_text) + if schema.get("imports"): + raise ReviewError("review schema must be self-contained; remote imports are forbidden") + with _pure_linkml_loader(): + generated = json.loads(JsonSchemaGenerator( + SchemaDefinition(**schema), topClass="RecordReview", not_closed=False, + include_null=False, + ).serialize()) + validator = validator_for(generated) + validator.check_schema(generated) + return validator(generated) + + +def _unique(values: list, label: str) -> set: + if len(values) != len(set(values)): + raise ReviewError(f"{label} contains duplicates") + return set(values) + + +def _references(values: list, allowed: set, label: str) -> None: + selected = _unique(values, label) + if not selected <= allowed: + raise ReviewError(f"{label} contains dangling references: {sorted(selected - allowed)}") + + +def _owners(item: dict, label: str) -> None: + owners = item.get("owner_paths", []) + for owner in owners: + relative_path(owner["path"]) + if not owners and not item.get("ownership_note"): + raise ReviewError(f"{label} requires maintained owner paths or an explicit ownership limit") + + +def _acyclic(actions: list[dict]) -> None: + pending = {item["action_id"]: set(item.get("depends_on", [])) for item in actions} + while pending: + ready = {key for key, dependencies in pending.items() if not dependencies} + if not ready: + raise ReviewError("action dependencies contain a cycle") + pending = {key: dependencies - ready for key, dependencies in pending.items() + if key not in ready} + + +def validate_review(review: dict, schema_path: Path = SCHEMA_PATH) -> None: + """Validate shape and semantics only, without reading current target files. + + Historical reviews remain valid observations when records later change. + Persistence separately binds the supplied provenance to the current bytes. + """ + try: + json.dumps(review, allow_nan=False) + validator = _schema_validator(schema_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, RecursionError) as exc: + raise ReviewError(f"cannot validate review: {exc}") from exc + errors = sorted(validator.iter_errors(review), key=lambda error: str(list(error.path))) + if errors: + raise ReviewError("; ".join( + f"{'.'.join(map(str, error.path)) or ''}: {error.message}" + for error in errors[:20] + )) + start, finish = timestamp(review["started_at"]), timestamp(review["finished_at"]) + if finish < start: + raise ReviewError("review finished_at precedes started_at") + if not review["review_id"].startswith(finish.strftime("%Y%m%dT%H%M%SZ") + "-"): + raise ReviewError("review_id must begin with the finished UTC timestamp and a hyphen") + targets = _unique([item["target_id"] for item in review["targets"]], "target IDs") + _unique([(item["path"], item.get("selector")) for item in review["targets"]], "target locators") + inputs = _unique([item["path"] for item in review["source"]["inputs"]], "input paths") + for name in inputs: + relative_path(name) + for item in review["targets"]: + relative_path(item["path"]) + if item["path"] not in inputs: + raise ReviewError("each target path must identify a hashed source input") + if item["kind"] == "snapshot_row" and not item.get("selector"): + raise ReviewError("snapshot-row targets require an exact selector") + _owners(item, f"target {item['target_id']}") + scope = review["scope"] + _references(scope["reviewed_target_ids"], targets, "reviewed target IDs") + if scope["population_size"] < len(targets): + raise ReviewError("population_size is smaller than the explicitly selected targets") + if scope["coverage"] == "full" and scope["population_size"] != len(scope["reviewed_target_ids"]): + raise ReviewError("full coverage requires every population member to be reviewed") + if scope["coverage"] == "sampled" and not scope.get("sampling_method"): + raise ReviewError("sampled reviews require a sampling method") + if review["kind"] == "record" and (len(targets) != 1 or scope["population_size"] != 1): + raise ReviewError("a record review must identify exactly one record") + if review["completion"] == "completed" and set(scope["reviewed_target_ids"]) != targets: + raise ReviewError("completed review has explicitly selected targets left unreviewed") + if review["kind"] == "category" and not review.get("boundary_decisions"): + raise ReviewError("category reviews require explicit lump/split/retain/defer decisions") + evidence = _unique([item["evidence_id"] for item in review["evidence"]], "evidence IDs") + _unique([item["check_id"] for item in review["checks"]], "check IDs") + _unique([item["assessment_id"] for item in review["assessments"]], "assessment IDs") + findings = _unique([item["finding_id"] for item in review["findings"]], "finding IDs") + _unique([item["issue_key"] for item in review["findings"]], "issue keys within a review") + actions = _unique([item["action_id"] for item in review["actions"]], "action IDs") + for group in (review["checks"], review["assessments"], review["findings"], + review["actions"], review.get("boundary_decisions", [])): + for item in group: + _references(item["target_ids"], targets, "target references") + _references(item.get("evidence_ids", []), evidence, "evidence references") + for item in review["evidence"]: + if timestamp(item["accessed_at"]) > finish: + raise ReviewError("evidence cannot be accessed after the review finished") + if item["kind"] == "search" and not item.get("search_scope"): + raise ReviewError("search evidence requires a bounded search scope") + unavailable_required = [] + failed_required = [] + for check in review["checks"]: + status = check["status"] + if status in {"skipped", "unavailable", "not_applicable"} and ( + "exit_code" in check or "expected_exit_code" in check + ): + raise ReviewError("an unexecuted check cannot have an exit code") + if check.get("command") and status in {"passed", "failed"} and "exit_code" not in check: + raise ReviewError("executed command checks require their actual exit code") + if status == "passed" and check.get("exit_code", 0) != check.get("expected_exit_code", 0): + raise ReviewError("a passed command cannot have a failing exit code") + if "expected_exit_code" in check and "exit_code" not in check: + raise ReviewError("expected exit codes require an actual command result") + if check["required"] and status in {"skipped", "unavailable"}: + unavailable_required.append(check["check_id"]) + if check["required"] and status == "failed": + failed_required.append(check["check_id"]) + if unavailable_required and review["completion"] == "completed": + raise ReviewError("unavailable required checks cannot be a completed review") + if (review["completion"] != "completed" or scope["coverage"] != "full" + or unavailable_required) and not review["limitations"]: + raise ReviewError("partial, failed, or sampled reviews must state their limitations") + for assessment in review["assessments"]: + if assessment["outcome"] == "supported" and not assessment["evidence_ids"]: + raise ReviewError("supported assessments require inspected evidence") + dimensions = assessment.get("dimensions", []) + _unique([item["name"] for item in dimensions], "assessment dimension names") + for dimension in dimensions: + _references(dimension["evidence_ids"], evidence, "dimension evidence references") + for metric in assessment.get("metrics", []): + value = metric["value"] + if not math.isfinite(value): + raise ReviewError("metrics must be finite") + if metric.get("minimum", value) > value or metric.get("maximum", value) < value: + raise ReviewError("metric value is outside its declared scale") + assessed = {target for item in review["assessments"] for target in item["target_ids"]} + if set(scope["reviewed_target_ids"]) - assessed: + raise ReviewError("every reviewed target requires an explicit scoped assessment") + disposition_assessed = {target for item in review["assessments"] + if item["outcome"] in {"supported", "concern"} and item["evidence_ids"] + for target in item["target_ids"]} + open_severe = [] + for finding in review["findings"]: + _owners(finding, f"finding {finding['finding_id']}") + if finding.get("native_severity") and not finding.get("normalization_reason"): + raise ReviewError("native severity requires an explicit normalization reason") + if finding["status"] != "open" and not finding.get("disposition_reason"): + raise ReviewError("non-open findings require a disposition reason") + if finding["status"] in {"resolved", "rejected", "accepted_risk"}: + if not finding.get("previous_occurrences"): + raise ReviewError("closed dispositions require an explicit previous finding") + if (review["completion"] == "failed" + or not set(finding["target_ids"]) <= set(scope["reviewed_target_ids"]) + or not set(finding["target_ids"]) <= disposition_assessed): + raise ReviewError("terminal findings require actually reviewed, evidence-assessed targets") + predecessors = finding.get("previous_occurrences", []) + _unique([(p["repository"].lower(), p["review_id"], p["finding_id"]) for p in predecessors], + "previous occurrences") + for previous in predecessors: + if (previous["repository"].lower() != review["repository"].lower() + or previous["review_id"] == review["review_id"]): + raise ReviewError("previous_occurrences must identify an earlier review in this repository") + if finding["status"] in OPEN_STATUSES and finding["severity"] in {"blocker", "major"}: + open_severe.append(finding["finding_id"]) + for action in review["actions"]: + _owners(action, f"action {action['action_id']}") + _references(action["finding_ids"], findings, "action finding references") + _references(action.get("depends_on", []), actions, "action dependencies") + _acyclic(review["actions"]) + actionable = {f["finding_id"] for f in review["findings"] + if f["status"] == "open" and f["severity"] != "informational"} + planned = {finding_id for action in review["actions"] for finding_id in action["finding_ids"]} + if actionable - planned: + raise ReviewError("open findings require proposed actions and acceptance checks") + for decision in review.get("boundary_decisions", []): + members = set(decision["target_ids"]) + proposed = decision.get("proposed_groups", []) + for group in proposed: + _references(group["target_ids"], members, "proposed group members") + if decision["action"] == "split": + flattened = [target for group in proposed for target in group["target_ids"]] + if len(proposed) < 2 or _unique(flattened, "split members") != members: + raise ReviewError("a split requires a complete, disjoint partition into at least two groups") + if decision["action"] == "lump" and len(members) < 2: + raise ReviewError("a lump requires at least two targets") + if decision["action"] in {"lump", "split"} and not decision["evidence_ids"]: + raise ReviewError("lump and split decisions require evidence") + if review["verdict"] in {"pass", "pass_with_limitations"} and (open_severe or failed_required): + raise ReviewError("a passing verdict cannot conceal unresolved major/blocker findings or failed required checks") + if review["verdict"] == "pass" and ( + review["completion"] != "completed" or unavailable_required + or any(f["status"] in OPEN_STATUSES for f in review["findings"]) + ): + raise ReviewError("pass requires a completed review with no unresolved findings") + if review["completion"] == "failed" and review["verdict"] not in {"blocked", "not_assessed"}: + raise ReviewError("a failed review cannot carry an assessed verdict") + if review["scientific_review"] and ( + review["verdict"] == "seed_only" or review["reviewer"]["kind"] == "deterministic" + ): + raise ReviewError("seed metadata and deterministic validation are not scientific review") + if review["verdict"] == "pass_with_limitations" and not review["limitations"]: + raise ReviewError("pass_with_limitations requires explicit limitations") + if review["verdict"] in {"pass", "pass_with_limitations"}: + supportive_evidence = {e["evidence_id"] for e in review["evidence"] if e["support"] == "supports"} + positive = {target for item in review["assessments"] + if item["outcome"] == "supported" and supportive_evidence.intersection(item["evidence_ids"]) + for target in item["target_ids"]} + if not scope["reviewed_target_ids"] or not set(scope["reviewed_target_ids"]) <= positive: + raise ReviewError("passing verdicts require positive evidence-linked assessment of reviewed targets") + if review["verdict"] == "pass" and any( + item["outcome"] in {"unknown", "concern"} for item in review["assessments"] + ): + raise ReviewError("pass cannot conceal unknown or concerning assessments; qualify the verdict") + + +def content_sha256(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +@contextmanager +def _directory(root: Path, relative: str | None = None, *, create: bool = False): + """Pin every path component to a no-follow directory descriptor.""" + root = root.absolute() + if ".." in root.parts: + raise ReviewError("repository root must not contain parent traversal") + fd = os.open(root.anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for component in root.parts[1:]: + child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + if relative is not None: + for component in relative_path(relative).parts: + if create: + try: + os.mkdir(component, dir_fd=fd) + except FileExistsError: + pass + child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + yield fd + finally: + os.close(fd) + + +@contextmanager +def _input(root: Path, relative: str): + path = relative_path(relative) + parent = str(path.parent) if str(path.parent) != "." else None + with _directory(root, parent) as directory: + fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + if not stat.S_ISREG(os.fstat(fd).st_mode): + os.close(fd) + raise ReviewError(f"source must be a regular file: {relative}") + with os.fdopen(fd, "rb") as stream: + yield stream + + +def read_bytes(root: Path, relative: str, *, max_bytes: int = 32 * 1024 * 1024) -> bytes: + with _input(root, relative) as stream: + raw = stream.read(max_bytes + 1) + if len(raw) > max_bytes: + raise ReviewError(f"review document exceeds the {max_bytes} byte limit: {relative}") + return raw + + +def file_sha256(root: Path, relative: str) -> str: + digest = hashlib.sha256() + with _input(root, relative) as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _git(root: Path, *args: str) -> bytes: + try: + return subprocess.check_output(["git", "-C", str(root), *args], + stderr=subprocess.PIPE, timeout=60, + env={**os.environ, "GIT_NO_LAZY_FETCH": "1", + "GIT_TERMINAL_PROMPT": "0"}) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise ReviewError(f"Git provenance check failed: {' '.join(args)}") from exc + + +def repository_identity(root: Path) -> str: + with _directory(root): + pass + top = Path(_git(root, "rev-parse", "--show-toplevel").decode().strip()) + if top != root.absolute(): + raise ReviewError("repo-root must be the exact Git worktree root") + remote = _git(root, "remote", "get-url", "origin").decode().strip() + if remote.startswith("git@github.com:"): + identity = remote.removeprefix("git@github.com:") + else: + parsed = urlparse(remote) + if (parsed.scheme != "https" or parsed.netloc != "github.com" + or parsed.query or parsed.fragment): + raise ReviewError("origin must identify an explicit GitHub repository") + identity = parsed.path.lstrip("/") + identity = identity.removesuffix(".git") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]*/[A-Za-z0-9_.-]+", identity): + raise ReviewError("invalid GitHub origin identity") + return identity + + +def inspect_source(root: Path, targets: list[dict], extra_inputs: Sequence[str] = ()) -> dict: + """Capture working inputs; this is not a saved or scientifically assessed review.""" + repository = repository_identity(root) + before = _git(root, "rev-parse", "HEAD").decode().strip() + paths = sorted({item["path"] for item in targets} | set(extra_inputs)) + if not paths: + raise ReviewError("inspection requires at least one input") + inputs = [{"path": name, "sha256": file_sha256(root, name), + "role": "target" if any(t["path"] == name for t in targets) else "context"} + for name in paths] + source = {"git_revision": before, "state": "working_tree", "inputs": inputs} + assert_current_source(root, {"repository": repository, "source": source}) + return {"status": "inspection_only", "repository": repository, "source": source, + "targets": targets} + + +def assert_current_source(root: Path, review: dict) -> None: + if repository_identity(root).lower() != review["repository"].lower(): + raise ReviewError("review repository does not match the worktree origin") + source = review["source"] + if source["state"] == "working_tree": + if _git(root, "rev-parse", "HEAD").decode().strip() != source["git_revision"]: + raise ReviewError("working-tree base changed; inspect and reassess") + for item in source["inputs"]: + if file_sha256(root, item["path"]) != item["sha256"]: + raise ReviewError(f"reviewed input changed: {item['path']}") + else: + revision = source["git_revision"] + for item in source["inputs"]: + relative_path(item["path"]) + mode = _git(root, "ls-tree", "--format=%(objectmode)", revision, "--", item["path"]) + if mode.strip() not in {b"100644", b"100755"}: + raise ReviewError(f"Git input is not a regular tracked file: {item['path']}") + raw = _git(root, "cat-file", "blob", f"{revision}:{item['path']}") + if content_sha256(raw) != item["sha256"]: + raise ReviewError(f"committed input digest disagrees: {item['path']}") + + +def _text(value: Any) -> str: + return html.escape(str(value), quote=False).replace("|", "|").replace("`", "`") + + +def _table(headers: list[str], rows: list[list]) -> list[str]: + lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |"] + for row in rows: + lines.append("| " + " | ".join(_text(value).replace("\n", "
") for value in row) + " |") + return lines + + +def render_markdown(review: dict, schema_path: Path = SCHEMA_PATH) -> str: + """Derive a human report from the same validated data, never a second verdict.""" + validate_review(review, schema_path) + source, scope = review["source"], review["scope"] + lines = [f"# {_text(review['title'])}", "", f"- Review: {_text(review['review_id'])}", + f"- Repository: {_text(review['repository'])}", + f"- Started UTC: {review['started_at']}", f"- Finished UTC: {review['finished_at']}", + f"- Reviewer: {_text(review['reviewer']['identity'])} ({review['reviewer']['independence']})", + f"- Completion: {review['completion']}", f"- Verdict: {review['verdict']}", + f"- Scientific review: {str(review['scientific_review']).lower()}", + "", "## Summary", "", _text(review["summary"]), "", "## Scope And Provenance", "", + _text(scope["description"]), "", f"Selection: {_text(scope['selection'])}", + f"Coverage: {scope['coverage']}; {len(scope['reviewed_target_ids'])} reviewed / " + f"{scope['population_size']} in the declared population.", + f"Source: {source['state']} at Git base {source['git_revision']}.", + "Working-tree hashes do not imply those bytes were committed.", ""] + lines += _table(["Target", "Path / selector", "Kind", "Label"], [ + [t["target_id"], t["path"] + (" # " + t["selector"] if t.get("selector") else ""), + t["kind"], t["label"]] for t in review["targets"]]) + lines += ["", "## Validation", ""] + lines += _table(["Check", "Status", "Required", "Targets", "Result"], [ + [c["name"], c["status"], c["required"], ", ".join(c["target_ids"]), c["summary"]] + for c in review["checks"]]) + lines += ["", "## Scientific And Domain Assessments", ""] + for item in review["assessments"]: + lines += [f"### {_text(item['topic'])}", "", + f"{item['area']}: {item['outcome']}. Targets: {_text(', '.join(item['target_ids']))}.", + "", _text(item["summary"]), ""] + if item.get("details"): + lines += [_text(item["details"]), ""] + lines += ["## Findings", ""] + if not review["findings"]: + lines += ["No findings recorded within this review's declared scope.", ""] + for item in review["findings"]: + lines += [f"### {_text(item['finding_id'])}: {_text(item['title'])}", "", + f"{item['severity']} / {item['status']} / {item['certainty']}; " + f"issue key: {_text(item['issue_key'])}.", "", _text(item["description"]), ""] + if item.get("disposition_reason"): + lines += ["Disposition: " + _text(item["disposition_reason"]), ""] + lines += ["## Recommended Actions And Acceptance Checks", ""] + for item in review["actions"]: + lines += [f"### {_text(item['action_id'])}", "", _text(item["description"]), ""] + lines += ["- " + _text(check) for check in item["acceptance_checks"]] + lines += [""] + lines += ["## Category Boundaries", ""] + for item in review.get("boundary_decisions", []): + lines += [f"- {item['action']}: {_text(', '.join(item['target_ids']))}. " + _text(item["rationale"])] + lines += ["", "## Evidence", ""] + lines += _table(["Evidence", "Reference / locator", "Support", "Observation"], [ + [e["evidence_id"], e["reference"] + ("; " + e["locator"] if e.get("locator") else ""), + e["support"], e["summary"]] for e in review["evidence"]]) + lines += ["", "## Limits And Additional Notes", ""] + lines += ["- " + _text(note) for note in review["limitations"] + review.get("notes", [])] + raw = _dump_document(review) + fence = "`" * max(3, 1 + max((len(m.group()) for m in re.finditer(r"`+", raw)), default=0)) + lines += ["", "## Complete Structured Record", "", "The sibling review.yaml is authoritative.", + "", fence + "yaml", raw.rstrip(), fence, ""] + return "\n".join(lines) + + +def _write_new(directory: int, name: str, data: bytes) -> None: + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644, dir_fd=directory) + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + + +def save_review(root: Path, review: dict, schema_path: Path = SCHEMA_PATH) -> Path: + """Append one YAML/Markdown bundle; review.yaml is the atomic commit marker.""" + validate_review(review, schema_path) + if timestamp(review["finished_at"]) > datetime.now(timezone.utc): + raise ReviewError("cannot save a review with a future completion time") + assert_current_source(root, review) + raw = _dump_document(review).encode("utf-8") + markdown = render_markdown(review, schema_path).encode("utf-8") + if max(len(raw), len(markdown)) > 32 * 1024 * 1024: + raise ReviewError("review document exceeds the byte limit") + name = review["review_id"] + relative = f"{REPORT_ROOT}/{name}" + try: + with _directory(root, REPORT_ROOT): + pass + except FileNotFoundError: + pass + # Gitignore checks are prospective: silent local-only reports cannot be fleet evidence. + ignored = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--stdin", "-z"], + input=f"{relative}/review.yaml\0{relative}/review.md\0".encode(), + capture_output=True, timeout=30) + if ignored.returncode not in {0, 1}: + raise ReviewError("could not establish review artifact Git visibility") + if ignored.returncode == 0: + raise ReviewError("structured review artifacts are ignored; adopt the review path before saving") + with _directory(root, REPORT_ROOT, create=True) as parent: + try: + os.mkdir(name, dir_fd=parent) + except FileExistsError as exc: + raise ReviewError("review ID already exists; previous reviews are immutable") from exc + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + temporary = ".review-" + secrets.token_hex(12) + published = False + try: + _write_new(child, "review.md", markdown) + _write_new(child, temporary, raw) + assert_current_source(root, review) + os.link(temporary, "review.yaml", src_dir_fd=child, dst_dir_fd=child, + follow_symlinks=False) + published = True + os.unlink(temporary, dir_fd=child) + os.fsync(child) + os.fsync(parent) + finally: + if not published: + for owned in (temporary, "review.md"): + try: + os.unlink(owned, dir_fd=child) + except FileNotFoundError: + pass + os.close(child) + if not published: + os.rmdir(name, dir_fd=parent) + return root.absolute() / relative / "review.yaml" + + +def parse_review_bundle( + relative: str, raw: bytes, markdown: bytes, schema_path: Path = SCHEMA_PATH, +) -> dict: + """Validate a captured pair from disk or an immutable Git tree.""" + path = relative_path(relative) + if (len(path.parts) != 4 or path.parts[:2] != tuple(REPORT_ROOT.split("/")) + or path.name != "review.yaml"): + raise ReviewError("review must identify a canonical structured bundle") + review = load_document(raw) + validate_review(review, schema_path) + if path.parent.name != review["review_id"]: + raise ReviewError("review directory and review ID disagree") + if markdown.decode("utf-8") != render_markdown(review, schema_path): + raise ReviewError("Markdown does not match the authoritative structured review") + return review + + +def read_review(root: Path, relative: str, schema_path: Path = SCHEMA_PATH) -> dict: + path = relative_path(relative) + review = parse_review_bundle(relative, read_bytes(root, relative), + read_bytes(root, str(path.with_name("review.md"))), schema_path) + provenance = source_provenance(root, review) + if provenance["status"] in {"invalid", "unverified"}: + raise ReviewError(f"source provenance {provenance['status']}: {provenance['reason']}") + return review + + +def source_provenance(root: Path, review: dict) -> dict: + """Verify retained Git provenance without requiring historical bytes to be current.""" + source = review["source"] + revision = source["git_revision"] + try: + _git(root, "rev-parse", "--verify", f"{revision}^{{commit}}") + if source["state"] == "working_tree": + return {"status": "working_tree_attestation", + "reason": "Git base exists; dirty input hashes are a reviewer attestation, not committed-byte proof."} + for item in source["inputs"]: + mode = _git(root, "ls-tree", "--format=%(objectmode)", revision, "--", item["path"]) + if mode.strip() not in {b"100644", b"100755"}: + return {"status": "invalid", "reason": f"claimed committed input is absent or nonregular: {item['path']}"} + raw = _git(root, "cat-file", "blob", f"{revision}:{item['path']}") + if content_sha256(raw) != item["sha256"]: + return {"status": "invalid", "reason": f"claimed committed input hash disagrees: {item['path']}"} + return {"status": "git_commit_verified", "reason": "Historical regular-file blobs match the declared hashes."} + except (OSError, ReviewError) as exc: + return {"status": "unverified", "reason": str(exc)} + + +def review_paths(root: Path) -> list[str]: + """Include hidden/ignored bundles; refuse symlinks or incomplete publication.""" + try: + with _directory(root, REPORT_ROOT) as parent: + entries = sorted(os.listdir(parent)) + except FileNotFoundError: + return [] + paths = [] + for name in entries: + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,159}", name): + raise ReviewError(f"unexpected entry in structured reviews: {name}") + relative = f"{REPORT_ROOT}/{name}" + with _directory(root, relative) as directory: + if sorted(os.listdir(directory)) != ["review.md", "review.yaml"]: + raise ReviewError(f"incomplete or unexpected review bundle: {relative}") + paths.append(f"{relative}/review.yaml") + return paths + + +def assert_append_only(root: Path, base: str) -> None: + """Compare actual retained bytes with a trusted Git base, including dirty edits.""" + revision = _git(root, "rev-parse", "--verify", "--end-of-options", f"{base}^{{commit}}").decode().strip() + entries = _git(root, "ls-tree", "-r", "-z", revision, "--", REPORT_ROOT) + for entry in entries.split(b"\0"): + if not entry: + continue + metadata, encoded_path = entry.split(b"\t", 1) + mode, kind, object_id = metadata.split(b" ") + relative = encoded_path.decode("utf-8") + relative_path(relative) + if mode not in {b"100644", b"100755"} or kind != b"blob": + raise ReviewError(f"review baseline contains a nonregular entry: {relative}") + size = int(_git(root, "cat-file", "-s", object_id.decode()).strip()) + if size > 32 * 1024 * 1024: + raise ReviewError(f"review baseline exceeds the byte limit: {relative}") + original = _git(root, "cat-file", "blob", object_id.decode()) + try: + current = read_bytes(root, relative) + except OSError as exc: + raise ReviewError(f"structured reviews are append-only; missing or unsafe: {relative}") from exc + if current != original: + raise ReviewError(f"structured reviews are append-only; changed: {relative}") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--schema", type=Path, default=SCHEMA_PATH, + help="Canonical LinkML schema; vendored callers supply its explicit local path.") + commands = parser.add_subparsers(dest="command", required=True) + inspect = commands.add_parser("inspect", help="capture inputs; not a saved review") + inspect.add_argument("--targets", type=Path, required=True, + help="YAML/JSON mapping containing a targets list") + inspect.add_argument("--input", action="append", default=[], help="additional context input path") + validate = commands.add_parser("validate", help="validate supplied structured records, not source freshness") + validate.add_argument("paths", type=Path, nargs="+") + save = commands.add_parser("save", help="append supplied completed content as a YAML/Markdown bundle") + save.add_argument("--content", type=Path, required=True) + render = commands.add_parser("render", help="render supplied structured content to stdout") + render.add_argument("--content", type=Path, required=True) + check = commands.add_parser("check", help="validate all retained bundles, including ignored files") + check.add_argument("--require-reviews", action="store_true") + check.add_argument("--base", default=os.environ.get("RECORD_REVIEW_BASE", "HEAD"), + help="trusted Git base for append-only checks; CI must supply its event base") + commands.add_parser("list", help="emit validated review metadata") + args = parser.parse_args(argv) + try: + if not args.schema.is_file() or args.schema.is_symlink(): + raise ReviewError("supply --schema with the regular canonical record_review.yaml file") + if args.command == "inspect": + spec = load_document(args.targets.read_bytes()) + if set(spec) != {"targets"} or not isinstance(spec["targets"], list) or not spec["targets"]: + raise ReviewError("inspection requires a nonempty targets list") + for target in spec["targets"]: + if not isinstance(target, dict) or not isinstance(target.get("path"), str): + raise ReviewError("inspection targets require explicit repository-relative paths") + print(json.dumps(inspect_source(args.repo_root, spec["targets"], args.input), indent=2)) + elif args.command == "validate": + for path in args.paths: + validate_review(load_document(path.read_bytes()), args.schema) + print(json.dumps({"valid_reviews": len(args.paths)})) + elif args.command in {"save", "render"}: + content = load_document(args.content.read_bytes()) + if args.command == "save": + path = save_review(args.repo_root, content, args.schema) + print(json.dumps({"review": str(path), "markdown": str(path.with_name("review.md"))})) + else: + print(render_markdown(content, args.schema), end="") + else: + identity = repository_identity(args.repo_root) + if args.command == "check": + assert_append_only(args.repo_root, args.base) + paths = review_paths(args.repo_root) + if getattr(args, "require_reviews", False) and not paths: + raise ReviewError("no structured reviews exist; absence is not reviewed coverage") + records = [] + for relative in paths: + review = read_review(args.repo_root, relative, args.schema) + if review["repository"].lower() != identity.lower(): + raise ReviewError(f"foreign review found in {identity}: {relative}") + records.append({"path": relative, **{key: review[key] for key in ( + "review_id", "repository", "kind", "finished_at", "completion", "verdict", + "scientific_review", "scope", "targets", + )}}) + print(json.dumps(records if args.command == "list" else { + "repository": identity, "valid_reviews": len(records), + "coverage": "no_structured_reviews" if not records else "declared_scopes_only", + }, indent=2)) + except (ReviewError, OSError, UnicodeError, TypeError, subprocess.TimeoutExpired) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_record_review_contract.py b/tests/test_record_review_contract.py new file mode 100644 index 000000000..e735ee0c8 --- /dev/null +++ b/tests/test_record_review_contract.py @@ -0,0 +1,114 @@ +"""CLAW-governed contract: declared review skills persist validated observations.""" + +import importlib.util +import os +import subprocess +from copy import deepcopy +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(scope="module") +def reviews(): + spec = importlib.util.spec_from_file_location("shared_record_reviews", ROOT / "scripts/record_review.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_review_skill_routes_and_rubrics_exist(reviews): + profile = reviews.load_document((ROOT / "conf/record_review.yaml").read_bytes()) + assert set(profile) == {"version", "repository", "skills", "rubrics"} + assert profile["version"] == 1 + assert profile["repository"].lower() == reviews.repository_identity(ROOT).lower() + assert profile["skills"] and profile["rubrics"] + for key in ("skills", "rubrics"): + assert isinstance(profile[key], list) + assert len(profile[key]) == len(set(profile[key])) + for relative in profile[key]: + reviews.relative_path(relative) + body = reviews.read_bytes(ROOT, relative).decode("utf-8") + if key == "skills": + assert "docs/record-reviews.md" in body, f"{relative} bypasses the shared output contract" + assert reviews.read_bytes(ROOT, "docs/record-review-profile.md") + assert reviews.read_bytes(ROOT, "docs/record-reviews.md") + + +def test_every_saved_structured_bundle_is_valid(reviews): + identity = reviews.repository_identity(ROOT) + reviews.assert_append_only(ROOT, os.environ.get("RECORD_REVIEW_BASE", "HEAD")) + for path in reviews.review_paths(ROOT): + review = reviews.read_review(ROOT, path) + assert review["repository"].lower() == identity.lower() + + +@pytest.fixture +def sample(tmp_path, reviews): + root = (tmp_path / "synthetic-repository").resolve() + root.mkdir() + for args in (("init",), ("config", "user.name", "Contract fixture"), + ("config", "user.email", "fixture@example.invalid"), + ("remote", "add", "origin", "https://github.com/Example/FixtureMech.git")): + subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True) + (root / "record.yaml").write_text("id: EX:fixture\n") + subprocess.run(["git", "-C", str(root), "add", "record.yaml"], check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "commit", "-m", "Synthetic fixture"], + check=True, capture_output=True) + owner = {"repository": "Example/FixtureMech", "path": "record.yaml", "role": "test fixture"} + targets = [{"target_id": "EX:fixture", "path": "record.yaml", "label": "Synthetic fixture", + "kind": "maintained", "owner_paths": [owner]}] + inspection = reviews.inspect_source(root, targets) + record = { + "schema_version": "1.0.0", "review_id": "20260101T000000Z-synthetic-contract", + "kind": "record", "repository": "Example/FixtureMech", "title": "Synthetic contract test", + "started_at": "2026-01-01T00:00:00Z", "finished_at": "2026-01-01T00:00:00Z", + "reviewer": {"identity": "contract fixture", "kind": "deterministic", + "independence": "not_applicable", "independence_basis": "No scientific review."}, + "skill": "contract-test", "completion": "completed", "verdict": "pass", + "scientific_review": False, "summary": "Only the synthetic fixture is assessed.", + "source": inspection["source"], "targets": targets, + "scope": {"description": "Synthetic identity check.", "selection": "Explicit fixture.", + "coverage": "full", "population_size": 1, "reviewed_target_ids": ["EX:fixture"]}, + "checks": [{"check_id": "id", "name": "Fixture ID", "required": True, "status": "passed", + "summary": "Explicit fixture ID inspected.", "target_ids": ["EX:fixture"], + "evidence_ids": ["fixture"]}], + "evidence": [{"evidence_id": "fixture", "kind": "record_content", "reference": "record.yaml", + "accessed_at": "2026-01-01T00:00:00Z", "support": "supports", + "summary": "The synthetic fixture declares EX:fixture."}], + "assessments": [{"assessment_id": "identity", "area": "identity", "topic": "Fixture identity", + "outcome": "supported", "summary": "Fixture ID matches.", + "target_ids": ["EX:fixture"], "evidence_ids": ["fixture"]}], + "findings": [], "actions": [], "limitations": [], + } + return root, record + + +def test_shared_saver_round_trip_and_immutability(reviews, sample): + root, record = sample + path = reviews.save_review(root, record) + assert reviews.read_review(root, str(path.relative_to(root))) == record + assert path.with_name("review.md").read_text() == reviews.render_markdown(record) + with pytest.raises(reviews.ReviewError, match="immutable"): + reviews.save_review(root, record) + path.with_name("review.md").write_text("An independent, misleading verdict") + with pytest.raises(reviews.ReviewError, match="Markdown does not match"): + reviews.read_review(root, str(path.relative_to(root))) + + +def test_shared_saver_refuses_changed_inputs_and_false_passes(reviews, sample): + root, record = sample + invalid = deepcopy(record) + invalid["checks"][0]["status"] = "failed" + with pytest.raises(reviews.ReviewError, match="passing verdict"): + reviews.validate_review(invalid) + invalid = deepcopy(record) + invalid["scientific_review"] = True + with pytest.raises(reviews.ReviewError, match="not scientific"): + reviews.validate_review(invalid) + (root / "record.yaml").write_text("id: EX:changed\n") + with pytest.raises(reviews.ReviewError, match="input changed"): + reviews.save_review(root, record) + assert not (root / reviews.REPORT_ROOT).exists() diff --git a/tests/test_record_review_routes.py b/tests/test_record_review_routes.py new file mode 100644 index 000000000..8ad7717ab --- /dev/null +++ b/tests/test_record_review_routes.py @@ -0,0 +1,37 @@ +"""Evidence review must use the maintained assessment and persistence route.""" + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def test_curate_audit_saves_review_without_changing_scientific_inputs(): + text = (ROOT / ".claude/skills/curate-yaml-record/SKILL.md").read_text() + metadata = yaml.safe_load(text.split("---", 2)[1]) + assert metadata["metadata"]["version"] == "2.0.0" + assert "preserves scientific inputs and saves a new structured review" in text + + +def test_evidence_review_delegates_without_stale_producers(): + profile = yaml.safe_load((ROOT / "conf/record_review.yaml").read_text()) + path = ".claude/skills/evidence-curation/SKILL.md" + assert path in profile["skills"] + assert ".claude/skills/review-communities/SKILL.md" in profile["skills"] + text = (ROOT / path).read_text() + for required in ( + "../review-communities/SKILL.md", + "docs/record-reviews.md", + "reviews/structured/-/", + "scripts/record_review.py save --content", + "scientific_review: false", + "per-record P1-P4", + "participants, direction, mechanism", + "both native scores", + "explicit authorization", + "just validate-references-explained", + ): + assert required in text + assert "scripts/quick_literature_review.py" not in text + assert "scripts/review_literature.py" not in text diff --git a/tests/test_writers_leave_a_trace.py b/tests/test_writers_leave_a_trace.py index 599fd74a0..2cc4bc97a 100644 --- a/tests/test_writers_leave_a_trace.py +++ b/tests/test_writers_leave_a_trace.py @@ -42,6 +42,10 @@ # Writes YAML, but not a curated community record — so there is no record whose # history it could belong to. _EXEMPT = { + "scripts/record_review.py": ( + "writes immutable YAML/Markdown review bundles under reviews/structured, " + "not kb/communities records; review observations are not curation events" + ), "scripts/cache_supplements.py": ( "writes references_cache/.supplement.md, not kb records — it " "retrieves a paper's supplementary text and never edits a record. It is "