From 6acc5187e2732b7351b4c0cb97be893c61658f25 Mon Sep 17 00:00:00 2001 From: "jerod.wilkerson" <30474318+jerodw@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:53:13 -0600 Subject: [PATCH 1/3] Record the mandate conferred for story-152 --- .harness/history/mandate-log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.harness/history/mandate-log.jsonl b/.harness/history/mandate-log.jsonl index 4e24504..eed3bff 100644 --- a/.harness/history/mandate-log.jsonl +++ b/.harness/history/mandate-log.jsonl @@ -57,3 +57,4 @@ {"story_id": "story-149", "timestamp": "2026-09-15 08:36:13", "conferred_by": "jerod.wilkerson <30474318+jerodw@users.noreply.github.com>", "source_kind": "human", "recorded_by": "l5-plan", "discarded_session_block": false} {"story_id": "story-150", "timestamp": "2026-09-15 12:12:00", "conferred_by": "jerod.wilkerson <30474318+jerodw@users.noreply.github.com>", "source_kind": "human", "recorded_by": "l5-plan", "discarded_session_block": false} {"story_id": "story-151", "timestamp": "2026-09-15 14:59:07", "conferred_by": "jerod.wilkerson <30474318+jerodw@users.noreply.github.com>", "source_kind": "human", "recorded_by": "l5-plan", "discarded_session_block": false} +{"story_id": "story-152", "timestamp": "2026-09-16 09:53:13", "conferred_by": "jerod.wilkerson <30474318+jerodw@users.noreply.github.com>", "source_kind": "human", "recorded_by": "l5-plan", "discarded_session_block": false} From 98427576d53dbd48fa19061ed2e02804477d4265 Mon Sep 17 00:00:00 2001 From: "jerod.wilkerson" <30474318+jerodw@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:53:13 -0600 Subject: [PATCH 2/3] Plan story-152: Every finding routed to the story is fixed, filed or reported, and the run says which --- .harness/stories/story-152.yaml | 147 ++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 .harness/stories/story-152.yaml diff --git a/.harness/stories/story-152.yaml b/.harness/stories/story-152.yaml new file mode 100644 index 0000000..57b8d80 --- /dev/null +++ b/.harness/stories/story-152.yaml @@ -0,0 +1,147 @@ +story: + id: story-152 + title: Every finding routed to the story is fixed, filed or reported, and the run says which + description: | + Planned from the brief filed under 302. + + Since story-147 the pre-stage inspection splits what it finds at + `orchestration/story_inspection.py` `about_the_change`: findings about + files the story did not change are filed as briefs, and findings about the + story's own change are written to the run-directory artifact the verifier + reads. That artifact is read by nothing after the verdict. A finding the + verifier declines to act on -- which its prompt permits, because a finding + can be correct and still be too small to fail a run -- is therefore neither + fixed nor filed, and the inspection's summary line has already reported it + as "answered by the story" before the verifier read it at all. story-151 + lost two findings this way, one of them a defect worth a brief on its own + merits, and `inspection-log.jsonl` records nothing that would let a reader + tell a finding the story answered from one the verifier passed over. + + This story accounts for every routed finding after the verdict. A finding + is acted on when the verifier names its slug in the location of a blocking + issue or a correctable finding it raised from it; the verifier prompts are + asked to name the slug, and the coordinator matches it deterministically. + Every routed finding the verdict does not act on is filed as a brief + through the same `inspection.file_findings` call every other producer files + through -- the same floor, the same cap, the same dedupe and the same + queue -- from the artifact that already holds it, with no second invocation + of the Inspector. The split is reported in terms true at the moment they + are written: the inspection-time line says how many were routed, an event + after the verdict says how many of those were acted on, filed and dropped, + and the re-entry events -- a correction pass, a retry -- say how many of + the findings they carry are among the ones routed to this story. + + The accounting is done at every verdict and once per attempt, so a finding + passed over on attempt 1 is filed then rather than waiting on a retry that + may never re-find it, and the second verdict of one attempt -- the one a + correction pass returns to -- files nothing twice. The verifier's judgement + is not touched: declining a finding stays correct, and what changes is that + declining sends it where a declined finding went before story-147 rather + than deleting it. Slug matching rather than file matching is deliberate: a + correctable finding about a docstring in a file must not count as acting on + a defect the Inspector found in the same file, because that would be the + silent loss this story exists to close, one layer down. A verifier that + acts on a finding without naming its slug gets that finding filed as a + brief anyway -- a duplicate a human closes rather than a loss. + workflow: story-workflow + +tasks: + - Add to orchestration/story_inspection.py an accounting entry point that, given the run directory, the attempt's findings artifact name, the attempt number and the verdict, reads the routed findings, splits them into acted-on and remainder by slug match against the verdict's blocking_issues and correctable_findings locations, files the remainder through inspection.file_findings with the resolved floor and cap, writes an accounting artifact to the run directory keyed by attempt, and reports the split through _say. It is total in the way inspect_before_stage is - returns nothing a caller can branch on for routing, raises on no path, has its whole body under a guard - and it accounts an attempt once, decided by the presence of the accounting artifact. + - Call that entry point from the verifier branch of orchestration/story_coordinator.py after the verdict is archived and before any routing, only where the stage declares an inspection; the counts it records are read back off the accounting artifact for wording the events below and by no routing decision. + - Word the correction-pass-routed event and the verification-failed retry event with how many of the findings they carry are among the ones routed to this story, when any were routed, so a reader of events.log can tell a re-entry spent on the Inspector's reading from one spent on the verifier's own. + - Change _summary in orchestration/story_inspection.py so the inspection-time line says how many findings were routed to the story rather than that the story answered them, and add a post-verdict event that says, of the findings routed, how many were acted on, filed and dropped. + - Declare the accounting record in schemas/cross-run-history.schema.json so it is projected into inspection-log.jsonl beside the inspection's own record, carrying the routed, acted-on, filed and dropped counts, and carrying no cost and no invocation count because it made no invocation. + - Ask the verifier, in prompts/story-verifier.md and prompts/refactor-verifier.md where the inspection findings are rendered, to name the routed finding's slug in the location of any blocking issue or correctable finding it raises from one, and say why - a finding it acts on without naming is filed as a brief. + - Extend tests/test_inspection_findings_reach_the_story.py with the assertions listed under acceptance criteria, driving a run through the module's existing scripted-verdict fixture with no model invoked. + - Document the accounting in .harness/docs/ARCHITECTURE.md as a section after "A finding about the story's own change is answered by that story", and correct that section's sentence about the summary line carrying "answered by the story", which is no longer what the line says. + +acceptance_criteria: + - A run whose verifier passes and acts on none of the findings routed to it files every one of them as a brief through inspection.file_findings, and its events.log carries a line saying how many were routed, that none were acted on and how many were filed - the assertion attempt 1 of story-151 would have failed. + - A routed finding whose slug the verdict names in the location of a blocking issue or a correctable finding is counted as acted on and is not filed; a routed finding the verdict names by file alone, with no slug, is filed. + - The inspection-time summary line no longer says "answered by the story" and instead reports the routed count in words true when written, and the post-verdict line reports the split; the three figures on the two lines - routed, acted on, filed - agree with what the run directory's findings artifact and the outbox hold. + - An attempt is accounted once: a run whose first verdict routes a correction pass and whose second verdict, within the same attempt, passes clean files the remainder exactly once and writes one accounting artifact for that attempt, and a retry that inspects again is accounted again under the next attempt number. + - The correction-pass-routed event says how many of the correctable findings it carries are among the findings routed to this story, and the verification-failed retry event says the same of its blocking issues, in each case only when the attempt routed any findings; a run whose workflow declares no inspection writes those events exactly as before. + - inspection-log.jsonl receives one accounting record per accounted attempt, declared in schemas/cross-run-history.schema.json, carrying the routed, acted-on, filed and dropped counts and no cost_usd and no invocations, so a reader of that log can tell a finding the story answered from one the verifier passed over. + - The accounting makes no invocation of the Inspector: the findings it files come from the run directory artifact the pre-stage inspection wrote, and the agent runner is not called on its path. + - The accounting is total. It returns nothing a coordinator branch routes on, and a failure inside it - an unreadable artifact, a queue that cannot be reached - is printed and recorded and does not change the verdict's routing, the run's status or its exit code. + - about_the_change and the verifier's triage terms are unchanged: the file-level attribution still decides whose a finding is, and a verifier that declines a finding too small to fail the run still passes the run. + - The verifier prompts ask for the slug in the location of a verdict entry raised from a routed finding, and the inspection-findings section of both prompts says a routed finding the verdict does not name is filed as a brief. + - All existing tests pass unchanged, and the new tests invoke no model. + +technical_plan: + implementation_steps: + - In orchestration/story_inspection.py, add accounting_artifact_file(artifact, attempt) in the findings_artifact_file idiom, and account_after_verdict(run_dir, target_root, config, harness_root, story_id, *, artifact, attempt, verdict) -> None, guarded like inspect_before_stage. Its body reads the attempt's findings artifact (an artifact with ran false or no findings writes an accounting record saying nothing was routed and files nothing), collects the slugs the verdict's blocking_issues and correctable_findings name in their location fields, partitions the routed findings into acted-on and remainder by slug, resolves the bound through inspection.bounds(config), files the remainder through inspection.file_findings with the resolved floor and cap, writes the accounting artifact with routed, acted_on, filed and dropped, and reports through _say with a new event kind. Return before doing any of that where the accounting artifact already exists for this attempt. + - Add a small pure function that does the slug partition, so the tests can hold it directly the way partition tests hold about_the_change. + - In _summary, replace the "answered by the story" clause with one that says how many findings were routed to the story; keep the findings count including them so the no-silent-drop property of that line holds. + - In orchestration/story_coordinator.py's verifier branch, after the verdict is archived and before the passed/failed routing, when the stage declares an inspection, call account_after_verdict with the declared artifact name and the attempt; read the accounting artifact back to obtain the routed slugs and count for wording only. + - Extend _correction_pass_routed and the verification-failed append_event message with "K of the M findings routed to this story" when M is non-zero, computed by the same slug rule from the routed slugs; no routing input changes. + - In schemas/cross-run-history.schema.json, add the accounting event kind to inspection-log.jsonl's event enum and declare routed and acted_on as fields beside findings, filed and dropped; extend append_event's optional fields for them on the same omitted-when-None terms the three counts use. + - In prompts/story-verifier.md and prompts/refactor-verifier.md, in the paragraph introducing the inspection findings, ask that any blocking issue or correctable finding raised from a routed finding name that finding's slug in its location, and say that a routed finding the verdict does not name is filed as a brief after the verdict. + - In tests/test_inspection_findings_reach_the_story.py, add the assertions listed under acceptance criteria using the module's Runner and verdict helpers; a verdict helper that names a slug in a location and one that names a file alone. + - In .harness/docs/ARCHITECTURE.md, add the section and correct the sentence at the "The summary line carries answered by the story" paragraph; update the run directory anatomy for the accounting artifact. + likely_file_changes: + - file: orchestration/story_inspection.py + stage: implementer + reason: The accounting entry point, the slug partition, the accounting artifact, and the corrected summary wording. + - file: orchestration/story_coordinator.py + stage: implementer + reason: Calling the accounting from the verifier branch, wording the re-entry events with the routed count, and carrying the new counts through append_event. + - file: schemas/cross-run-history.schema.json + stage: implementer + reason: Declaring the accounting record's kind and fields so it is projected into inspection-log.jsonl. + - file: prompts/story-verifier.md + stage: implementer + reason: Asking the verifier to name a routed finding's slug in the location of a verdict entry raised from it. + - file: prompts/refactor-verifier.md + stage: implementer + reason: The same request, where that prompt renders the inspection findings. + - file: tests/test_inspection_findings_reach_the_story.py + stage: tester + reason: The assertions under acceptance criteria, driven through the module's existing fixture. + - file: .harness/docs/ARCHITECTURE.md + stage: documenter + reason: A section on the accounting and the correction of the sentence that says the summary line reports the story answered the findings. + +scope: + modify: + - orchestration/story_inspection.py + - orchestration/story_coordinator.py + - schemas/cross-run-history.schema.json + - prompts/story-verifier.md + - prompts/refactor-verifier.md + - tests/test_inspection_findings_reach_the_story.py + - .harness/docs/ARCHITECTURE.md + do_not_modify: + - orchestration/inspection.py + - prompts/inspector.md + - schemas/verification-result.schema.json + - schemas/inspection-findings.schema.json + - schemas/story-brief.schema.json + - workflows/ + - about_the_change in orchestration/story_inspection.py and the file-level attribution it makes + +verification_requirements: + - Confirm a run whose verifier passes without naming any routed finding files each of them as a brief and records the split in events.log and inspection-log.jsonl, with no Inspector invocation on the accounting path. + - Confirm a routed finding whose slug a verdict entry names is not filed, and one named by file alone is. + - Confirm one attempt is accounted once across a correction pass and a retried attempt is accounted under its own number. + - Confirm the correction-pass and retry events carry the routed count only when findings were routed, and are unchanged for a workflow declaring no inspection. + - Confirm the inspection-time summary line no longer claims the story answered anything, and that the cross-run schema declares the accounting record. + - Confirm about_the_change and the verifier's triage terms are unchanged and all existing tests pass. + +constraints: + - The coordinator stays deterministic: the accounting reads artifacts and a verdict and calls no model. + - The accounting is total in the way both existing inspection entry points are: no routing decision, run status or exit code depends on it, and its whole body is guarded. + - Filing goes through inspection.file_findings and nowhere else, so the floor, the cap, the dedupe and the queue are the ones every other producer uses. + - The Inspector is not re-invoked to recover a finding; the remainder is filed from the artifact the run directory already holds. + - Every count reported is true at the moment it is written; no line asserts an outcome that has not yet happened. + - No stage name and no artifact name is written into orchestration for this mechanism; the artifact name comes off the stage's inspection declaration as it does today. + - Slug matching against the verdict's location fields is the only rule that decides a finding was acted on; file-level matching is not used for that decision. + +mandate: + source: + kind: human + conferred_at: 2026-09-16 09:53:13 + conferred_by: jerod.wilkerson <30474318+jerodw@users.noreply.github.com> + recorded_by: l5-plan + +brief_key: 302 From b3a73fc9604af34aa57d3850c6469b06e58b3b75 Mon Sep 17 00:00:00 2001 From: "jerod.wilkerson" <30474318+jerodw@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:49:18 -0600 Subject: [PATCH 3/3] story-152: Every finding routed to the story is fixed, filed or reported, and the run says which Implemented by the l5 harness story workflow. --- .harness/docs/ARCHITECTURE.md | 31 +- .harness/history/execution-log.jsonl | 1 + .harness/history/inspection-log.jsonl | 2 + .harness/history/retry-log.jsonl | 1 + orchestration/story_coordinator.py | 80 +- orchestration/story_inspection.py | 363 +++++++- prompts/refactor-verifier.md | 9 +- prompts/story-verifier.md | 9 +- schemas/cross-run-history.schema.json | 16 +- schemas/execution-history.schema.json | 9 + tests/test_execution_history.py | 5 +- ...est_inspection_findings_reach_the_story.py | 876 +++++++++++++++++- 12 files changed, 1360 insertions(+), 42 deletions(-) diff --git a/.harness/docs/ARCHITECTURE.md b/.harness/docs/ARCHITECTURE.md index 11a4ff7..1a92036 100644 --- a/.harness/docs/ARCHITECTURE.md +++ b/.harness/docs/ARCHITECTURE.md @@ -1570,7 +1570,7 @@ A stage learned that it had left a required output unwritten only after its turn A run directory is execution state and is correctly not versioned, so everything the harness knew about an execution goes with the directory when it is deleted — and deleting one is the *documented* way to run a story again, named in the already-ended refusal's own message. story-081 adds a second scope beside it: a collection of append-only records under `history_dir` (default `.harness/history`, versioned), written through `append_event` as events occur rather than assembled as a terminal summary. The two scopes divide by lifetime and by grain. The run directory holds the detail for as long as the run is interesting; the history holds a summary of every execution for as long as retention says. Deleting a run directory is not a loss of the record: `tests/test_cross_run_history.py::test_the_records_survive_deletion_of_the_run_directory` deletes the directory a run produced and reads that run's records out of the logs afterwards, which is the property the whole mechanism exists for. -**The layout is one JSONL file per kind**, `execution-log.jsonl` and `retry-log.jsonl`, because that is append-only with no directory bookkeeping and matches "recorded as they occur". `inspection-log.jsonl` and `adjudication-log.jsonl` were named as **reserved** in the schema's own top-level description and declared nowhere: inspection and adjudication were concepts this harness did not have, and shipping an empty file for a producer that does not exist would be a store that says something it cannot know. A later harness that acquires a producer declares its log as one more property and gets the projection with no change to orchestration. story-100 acquired the first of those producers and did exactly that — `inspection-log.jsonl` is declared beside the others and is no longer named as reserved, `adjudication-log.jsonl` still is — so the sentence above is now something the repository has done rather than something it predicts. See "A completed story is inspected" below. story-101 gave that log a **second producer**: broad mode writes its own line through the same per-log append with an explicit history directory that `mandate-log.jsonl`'s producer uses, because an `l5-inspect` invocation has no run directory to resolve one from. Adding the fields both producers carry was an edit to the declaration and nowhere else, which is what makes the log declaration the single place a record's shape is decided; `story_id` left that log's required list at the same time, since a broad-mode inspection has no story to name. See "An inspection records what it cost" below. +**The layout is one JSONL file per kind**, `execution-log.jsonl` and `retry-log.jsonl`, because that is append-only with no directory bookkeeping and matches "recorded as they occur". `inspection-log.jsonl` and `adjudication-log.jsonl` were named as **reserved** in the schema's own top-level description and declared nowhere: inspection and adjudication were concepts this harness did not have, and shipping an empty file for a producer that does not exist would be a store that says something it cannot know. A later harness that acquires a producer declares its log as one more property and gets the projection with no change to orchestration. story-100 acquired the first of those producers and did exactly that — `inspection-log.jsonl` is declared beside the others and is no longer named as reserved, `adjudication-log.jsonl` still is — so the sentence above is now something the repository has done rather than something it predicts. See "A completed story is inspected" below. story-101 gave that log a **second producer**: broad mode writes its own line through the same per-log append with an explicit history directory that `mandate-log.jsonl`'s producer uses, because an `l5-inspect` invocation has no run directory to resolve one from. Adding the fields both producers carry was an edit to the declaration and nowhere else, which is what makes the log declaration the single place a record's shape is decided; `story_id` left that log's required list at the same time, since a broad-mode inspection has no story to name. See "An inspection records what it cost" below. story-152 gave the same log a **second kind of record** rather than a third producer: `inspection-accounted` joined its enum beside `inspection-completed`, so the accounting a run writes after the verdict is projected beside the inspection it accounts for, and because the kind is not projected the two are told apart by their fields. See "Every finding routed to the story is fixed, filed or reported" below. **A record is a summary, never a copy.** The per-run artifacts hold the detail; a log that duplicated them would inherit their size and lose the queryability that is the only reason to keep it. An execution record carries `story_id`, `status`, `retry_count`, `verifier_outcome` and `timestamp`; a retry record carries `story_id`, `timestamp`, `stage`, `retry_decision`, `retry_category` and `retry_stage`. Three of those are *derived* rather than threaded through the coordinator to serve the projection — the story id from the run directory's own name, the status from the event kind, and the retry count from the retry decisions already present in the run's own execution history, which `append_event` has loaded to number the entry it is appending — so no new value crosses the coordinator for this. Both limits are checked, and both checks carry a control that reddens when the thing it is looking for is planted. @@ -2038,7 +2038,7 @@ The two sections above build an inspection that fires on every completed story a **It makes no commit of its own.** The record it writes is carried by the run's completion, escalation or pause commit, all of which stage the tree. An inspection commit at HEAD from inside the loop would sit where `_complete` reads HEAD to recognise a resumed escalation. The events line, the cost record in that run's `cost.json` and the cross-run `inspection-log.jsonl` record are written exactly as the post-story path writes them, and the cost stays out of `state.entry_cost_usd`, which is the live allowance the run ceiling is compared against. -**The summary line carries `answered by the story: N`.** A reader comparing the findings count against the filed count would otherwise read the difference as a silent drop, which is the one thing every count on that line exists to make impossible — the no-silent-bound rule the sweep, the query and the Inspector already carry, met here in a new place. +**The summary line carries `routed to this story: N`.** A reader comparing the findings count against the filed count would otherwise read the difference as a silent drop, which is the one thing every count on that line exists to make impossible — the no-silent-bound rule the sweep, the query and the Inspector already carry, met here in a new place. story-147 worded the clause `answered by the story`, which asserted an outcome that had not happened yet: the line is written before the verifier has read the findings, and nothing at that moment knows what the verdict will do with them. story-152 reworded it to what is true when it is written, and what became of the routed findings is said after the verdict, by the section below. **Nothing about what an inspection is changed.** The Inspector prompt, the categories, the severity floor, the brief cap, how a scope is computed, the file cap and what a finding must contain are all untouched. Broad `l5-inspect` is untouched too, and for a reason rather than by omission: an inspection invoked against no story has no story for a finding to be answered by, so it goes on filing everything it finds. @@ -2046,6 +2046,22 @@ The two sections above build an inspection that fires on every completed story a **A second producer in that module cost this story a retry, and the cause is worth carrying forward.** story-114's floor was held by a test that read `orchestration/story_inspection.py` and asserted it contained exactly one `file_findings` call. The invariant it exists for is that a producer hands on the floor it resolved instead of partitioning on its own, and adding a second producer that does exactly that turned the assertion red without weakening anything it guards. The test now reads over *every* filing call in the module and every mention of the floor in the function making it, with a planted source carrying both divergences as its control. The rule this is an instance of is the one the prose layer already states about counts: a structural test that asserts how many of a thing a module has, when what it means is that each of them behaves, goes stale the moment a legitimate one is added — so read the property over all of them and let the count be whatever it is. +## Every finding routed to the story is fixed, filed or reported + +The section above hands the Inspector's findings about the story's own change to the verifier and files nothing about them, and the verifier's prompt permits it to decline a finding that is correct but too small to fail a run. Until story-152 nothing read the findings artifact after the verdict, so a declined finding was neither fixed nor filed — it had been *deleted*, by a mechanism built to make sure findings reach someone — and the summary line had already reported it as answered. story-151's run lost two findings this way, one of them a defect worth a brief on its own. story-152 accounts for every routed finding after the verdict: what the verdict acted on is counted as acted on, and everything else is sent where a declined finding went before story-147, to the backlog as a brief. + +**Acted on means the verdict named the slug, and nothing else counts.** `acted_on_by(findings, verdict)` in `orchestration/story_inspection.py` splits the routed findings on whether the `location` of any of the verdict's `blocking_issues` or `correctable_findings` names the finding's `slug` as a whole token — `names_a_slug` tokenises the location so that a slug which is a prefix of another slug is not the other. File-level matching is deliberately not used: a correctable finding about a docstring in a file would otherwise count as acting on a defect the Inspector found in the same file, which is the silent loss this section exists to close, one layer down. The consequence is stated in both verifier prompts where the findings are rendered: an entry raised from a routed finding names that finding's slug in its location, and a finding acted on without naming it is filed anyway — a duplicate a person closes rather than a loss. A finding carrying no slug at all is remainder for the same reason. The pure pieces — `acted_on_by`, `names_a_slug`, `verdict_locations`, `entries_among_the_routed` — are held by the tests directly, the way the partition tests hold `about_the_change`, and `about_the_change` itself is untouched: the file-level attribution still decides *whose* a finding is, and the slug rule decides only what became of it. + +**The remainder is filed through the one filing call, from the artifact already in the run directory.** `account_after_verdict(run_dir, target_root, config, harness_root, story_id, *, artifact, attempt, verdict) -> None` reads the attempt's findings artifact, splits it, resolves the bound through `inspection.bounds(config)`, and hands the remainder to `inspection.file_findings` with the resolved floor and cap — so the severity floor, the brief cap, the named drop reasons, the dedupe and the queue are exactly the ones every other producer uses, and story-114's test that reads over every filing call in the module holds over this one too. No Inspector is invoked: the findings come from what the pre-stage inspection wrote, and the agent runner is not on this path. It is total in the shape both existing entry points are — returns nothing, raises on no path, whole body under a guard — and a failure inside it (an unreadable artifact, a queue that cannot be reached) is printed and recorded and changes neither the verdict's routing, the run's status nor its exit code. The coordinator calls it in the verifier branch after the verdict is archived and before the passed/failed routing, only where the stage's own `inspection` declaration is present, and reads the artifact name off that declaration; the same scan that holds the pre-stage call site to "called once, read from by nothing that routes" holds this one. + +**Once per attempt, decided by a second file.** `accounting_artifact_file(artifact, attempt)` names the accounting record `accounting-of-` — prefixed rather than suffixed, so the findings artifact's own attempt-wildcarded glob does not match it and a reader listing an attempt's findings records does not count its accounting among them. Its presence is the whole of the rule: the second verdict of one attempt, the one a correction pass returns to, finds it and files nothing twice; a retry inspects again and is accounted under the next attempt's number. `inspection_artifacts` globs both names, so a resume carries the accounting into `resumes/resume-K/` beside the findings record for the reason the findings record moves — a resumed entry's attempt 1 would otherwise find it and file nothing for findings it routed. The record carries the routed and acted-on *slugs* as well as the counts, because the re-entry events are worded from it and that wording is decidable only from the slugs. + +**An attempt that routed nothing is not accounted, which is a deviation from the plan and deliberate.** The plan asked for a record saying nothing was routed. A findings artifact that is absent, records `ran` false, or carries no findings now writes no accounting artifact and no event, because doing otherwise would put a line into the `events.log` of every run under a workflow whose file cap is unset — and story-100's property, that the key unset leaves `events.log` byte-for-byte what it was, forbids exactly that. Where findings were routed the artifact and the event are always written, so the once-per-attempt rule holds where there is anything to hold. + +**Three lines, each true when written.** The inspection-time summary says `routed to this story: N`. After the verdict, an event of kind `inspection-accounted` says of the routed findings how many the verdict acted on, how many were filed as briefs and how many dropped, with each drop reason named with its count as the inspection line names its own. And the two re-entry events — the correction-pass-routed line and the verification-failed retry line — carry a clause, composed by `routed_among` in the coordinator through the same `names_a_slug` rule, saying how many of the findings they carry are among the ones routed to this story, so a reader of `events.log` can tell a re-entry spent on the Inspector's reading from one spent on the verifier's own. That clause is empty when nothing was routed, so a workflow declaring no inspection writes both events exactly as before. The accounting reaches `inspection-log.jsonl` as a second kind of record beside the inspection's own: `schemas/cross-run-history.schema.json` names `inspection-accounted` in that log's enum and declares `routed` and `acted_on` beside the counts, with `acted_on` present even at zero because none acted on is an answer; it carries no `mode`, no `cost_usd`, no `invocations` and no `dedupe_ran`, because it made no invocation and asked no query, and the log's description says those fields are how the two kinds are told apart since the kind itself is not projected. `execution-history.schema.json` had to declare the kind and the two fields as well — its enum is closed and its optional properties are held equal to `append_event`'s keywords in both directions — which is the recurring bill this document records for stories 055, 100, 101, 106 and 131, paid again. + +**A filed finding is not shown to the verifier twice.** The dedupe every producer files through runs inside `inspection.inspect_scope`, before `about_the_change` partitions what was found. So a finding the verifier declined on attempt 1 and the accounting filed is, if the retry's inspection finds it again, dropped as already queued before it could be routed — attempt 2 routes nothing about it, is not accounted for it, and the queue holds it once. That is the right outcome for a backlog item, and it means a routed finding gets exactly one reading by the verifier before it becomes the tracker's. + ## The outbox reaches the project board story-105 is where the two halves of the outbox pair finally reach this deployment's own tracker. The queue had briefs sitting pending that landed as issues and nowhere else, and `filed_query_command` was unconfigured, so dedupe could not run and each post-story inspection refiled what the one before it had filed — the failure "The queue is its own dedupe index" above names as the reason its own mechanism was built. Nothing was added to the harness: no script, no module, no configuration key. What changed is the board mechanics in the sync command's template, this target's values in its installed copy, one line of `.harness/config.yaml`, and a repair the board work could not be built on top of. @@ -2551,6 +2567,17 @@ story-139 moves a sorting a person did after filing into the sync branch of `tem reason where no invocation was made; absent from every run under a workflow whose stages declare no inspection + accounting-of-story-inspection-findings-N.json + since story-152, the accounting of what became + of the findings the attempt's inspection routed + to the story — the routed and acted-on slugs and + counts, and how many of the rest were filed as + briefs or dropped — written after the attempt's + first verdict; its presence is what makes an + attempt accounted once; absent from an attempt + that routed nothing, so absent from every run + whose inspection is switched off; carried by a + resume beside the findings record it accounts for output-baseline.json since story-112, the snapshot of this stage's declared outputs taken at stage entry — the stage it was taken for, and each artifact's diff --git a/.harness/history/execution-log.jsonl b/.harness/history/execution-log.jsonl index 92816ec..2519bf4 100644 --- a/.harness/history/execution-log.jsonl +++ b/.harness/history/execution-log.jsonl @@ -86,3 +86,4 @@ {"story_id": "story-151", "status": "resumed", "retry_count": 0, "timestamp": "2026-09-16 07:36:26"} {"story_id": "story-151", "status": "resumed", "retry_count": 0, "timestamp": "2026-09-16 07:36:54"} {"story_id": "story-151", "status": "completed", "retry_count": 1, "timestamp": "2026-09-16 09:07:08"} +{"story_id": "story-152", "status": "completed", "retry_count": 1, "timestamp": "2026-09-16 10:49:18"} diff --git a/.harness/history/inspection-log.jsonl b/.harness/history/inspection-log.jsonl index 3234e0a..365a4f8 100644 --- a/.harness/history/inspection-log.jsonl +++ b/.harness/history/inspection-log.jsonl @@ -45,3 +45,5 @@ {"story_id": "story-150", "mode": "narrow", "cost_usd": 2.4072125, "scope_files": 60, "invocations": 1, "timestamp": "2026-09-15 12:37:54", "findings": 1, "filed": 0, "dropped": 0, "dedupe_ran": true} {"story_id": "story-151", "mode": "narrow", "cost_usd": 2.9910560000000004, "scope_files": 60, "invocations": 1, "timestamp": "2026-09-16 07:55:55", "findings": 1, "filed": 0, "dropped": 0, "dedupe_ran": true} {"story_id": "story-151", "mode": "narrow", "cost_usd": 3.063886, "scope_files": 60, "invocations": 1, "timestamp": "2026-09-16 08:31:20", "findings": 1, "filed": 0, "dropped": 0, "dedupe_ran": true} +{"story_id": "story-152", "mode": "narrow", "cost_usd": 3.002376, "scope_files": 60, "invocations": 1, "timestamp": "2026-09-16 10:28:07", "findings": 2, "filed": 0, "dropped": 0, "dedupe_ran": true} +{"story_id": "story-152", "mode": "narrow", "cost_usd": 2.6565825000000003, "scope_files": 60, "invocations": 1, "timestamp": "2026-09-16 10:40:25", "findings": 2, "filed": 0, "dropped": 0, "dedupe_ran": true} diff --git a/.harness/history/retry-log.jsonl b/.harness/history/retry-log.jsonl index f73ed50..cdfd87c 100644 --- a/.harness/history/retry-log.jsonl +++ b/.harness/history/retry-log.jsonl @@ -62,3 +62,4 @@ {"story_id": "story-145", "timestamp": "2026-09-14 13:16:08", "stage": "verifier", "retry_decision": "retry", "retry_category": "implementation", "retry_stage": "implementer"} {"story_id": "story-147", "timestamp": "2026-09-14 18:26:38", "stage": "verifier", "retry_decision": "retry", "retry_category": "validation", "retry_stage": "tester"} {"story_id": "story-151", "timestamp": "2026-09-15 15:47:59", "stage": "tester", "retry_stage": "tester"} +{"story_id": "story-152", "timestamp": "2026-09-16 10:28:50", "stage": "verifier", "retry_decision": "retry", "retry_category": "validation", "retry_stage": "tester"} diff --git a/orchestration/story_coordinator.py b/orchestration/story_coordinator.py index 00a3a71..dfabe35 100644 --- a/orchestration/story_coordinator.py +++ b/orchestration/story_coordinator.py @@ -1119,6 +1119,8 @@ def append_event( scope_files: int | None = None, invocations: int | None = None, dedupe_ran: bool | None = None, + routed: int | None = None, + acted_on: int | None = None, ) -> None: """Append one event in both renderings, from one call. @@ -1182,6 +1184,13 @@ def append_event( # treatment `discarded_session_block` already gets one log over. The # `is not None` filter below is what lets a false through. "dedupe_ran": dedupe_ran, + # The accounting after a verdict: how many findings the pre-stage + # inspection routed to the story, and how many of those the verdict + # acted on. On the same omitted-when-None terms as the counts above; + # present on an accounting entry even where a count is zero, because + # none acted on is an answer. + "routed": routed, + "acted_on": acted_on, } entry.update({key: value for key, value in optional.items() if value is not None}) history.append(entry) @@ -2245,8 +2254,17 @@ def inspection_artifacts( declared = inspection_declaration(stages).get("result") if not declared: return [] - pattern = story_inspection.findings_artifact_file(declared, attempt) - return sorted(path.name for path in run_dir.glob(pattern)) + names: list[str] = [] + # The accounting artifact travels with the findings record it accounts + # for, and for the same reason: its presence is what says an attempt was + # accounted, so one left behind would have the next entry's attempt 1 + # file nothing for findings it routed. + for pattern in ( + story_inspection.findings_artifact_file(declared, attempt), + story_inspection.accounting_artifact_file(declared, attempt), + ): + names.extend(path.name for path in run_dir.glob(pattern)) + return sorted(names) def entry_artifacts(run_dir: Path, stages: list[dict]) -> list[str]: @@ -6013,17 +6031,48 @@ def _correction_pass_routed( findings: list[dict], number: int, budget: int, + routed_slugs=(), ) -> None: + """Record a correction pass, and where its findings came from. + + `routed_slugs` is what the pre-stage inspection routed to this story on + this attempt, read back off the accounting artifact. When any were, the + line says how many of the findings the pass carries are among them — by + the accounting's own slug rule — so a reader of events.log can tell a + re-entry spent on the Inspector's reading from one spent on the verifier's + own. A workflow declaring no inspection routes none and writes the line + exactly as before. + """ append_event( run_dir, f"correction pass {number} of {budget}: verification passed carrying " - f"{len(findings)} correctable finding(s); re-entering at {destination}", + f"{len(findings)} correctable finding(s); re-entering at {destination}" + + routed_among(findings, routed_slugs), kind="correction-pass-routed", stage=stage_name, retry_stage=destination, ) +def routed_among(entries: list[dict], routed_slugs) -> str: + """The clause a re-entry event carries when findings were routed. + + Empty when none were, so an event under a workflow declaring no inspection + is byte-for-byte what it was. Otherwise it says, of the entries the event + carries, how many name a routed finding's slug in their location — the + accounting's rule, reached through `story_inspection` rather than + respelled here, so the event and the accounting cannot count differently. + """ + slugs = tuple(routed_slugs) + if not slugs: + return "" + locations = [entry.get("location") for entry in entries + if isinstance(entry, dict)] + among = story_inspection.entries_among_the_routed(locations, slugs) + return (f"; {among} of them are among the {len(slugs)} findings " + f"routed to this story") + + def _correction_pass_recorded( run_dir: Path, stage_name: str, findings: list[dict], budget: int ) -> None: @@ -9232,6 +9281,26 @@ def elapsed() -> float | None: state.verification_iterations += 1 archive = run_dir / "verification" / f"iteration-{state.verification_iterations}.json" archive.write_text(json.dumps(verdict, indent=2) + "\n", encoding="utf-8") + # Every finding the pre-stage inspection routed to this story is + # accounted for here, after the verdict is archived and before any + # routing: what the verdict acted on is named by slug, and the rest + # is filed as briefs from the artifact already in the run directory. + # Only where the stage declares an inspection, and read off that + # declaration, so no artifact name is written here. The call is a + # bare statement and returns nothing; what it recorded is read back + # off its artifact for wording the two re-entry events below and by + # no routing decision. + routed_slugs: tuple = () + declared_inspection = (stage.get("inspection") or {}).get("result") + if declared_inspection: + story_inspection.account_after_verdict( + run_dir, target_root, config or {}, harness_root, + state.story_id, artifact=declared_inspection, + attempt=attempt, verdict=verdict, + ) + routed_slugs = story_inspection.routed_slugs( + run_dir, declared_inspection, attempt + ) # The artifacts an entry names come off the stage's declared # outputs in the loaded workflow, never a list written here. outputs = stage.get("outputs", []) @@ -9352,6 +9421,7 @@ def elapsed() -> float | None: findings, state.correction_pass_count, budget, + routed_slugs, ) # Nothing a retry spends is spent: retry_count is # untouched, archive_attempt is not reached, and no @@ -9643,7 +9713,9 @@ def elapsed() -> float | None: run_dir, f"verification failed; retry {state.retry_count} of " f"{rules['max_retries']} rerouted to {destination} " - f"for {target}", + f"for {target}" + + routed_among(verdict.get("blocking_issues") or [], + routed_slugs), kind="verification-failed", stage=name, artifacts=outputs, diff --git a/orchestration/story_inspection.py b/orchestration/story_inspection.py index 69b0876..fd8d61e 100644 --- a/orchestration/story_inspection.py +++ b/orchestration/story_inspection.py @@ -59,10 +59,19 @@ with `git add -A`, so a file the inspection agent changed elsewhere in the repository is left in the working tree rather than folded into a commit this module made. + +**A finding routed to the story is accounted for after the verdict.** The +pre-stage inspection hands the findings about the story's own change to the +stage that judges it, and the verdict names what it acted on by slug. Every +routed finding the verdict did not name is filed as a brief from the artifact +already in the run directory — through the same filing call, with no second +invocation — once per attempt, and an event says how many were acted on, filed +and dropped. Declining a finding stays correct; declining no longer deletes it. """ from __future__ import annotations import json +import re import subprocess import sys from dataclasses import dataclass @@ -85,6 +94,13 @@ INSPECTION_EVENT = inspection.INSPECTION_EVENT MODE = inspection.MODE_NARROW +#: The kind the accounting after a verdict is appended under. It is a second +#: record about one attempt's inspection rather than a second inspection, so it +#: has a kind of its own: a reader counting `INSPECTION_EVENT` lines counts +#: inspections, and this line says what became of what one of them routed. +#: Which cross-run log it reaches is the declaration's to say, as above. +ACCOUNTING_EVENT = "inspection-accounted" + #: What a post-story inspection's cost is recorded as in that run's cost.json. #: It is not an attempt at a stage, so it carries no attempt number of its own #: and is recorded at attempt 0; the stage name says what the invocation was @@ -380,7 +396,10 @@ def _say(run_dir: Path, message: str, *, findings: int | None = None, filed: int | None = None, dropped: int | None = None, cost_usd: float | None = None, scope_files: int | None = None, invocations: int | None = None, - dedupe_ran: bool | None = None) -> None: + dedupe_ran: bool | None = None, + kind: str = INSPECTION_EVENT, + routed: int | None = None, + acted_on: int | None = None) -> None: """Append the inspection's record through the coordinator's shared append. Every value goes through that one call, so events.log, the run's structured @@ -390,10 +409,15 @@ def _say(run_dir: Path, message: str, *, findings: int | None = None, which is how an invocation that reported no cost records that it reported none rather than recording a zero. - `dedupe_ran` is not on those terms and is passed by every call: it is a - boolean, so absence would mean either false or a record written before the - field existed, and a record that cannot say which is a record nobody can - query for the runs that inspected without dedupe. + `dedupe_ran` is not on those terms and is passed by every inspection call: + it is a boolean, so absence would mean either false or a record written + before the field existed, and a record that cannot say which is a record + nobody can query for the runs that inspected without dedupe. + + `kind` defaults to the inspection's own event and is overridden by the one + other record this module writes, the accounting after the verdict, which + carries `routed` and `acted_on` and no cost, no invocation count and no + dedupe verdict — it made no invocation and asked no query. Imported inside the body, the idiom the queue module already uses for its own coordinator import: the coordinator imports this module, and a @@ -405,10 +429,12 @@ def _say(run_dir: Path, message: str, *, findings: int | None = None, from story_coordinator import append_event append_event( - Path(run_dir), message, kind=INSPECTION_EVENT, + Path(run_dir), message, kind=kind, findings=findings, filed=filed, dropped=dropped, - mode=MODE, cost_usd=cost_usd, scope_files=scope_files, + mode=MODE if kind == INSPECTION_EVENT else None, + cost_usd=cost_usd, scope_files=scope_files, invocations=invocations, dedupe_ran=dedupe_ran, + routed=routed, acted_on=acted_on, ) except Exception: # noqa: BLE001 - reporting may not become the failure pass @@ -623,6 +649,23 @@ def findings_artifact_file(artifact: str, attempt: int) -> str: return f"{stem}-{attempt}.{suffix}" +def accounting_artifact_file(artifact: str, attempt: int) -> str: + """The name the accounting of one attempt's routed findings is written under. + + Keyed by the attempt for the reason the findings artifact is, and the + presence of this file is the whole of the accounted-once rule: the second + verdict of one attempt — the one a correction pass returns to — finds it + and accounts nothing, and a retry, whose findings artifact is its own, + accounts under the next attempt's name. + + The declared name is *prefixed* rather than suffixed, so this file is not + matched by the findings artifact's own attempt-wildcarded glob and a reader + listing an attempt's findings records does not count its accounting among + them. + """ + return f"accounting-of-{findings_artifact_file(artifact, attempt)}" + + def write_findings(run_dir: Path, artifact: str, attempt: int, story_id: str, *, ran: bool, reason: str = "", findings=()) -> None: """Write what the inspection found about the story's own change. @@ -729,11 +772,14 @@ def _summary(label: str, report, excluded, trimmed, log: str, *, run decided. `to_the_story` is how many findings were about the files the story itself - changed and so were given to the stage rather than filed. It is counted as + changed and so were routed to the stage rather than filed. It is counted as a finding and reported on its own terms, because a reader comparing the findings count against the filed count would otherwise read the difference as a silent drop — which is the one thing every count on this line exists - to make impossible. + to make impossible. The line says they were *routed* and nothing more, + because at the moment it is written nothing has been decided about them; + what the verdict acted on, and what was filed because it did not, is said + after the verdict by `account_after_verdict`. """ findings, filed, dropped = _counts(report) findings += to_the_story @@ -742,7 +788,10 @@ def _summary(label: str, report, excluded, trimmed, log: str, *, f"{filed} filed, {dropped} dropped" ) if to_the_story: - line += f"; answered by the story: {to_the_story}" + # Worded for the moment it is written: the findings have been handed + # to the stage and nothing has yet been decided about them. What the + # verdict does with them is said afterwards, by the accounting. + line += f"; routed to this story: {to_the_story}" reasons = [ inspection.ALREADY_FILED, inspection.ALREADY_FILED_LOCALLY, @@ -1204,3 +1253,297 @@ def _inspect_before_stage(run_dir: Path, target_root: Path, config: dict, ) +# -------------------------------------------------------------------------- +# Accounting for what was routed, after the verdict +# -------------------------------------------------------------------------- +# +# A finding routed to the story is written to the artifact above and read by +# the stage that judges the change — and by nothing after the verdict. A +# finding the verifier declines to act on, which its prompt permits because a +# finding can be correct and still be too small to fail a run, was therefore +# neither fixed nor filed. What follows sends a declined finding where a +# declined finding went before the inspection was moved ahead of the verifier: +# to the backlog, as a brief, through the same call every other producer files +# through. The verifier's judgement is not touched; what changes is that +# declining no longer deletes. + + +#: What a location must name for a routed finding to count as acted on: the +#: finding's slug, as one whole token. Slug matching rather than file matching +#: is deliberate — a correctable finding about a docstring in a file must not +#: count as acting on a defect the Inspector found in the same file, which +#: would be the silent loss this accounting exists to close, one layer down. +_TOKEN_BOUNDARY = re.compile(r"[^A-Za-z0-9_-]+") + + +def _tokens(location) -> frozenset: + """The slug-shaped tokens a verdict entry's location carries.""" + if not isinstance(location, str): + return frozenset() + return frozenset(one for one in _TOKEN_BOUNDARY.split(location) if one) + + +def _slug_of(finding) -> str: + try: + slug = finding.get("slug") + except AttributeError: + return "" + return slug if isinstance(slug, str) else "" + + +def verdict_locations(verdict) -> tuple: + """Every location the verdict names, on a blocking issue or a correctable + finding, as the verdict wrote it. + """ + if not isinstance(verdict, dict): + return () + named: list = [] + for key in ("blocking_issues", "correctable_findings"): + for entry in verdict.get(key) or (): + if isinstance(entry, dict): + named.append(entry.get("location")) + return tuple(named) + + +def names_a_slug(location, slugs) -> bool: + """Whether one verdict entry's location names any of `slugs`. + + The one rule that decides a finding was acted on, shared by the accounting + and by the wording of the re-entry events so the two cannot count + differently. A slug is matched as a whole token of the location — a slug + that is a prefix of another is not the other — and by nothing else: not the + file, not the line, not the finding's title. + """ + return bool(_tokens(location) & frozenset(slugs)) + + +def acted_on_by(findings, verdict) -> tuple[list, list]: + """Split routed findings into what the verdict acted on and the rest. + + Returns `(acted_on, remainder)`, in the order given. A finding is acted on + when the verdict names its slug in the location of a blocking issue or a + correctable finding; every other finding is the remainder, including one + the verdict named by file alone and one carrying no slug at all — a + finding nothing can name is a finding nothing acted on, and filing it is + the loss-free reading. + + Pure over its inputs, so it can be held directly the way the partition + tests hold `about_the_change`. + """ + locations = verdict_locations(verdict) + acted: list = [] + remainder: list = [] + for one in findings: + slug = _slug_of(one) + if slug and any(names_a_slug(location, (slug,)) for location in locations): + acted.append(one) + else: + remainder.append(one) + return acted, remainder + + +def entries_among_the_routed(locations, routed_slugs) -> int: + """How many verdict entries name one of the routed findings' slugs. + + The re-entry events' number: of the findings an event carries, how many are + among the ones routed to this story. Counted over the entries and decided + by `names_a_slug`, so it is the accounting's own rule read from the other + side rather than a second one. + """ + slugs = frozenset(one for one in routed_slugs if isinstance(one, str)) + if not slugs: + return 0 + return sum(1 for location in locations if names_a_slug(location, slugs)) + + +@dataclass(frozen=True) +class _Routed: + """One routed finding on its way to the queue, in the shape + `inspection.file_findings` files: the finding, and where it came from. + """ + + finding: dict + scope: object + + +def _read_findings_record(run_dir: Path, artifact: str, attempt: int): + """The attempt's findings artifact, or None where there is none to read.""" + path = Path(run_dir) / findings_artifact_file(artifact, attempt) + if not path.is_file(): + return None + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return record if isinstance(record, dict) else None + + +def write_accounting(run_dir: Path, artifact: str, attempt: int, + story_id: str, *, routed, acted_on, filed: int, + dropped: int, reason: str = "") -> None: + """Write the accounting artifact for one attempt. + + It carries the slugs and not only the counts, because the re-entry events + are worded from it: a correction pass or a retry says how many of the + findings it carries are among the ones routed, and that is decidable only + from the slugs. Nothing routes on any of it. + """ + try: + path = Path(run_dir) / accounting_artifact_file(artifact, attempt) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "story_id": story_id, + "attempt": attempt, + "routed": len(routed), + "routed_slugs": [_slug_of(one) for one in routed], + "acted_on": len(acted_on), + "acted_on_slugs": [_slug_of(one) for one in acted_on], + "filed": filed, + "dropped": dropped, + "reason": reason, + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + except Exception: # noqa: BLE001 - the totality is the guarantee + pass + + +def routed_slugs(run_dir: Path, artifact: str, attempt: int) -> tuple: + """The slugs the accounting artifact records as routed on one attempt. + + Read back off the artifact rather than carried in memory, so what an + event says about the routed findings and what the run directory records + are one thing. Empty where no attempt was accounted, and read by nothing + that routes: the coordinator words two events from it and decides nothing. + """ + try: + path = Path(run_dir) / accounting_artifact_file(artifact, attempt) + if not path.is_file(): + return () + record = json.loads(path.read_text(encoding="utf-8")) + return tuple( + one for one in record.get("routed_slugs", ()) + if isinstance(one, str) and one + ) + except Exception: # noqa: BLE001 - reading a record may not become a failure + return () + + +# The same guarantee a third time, for the caller that runs after the verdict +# and before any routing. It returns nothing, raises on no path, and declares +# no parameter by which a caller could be told to stop: a failure inside it — +# an unreadable artifact, a queue that cannot be reached — is printed and +# recorded and changes neither the verdict's routing, the run's status nor its +# exit code. +def account_after_verdict(run_dir: Path, target_root: Path, config: dict, + harness_root: Path, story_id: str, *, + artifact: str, attempt: int, verdict) -> None: + """Account for every finding routed to this story, once per attempt. + + Called from the verifier branch after the verdict is archived and before + the passed/failed routing, only where the stage declares an inspection. It + reads the attempt's findings artifact, splits the routed findings into the + ones the verdict acted on — named by slug in the location of a blocking + issue or a correctable finding — and the remainder, files the remainder + through `inspection.file_findings` with the resolved floor and cap, writes + the accounting artifact, and says the split in the run's own record. + + Once per attempt, decided by the presence of the accounting artifact: the + second verdict of one attempt — the one a correction pass returns to — + finds it and files nothing twice, and a retry, whose findings artifact is + its own, is accounted under the next attempt's number. + + An attempt that routed nothing is not accounted: a findings artifact that + is absent, that records `ran` false, or that carries no findings leaves + nothing to act on, nothing to file and nothing to say, so no artifact and + no event are written for it. That is what keeps the events.log of a run + whose inspection was switched off byte-for-byte what it was. + + No Inspector is invoked: the findings it files come from the artifact the + pre-stage inspection already wrote, and the agent runner is not on this + path at all. + """ + label = f"accounting for the findings routed to {story_id}" + try: + _account_after_verdict( + run_dir, target_root, config, harness_root, story_id, + artifact=artifact, attempt=attempt, verdict=verdict, + ) + except Exception as error: # noqa: BLE001 - the totality is the guarantee + print(f"the accounting after the verdict could not run: {error}", + file=sys.stderr) + try: + _say(run_dir, f"{label}: it could not run: {error}", + kind=ACCOUNTING_EVENT, routed=0, acted_on=0, + filed=0, dropped=0) + except Exception: # noqa: BLE001 - reporting may not become the failure + pass + + +def _account_after_verdict(run_dir: Path, target_root: Path, config: dict, + harness_root: Path, story_id: str, *, + artifact: str, attempt: int, verdict) -> None: + """The body of the above, so the guard has one thing to guard.""" + label = f"accounting for the findings routed to {story_id}" + if (Path(run_dir) / accounting_artifact_file(artifact, attempt)).is_file(): + # This attempt is accounted. A second verdict within it — the one a + # correction pass returns to — changes nothing about what was routed. + return + record = _read_findings_record(run_dir, artifact, attempt) + if record is None or not record.get("ran"): + return + routed = [one for one in record.get("findings") or () if isinstance(one, dict)] + if not routed: + return + + acted, remainder = acted_on_by(routed, verdict) + + bound, problem = inspection.bounds(config) + if bound is None: + # Unreachable while the inspection that wrote the artifact resolved the + # same bound, and handled anyway: nothing is filed on a bound the target + # got wrong, and the record says so rather than obeying a default. + write_accounting(run_dir, artifact, attempt, story_id, routed=routed, + acted_on=acted, filed=0, dropped=len(remainder), + reason=problem) + _say(run_dir, + f"{label}: {len(routed)} routed on attempt {attempt}; " + f"{len(acted)} acted on by the verdict, 0 filed, " + f"{len(remainder)} dropped: {problem}", + kind=ACCOUNTING_EVENT, routed=len(routed), acted_on=len(acted), + filed=0, dropped=len(remainder)) + return + + scope = inspection.Scope( + path="", kind=inspection.CHANGE, + origin=ORIGIN.format(story_id=story_id), framing=PRE_STAGE_FRAMING, + ) + # The remainder is filed through exactly the call every other producer + # files through — the same floor, the same cap, the same named drop + # reasons, the same dedupe and the same queue — from the artifact the run + # directory already holds, with no second invocation of the Inspector. + filed, dropped = inspection.file_findings( + target_root, [_Routed(one, scope) for one in remainder], + bound.max_findings, min_severity=bound.min_severity, + ) + write_accounting(run_dir, artifact, attempt, story_id, routed=routed, + acted_on=acted, filed=len(filed), dropped=len(dropped)) + + line = ( + f"{label}: {len(routed)} routed on attempt {attempt}; " + f"{len(acted)} acted on by the verdict, {len(filed)} filed as briefs, " + f"{len(dropped)} dropped" + ) + reasons: dict = {} + for one in dropped: + reasons[one.reason] = reasons.get(one.reason, 0) + 1 + for reason, count in reasons.items(): + line += f"; {reason}: {count}" + _say(run_dir, line, kind=ACCOUNTING_EVENT, routed=len(routed), + acted_on=len(acted), filed=len(filed), dropped=len(dropped)) + + diff --git a/prompts/refactor-verifier.md b/prompts/refactor-verifier.md index f33e552..c27f550 100644 --- a/prompts/refactor-verifier.md +++ b/prompts/refactor-verifier.md @@ -283,7 +283,14 @@ Inspector read this change and found nothing about it to report. A record that is absent entirely means the coordinator could not write one, which is the same statement about your evidence and not a statement about the work. Findings about files this change did not touch are not here: those are filed as briefs, which -is where a backlog item belongs: +is where a backlog item belongs. When you raise a blocking issue or a +correctable finding from one of these findings, name that finding's `slug` in +the entry's `location` beside the file. The slug is how the coordinator tells +which of these you acted on: after your verdict, every finding here whose slug +no entry names is filed as a brief, so declining one is still right and sends +it to the backlog rather than losing it — and a finding you act on without +naming its slug is filed too, as a duplicate a person closes rather than a +loss: {{inspection_findings}} Repository standards: diff --git a/prompts/story-verifier.md b/prompts/story-verifier.md index 0e8f322..fde5534 100644 --- a/prompts/story-verifier.md +++ b/prompts/story-verifier.md @@ -274,7 +274,14 @@ Inspector read this change and found nothing about it to report. A record that is absent entirely means the coordinator could not write one, which is the same statement about your evidence and not a statement about the work. Findings about files this story did not change are not here: those are filed as briefs, which -is where a backlog item belongs: +is where a backlog item belongs. When you raise a blocking issue or a +correctable finding from one of these findings, name that finding's `slug` in +the entry's `location` beside the file. The slug is how the coordinator tells +which of these you acted on: after your verdict, every finding here whose slug +no entry names is filed as a brief, so declining one is still right and sends +it to the backlog rather than losing it — and a finding you act on without +naming its slug is filed too, as a duplicate a person closes rather than a +loss: {{inspection_findings}} Repository standards: diff --git a/schemas/cross-run-history.schema.json b/schemas/cross-run-history.schema.json index 9adccd0..af21798 100644 --- a/schemas/cross-run-history.schema.json +++ b/schemas/cross-run-history.schema.json @@ -55,15 +55,15 @@ }, "inspection-log.jsonl": { "type": "array", - "description": "One record per inspection, in either mode: the narrow one a completed run makes of what its story changed, and the broad one an l5-inspect invocation makes of a part of the tree. Written to .harness/history/inspection-log.jsonl, one JSON object per line, appended as the inspection ends. It exists because the run directory is gitignored and reaches no clone: an inspection reporting only into that run's events.log would leave no evidence in the repository that any inspection had ever run, and a broad-mode inspection has no run directory to report into at all. A record is a summary and never a copy — what was found, filed and dropped is here as three counts, and which finding went which way is in the run's own events.log and in the outbox entries the inspection wrote. Both modes carry the same fields, so one read of one file answers what inspection has cost across both without walking any run directory. The inspection is a total function that may not block, delay or refuse a run, so a record here is also what a configuration it could not obey is named in: an inspection that was not made because the file cap was unusable leaves a line saying so with its counts at zero, because naming the problem is the only thing a function that cannot refuse can do about it.", + "description": "One record per inspection, in either mode: the narrow one a run makes of what its story changed, and the broad one an l5-inspect invocation makes of a part of the tree. Written to .harness/history/inspection-log.jsonl, one JSON object per line, appended as the inspection ends. It exists because the run directory is gitignored and reaches no clone: an inspection reporting only into that run's events.log would leave no evidence in the repository that any inspection had ever run, and a broad-mode inspection has no run directory to report into at all. A record is a summary and never a copy — what was found, filed and dropped is here as three counts, and which finding went which way is in the run's own events.log and in the outbox entries the inspection wrote. Both modes carry the same fields, so one read of one file answers what inspection has cost across both without walking any run directory. The inspection is a total function that may not block, delay or refuse a run, so a record here is also what a configuration it could not obey is named in: an inspection that was not made because the file cap was unusable leaves a line saying so with its counts at zero, because naming the problem is the only thing a function that cannot refuse can do about it. Beside the inspection records sits a second kind of record about one of them: the accounting a run writes after the verdict, once per attempt that routed findings to the story. An inspection made ahead of the judging stage routes its findings about the story's own change to that stage rather than filing them, and the inspection's own record says how many were routed; the accounting record says what became of them — how many the verdict acted on, and how many of the rest were filed as briefs or dropped. It carries no cost_usd and no invocations, because it made no invocation: what it files comes from the artifact the inspection already wrote. It is what lets a reader of this log tell a finding the story answered from one the verifier passed over, which the inspection record alone cannot say because it is written before the verdict exists.", "items": { "type": "object", "required": ["timestamp"], "properties": { "event": { "type": "string", - "description": "The execution-history event kinds routed to this log, on the terms the execution log's own event property states: this enum is the selection, and the property is not projected into a record. It names this kind alone, which is what keeps an inspection out of execution-log.jsonl, mandate-log.jsonl and retry-log.jsonl: none of their enums names it.", - "enum": ["inspection-completed"] + "description": "The execution-history event kinds routed to this log, on the terms the execution log's own event property states: this enum is the selection, and the property is not projected into a record. It names the inspection's own kind and the accounting's, and no other log's enum names either, which is what keeps both out of execution-log.jsonl, mandate-log.jsonl and retry-log.jsonl. Because the kind is not projected, which of the two a line is has to be read off its fields: an accounting record carries `routed` and `acted_on`, and an inspection record carries `mode`, `invocations` and `dedupe_ran`.", + "enum": ["inspection-completed", "inspection-accounted"] }, "story_id": { "type": "string", @@ -100,7 +100,15 @@ }, "dropped": { "type": "integer", - "description": "How many were dropped, for every reason together. Which way each went is not summarised here: it is in the run's own events.log, where each reason is named with how many went that way." + "description": "How many were dropped, for every reason together. Which way each went is not summarised here: it is in the run's own events.log, where each reason is named with how many went that way. On an accounting record, `filed` and `dropped` are what became of the routed findings the verdict did not act on." + }, + "routed": { + "type": "integer", + "description": "On an accounting record: how many findings the pre-stage inspection routed to the story on the attempt being accounted — the findings about the story's own change, handed to the stage that judges it rather than filed. Absent on an inspection record, whose `findings` count includes them. Carried here because the inspection record is written before the verdict exists and can say only that they were routed; this record is written after it and says what became of them." + }, + "acted_on": { + "type": "integer", + "description": "On an accounting record: how many of the routed findings the verdict acted on, decided by the verdict naming the finding's slug in the location of a blocking issue or a correctable finding — never by file. The routed findings not counted here were filed as briefs or dropped, and `filed` and `dropped` say which. Present even where it is zero, because none acted on is an answer, and absent on an inspection record. An accounting record carries no cost_usd, no invocations and no dedupe_ran: it made no invocation and asked no query." }, "dedupe_ran": { "type": "boolean", diff --git a/schemas/execution-history.schema.json b/schemas/execution-history.schema.json index a077257..c19437a 100644 --- a/schemas/execution-history.schema.json +++ b/schemas/execution-history.schema.json @@ -43,6 +43,7 @@ "escalated", "story-completed", "inspection-completed", + "inspection-accounted", "note" ] }, @@ -117,6 +118,14 @@ "dedupe_ran": { "type": "boolean", "description": "Whether the inspection's filed query answered for every scope it covered. False is an inspection made without dedupe: the findings were filed anyway, because a query that could not answer costs dedupe and costs nothing else, so what this records is that the inspection may have refiled what is already in the tracker. It is a statement about the filed query alone and not about the local queue read beside it, which knows only what this machine filed and cannot see a duplicate filed from another checkout. Present on every entry that is an inspection, including one where the query did answer, because a boolean has a false to say and absence would otherwise mean either that or an entry written before this field existed. An inspection that made no invocation at all records false, because no query answered for it. Absent on every entry that is not an inspection, including the note a run appends beside an inspection's own entry naming the scope whose dedupe failed." + }, + "routed": { + "type": "integer", + "description": "How many findings the pre-stage inspection routed to the story on the attempt this entry accounts for — the findings about the story's own change, handed to the stage that judges it. Present on an accounting entry only, the one written after that attempt's verdict, and absent on every other entry including the inspection's own, whose message says the same count in words true when it was written. On an accounting entry `filed` and `dropped` are what became of the routed findings the verdict did not act on: the accounting files those as briefs from the artifact the inspection wrote, with no invocation of its own, which is why an accounting entry carries no cost_usd, no invocations and no dedupe_ran." + }, + "acted_on": { + "type": "integer", + "description": "How many of the routed findings the verdict acted on, decided by one rule: the verdict named the finding's slug in the location of a blocking issue or a correctable finding. A finding named by file alone is not acted on and is filed. Present on an accounting entry even where it is zero, because none acted on is an answer, and absent on every other entry." } } } diff --git a/tests/test_execution_history.py b/tests/test_execution_history.py index b7b3701..bd7a06e 100644 --- a/tests/test_execution_history.py +++ b/tests/test_execution_history.py @@ -644,12 +644,15 @@ def test_the_schema_uses_only_the_keywords_the_validator_supports(): #: "optional" one step sharper — absent means the harness was told no figure. #: `dedupe_ran` is an inspection's too and is a boolean, so it is written on #: every inspection entry including one where dedupe did run: absent there would -#: mean either false or an entry written before the field existed. +#: mean either false or an entry written before the field existed. `routed` and +#: `acted_on` are the accounting's, written after a verdict on the one entry +#: that says what became of the findings an inspection routed to the story. OPTIONAL_FIELDS = { "stage", "artifacts", "duration_seconds", "verifier_outcome", "retry_decision", "retry_reason", "retry_category", "retry_stage", "findings", "filed", "dropped", "mode", "cost_usd", "scope_files", "invocations", "dedupe_ran", + "routed", "acted_on", } diff --git a/tests/test_inspection_findings_reach_the_story.py b/tests/test_inspection_findings_reach_the_story.py index c8818a1..8954cc9 100644 --- a/tests/test_inspection_findings_reach_the_story.py +++ b/tests/test_inspection_findings_reach_the_story.py @@ -56,6 +56,25 @@ * "the judging stage renders a statement rather than a blank" sits beside the same rendering with the record removed, which is blank. +story-152 adds the other end of the route. A finding routed to the story was +read by the verdict and by nothing after it, so one the verifier declined — +which its prompt permits — was neither fixed nor filed. The section at the end +drives the accounting made after the verdict: what the verdict named by slug is +acted on, every other routed finding is filed as a brief from the artifact the +run directory already holds, once per attempt, and the split is said in words +true when written. Its absences carry their controls too: + + * "a routed finding the verdict named by slug is not filed" sits beside the + same finding under a verdict naming its file alone, which is filed; + * "no accounting is written where nothing was routed" sits beside a run + that routed one finding, where one is; + * "the accounting is made once within one attempt" sits beside the retried + run, where it is made twice under two attempt numbers; + * "the re-entry events say nothing about routing when nothing was routed" + sits beside the same events under a run that routed a finding, which do; + * "the coordinator branches on nothing the accounting wrote" sits beside a + planted branch on the same name, which the same reading reports. + Nothing here reaches a model. `agent_runner.run_agent` is replaced for every test in this module by a fake that fails the test if it is called without having been installed deliberately, and the stages run through a fake runner @@ -439,6 +458,12 @@ def __init__(self, target_root: Path, journal: Path, verdicts: list[dict], # first: what each entry was *given* is a question about that instant, # and a later read of the file answers a different one. self.prompts: dict[str, list[str]] = {} + # What the queue held each time the judging stage was entered. Since + # story-152 a routed finding the verdict does not act on is filed + # *after* the verdict, so what the queue holds once the run has + # returned answers a different question from what the judge saw; the + # cases about where a finding goes before the verdict read this. + self.queue_at_judging: list[list[dict]] = [] def __call__(self, prompt, *, stage, cwd=None, log_path=None, permission_mode=None, model=None, **declared): @@ -468,6 +493,9 @@ def __call__(self, prompt, *, stage, cwd=None, log_path=None, "def check():\n assert True # and again\n", encoding="utf-8") elif stage == JUDGING: + self.queue_at_judging.append([ + json.loads(path.read_text(encoding="utf-8")) + for path in outbox.entry_files(outbox.queue_dir(tree))]) verdict = conftest.answering_guidance( self.verdicts.pop(0), self.run_dir) _write(self.run_dir / conftest.VERIFICATION_RESULT, verdict) @@ -566,8 +594,21 @@ def cost(self) -> dict: @property def history_records(self) -> list[dict]: - """The cross-run inspection log's records, found by the declaration - that routes this kind rather than by a filename written here.""" + """The cross-run inspection log's records of the *inspection*, found + by the declaration that routes this kind rather than by a filename + written here. + + Since story-152 the same log also carries the accounting written after + the verdict, one line per attempt that routed findings; it is told + apart by the fields it carries, which is how the declaration says the + two kinds are told apart, and it is not what the cases reading this + are about. + """ + return [one for one in self.log_records if "invocations" in one] + + @property + def log_records(self) -> list[dict]: + """Every record the cross-run inspection log holds, of either kind.""" found: list[dict] = [] for relative in story_inspection.record_paths(self.tree, {}): path = self.tree / relative @@ -899,15 +940,21 @@ def test_both_positions_reach_the_scope_through_one_call_with_one_shape(): # ========================================================================== -def filed_markers(run: Run) -> list[str]: +def filed_markers(run: Run, *, at_judging: bool = False) -> list[str]: """The marker text of every finding the run's queue holds. Matched against the entry as a whole rather than against its key: a queue entry's key is the identity the filing derives, and what says *which finding* was filed is the finding's own words, which is what a marker is for. + + `at_judging` reads the queue as the judging stage first saw it rather than + as the run left it: since story-152 a routed finding the verdict does not + act on is filed after the verdict, so a case about where a finding goes + *before* the verdict has to read the earlier instant. """ - entries = [json.dumps(entry) for entry in run.queue] + held = run.runner.queue_at_judging[0] if at_judging else run.queue + entries = [json.dumps(entry) for entry in held] return sorted( marker for marker in (OWN_FINDING["body"], SIBLING_FINDING["body"], BOTH_FINDING["body"], CREATED_FINDING["body"]) @@ -918,7 +965,8 @@ def test_a_finding_about_the_change_reaches_the_artifact_and_no_queue_entry( driven): """The criterion this story exists for: a review comment on the change is given to the stage that can still answer it, and is not filed as a backlog - item against whatever story comes next.""" + item against whatever story comes next — read as the judge saw it, since + story-152 files what the verdict then declines to act on.""" run = driven(findings=[OWN_FINDING]) record = run.findings_record() @@ -926,7 +974,7 @@ def test_a_finding_about_the_change_reaches_the_artifact_and_no_queue_entry( assert record["findings"] == [OWN_FINDING] assert record["attempt"] == 1 assert record["story_id"] == STORY_ID - assert run.queue == [] + assert run.runner.queue_at_judging == [[]] def scope_lines(run: Run) -> list[str]: @@ -973,8 +1021,8 @@ def test_a_finding_about_a_file_the_story_created_is_answered_by_the_story( run = driven(findings=[CREATED_FINDING], creates=[CREATED_FILE]) assert run.findings_record()["findings"] == [CREATED_FINDING] - assert filed_markers(run) == [] - assert run.queue == [] + assert filed_markers(run, at_judging=True) == [] + assert run.runner.queue_at_judging == [[]] def test_a_finding_beside_the_change_is_filed_and_is_not_in_the_artifact( @@ -985,7 +1033,7 @@ def test_a_finding_beside_the_change_is_filed_and_is_not_in_the_artifact( run = driven(findings=[SIBLING_FINDING]) assert run.findings_record()["findings"] == [] - assert filed_markers(run) == [SIBLING_FINDING["body"]] + assert filed_markers(run, at_judging=True) == [SIBLING_FINDING["body"]] def test_a_finding_naming_a_changed_file_and_one_beside_it_reaches_the_stage( @@ -995,7 +1043,7 @@ def test_a_finding_naming_a_changed_file_and_one_beside_it_reaches_the_stage( run = driven(findings=[BOTH_FINDING]) assert run.findings_record()["findings"] == [BOTH_FINDING] - assert run.queue == [] + assert run.runner.queue_at_judging == [[]] def test_the_two_kinds_are_separated_within_one_invocation(driven): @@ -1005,23 +1053,26 @@ def test_the_two_kinds_are_separated_within_one_invocation(driven): record = run.findings_record() assert record["findings"] == [OWN_FINDING, BOTH_FINDING] - assert filed_markers(run) == [SIBLING_FINDING["body"]] + assert filed_markers(run, at_judging=True) == [SIBLING_FINDING["body"]] -def test_the_summary_says_how_many_were_answered_by_the_story(driven): +def test_the_summary_says_how_many_were_routed_to_the_story(driven): """A reader comparing the findings count against the filed count would read the difference as a silent drop, which is the one thing every count on that - line exists to make impossible — so the line says where they went.""" + line exists to make impossible — so the line says where they went. It says + they were *routed*, in words true when the line is written: nothing has + been decided about them yet, and what the verdict did is said afterwards.""" run = driven(findings=[OWN_FINDING, SIBLING_FINDING]) summaries = [line for line in run.messages if "finding(s)" in line] assert len(summaries) == 1, run.messages assert "2 finding(s)" in summaries[0] assert "1 filed" in summaries[0] - assert "answered by the story: 1" in summaries[0] + assert "routed to this story: 1" in summaries[0] + assert "answered by the story" not in summaries[0] -def test_a_run_that_answered_nothing_says_nothing_about_answering(driven): +def test_a_run_that_routed_nothing_says_nothing_about_routing(driven): """The control for the clause above: a run whose every finding was filed carries the counts and not the clause, so the clause is a fact about this run rather than a fixed suffix.""" @@ -1029,7 +1080,7 @@ def test_a_run_that_answered_nothing_says_nothing_about_answering(driven): summaries = [line for line in run.messages if "finding(s)" in line] assert len(summaries) == 1, run.messages - assert "answered by the story" not in summaries[0] + assert "routed to this story" not in summaries[0] def test_the_filing_from_here_goes_through_the_same_severity_floor( @@ -1356,7 +1407,7 @@ def test_the_entry_point_offers_no_way_to_stop_a_run(driven): assert answered is None -def pre_stage_calls_in(source: str) -> list[ast.AST]: +def pre_stage_calls_in(source: str, entry_point=None) -> list[ast.AST]: """Every call to the pre-stage entry point in a source, as the node enclosing it. @@ -1364,7 +1415,12 @@ def pre_stage_calls_in(source: str) -> list[ast.AST]: is not that the inspection is called but what is done with what it answers: a bare expression statement discards it, and anything else is a call site that could turn an inspection into a decision. + + `entry_point` is the function whose call sites are wanted; the pre-stage + inspection by default, and since story-152 the accounting after the verdict + is asked the same question through the same reading. """ + wanted = (entry_point or story_inspection.inspect_before_stage).__name__ found = [] for node in ast.walk(ast.parse(source)): for child in ast.iter_child_nodes(node): @@ -1372,8 +1428,7 @@ def pre_stage_calls_in(source: str) -> list[ast.AST]: continue target = child.func if (isinstance(target, ast.Attribute) - and target.attr == - story_inspection.inspect_before_stage.__name__ + and target.attr == wanted and isinstance(target.value, ast.Name) and target.value.id == story_inspection.__name__): found.append(node) @@ -1544,3 +1599,786 @@ def test_the_declaration_carries_no_budget_and_grants_no_veto(): def budget_keys(declaration: dict) -> list[str]: return sorted(key for key in declaration if any(word in key.lower() for word in ALLOWANCE_WORDS)) + + +# ========================================================================== +# story-152: every routed finding is fixed, filed or reported after the verdict +# ========================================================================== + + +def naming_the_slug(found: dict, entry: dict = CORRECTABLE_FINDING) -> dict: + """`entry` with its location naming `found`'s slug beside the file — the + shape the verifier prompts ask for when an entry is raised from a routed + finding.""" + return {**entry, "location": f"{CHANGED_FILE} ({found['slug']})"} + + +def failing_naming(found: dict) -> dict: + """A failed verdict whose one blocking issue names `found`'s slug.""" + verdict = failing_into(RETRY_CATEGORY) + return {**verdict, "blocking_issues": [ + naming_the_slug(found, verdict["blocking_issues"][0])]} + + +def accounting_path(run: Run, attempt: int = 1) -> Path: + """Where this attempt's accounting is, through the function that writes + the name.""" + return run.run_dir / story_inspection.accounting_artifact_file( + FINDINGS_ARTIFACT, attempt) + + +def accounting(run: Run, attempt: int = 1) -> dict: + return json.loads(accounting_path(run, attempt).read_text(encoding="utf-8")) + + +def accounting_files(run: Run) -> list[str]: + """Every accounting artifact the run directory holds, of any attempt.""" + return sorted(path.name for path in run.run_dir.glob( + story_inspection.accounting_artifact_file(FINDINGS_ARTIFACT, "*"))) + + +def events_of_kind(run: Run, kind: str) -> list[dict]: + """The run's structured history entries of one kind, read off the same + record events.log is rendered from.""" + return [one for one in story_coordinator.load_history(run.run_dir) + if one["event"] == kind] + + +def accounting_events(run: Run) -> list[dict]: + return events_of_kind(run, story_inspection.ACCOUNTING_EVENT) + + +def accounting_records(run: Run) -> list[dict]: + """The cross-run log's accounting records, told apart from the + inspection's own by the fields the declaration says they carry.""" + return [one for one in run.log_records if "routed" in one] + + +def routed_markers(run: Run) -> list[str]: + """The marker text of every *routed* finding the run's queue holds as the + run left it — the findings the story's own change was the subject of.""" + entries = [json.dumps(entry) for entry in run.queue] + return sorted(marker for marker in (OWN_FINDING["body"], BOTH_FINDING["body"]) + if any(marker in entry for entry in entries)) + + +@pytest.fixture +def filings(monkeypatch) -> list[list[str]]: + """The slugs handed to `inspection.file_findings` on every call the run + makes, in order, with the filing itself left to the original. + + Recorded rather than inferred from the queue: `outbox.enqueue` writes one + entry per identity however many times it is asked, so a queue holding one + entry cannot say whether the finding was filed once or twice. + """ + calls: list[list[str]] = [] + original = inspection.file_findings + + def recording(target_root, found, *args, **keywords): + calls.append([one.finding["slug"] for one in found]) + return original(target_root, found, *args, **keywords) + + monkeypatch.setattr(inspection, "file_findings", recording) + return calls + + +def filings_of(calls: list[list[str]], found: dict) -> int: + """How many filing calls carried `found`.""" + return sum(1 for slugs in calls if found["slug"] in slugs) + + +# -------------------------------------------------------------------------- +# The slug rule, driven as a function +# -------------------------------------------------------------------------- + + +def test_a_routed_finding_named_by_slug_is_acted_on_and_by_file_alone_is_not(): + """The one rule that decides a finding was acted on, held directly the way + the partition tests hold `about_the_change`. + + A slug in the location of a correctable finding or a blocking issue is the + verdict acting on it. The file alone is not: a correctable finding about a + docstring in a file must not count as acting on a defect the Inspector + found in the same file, which would be the loss this story closes one + layer down. + """ + acted, remainder = story_inspection.acted_on_by( + [OWN_FINDING], passing_with(naming_the_slug(OWN_FINDING))) + assert acted == [OWN_FINDING] + assert remainder == [] + + acted, remainder = story_inspection.acted_on_by( + [OWN_FINDING], passing_with(CORRECTABLE_FINDING)) + assert CORRECTABLE_FINDING["location"].startswith(CHANGED_FILE), \ + "the control names the file the finding is about" + assert acted == [] + assert remainder == [OWN_FINDING] + + acted, remainder = story_inspection.acted_on_by( + [OWN_FINDING], failing_naming(OWN_FINDING)) + assert acted == [OWN_FINDING] + assert remainder == [] + + acted, remainder = story_inspection.acted_on_by( + [OWN_FINDING], failing_into(RETRY_CATEGORY)) + assert acted == [] + assert remainder == [OWN_FINDING] + + +def test_the_slug_rule_keeps_every_finding_and_names_one_slug_at_a_time(): + """Nothing is dropped between the two halves, and naming one routed + finding's slug acts on that finding and no other — so two findings in one + file are told apart, which file matching could not do.""" + both = [OWN_FINDING, BOTH_FINDING] + acted, remainder = story_inspection.acted_on_by( + both, passing_with(naming_the_slug(BOTH_FINDING))) + assert acted == [BOTH_FINDING] + assert remainder == [OWN_FINDING] + assert sorted(json.dumps(one) for one in acted + remainder) == \ + sorted(json.dumps(one) for one in both) + + +def test_a_slug_that_is_a_prefix_of_another_is_not_the_other(): + """The match is on the whole slug: a location naming + `zzz-about-the-change` has not named `zzz-about`, and a finding with no + slug at all is one nothing can name, so it is the remainder.""" + shorter = finding("zzz-about", [CHANGED_FILE], "MARKER-SHORTER") + assert OWN_FINDING["slug"].startswith(shorter["slug"]), "the premise" + acted, remainder = story_inspection.acted_on_by( + [shorter], passing_with(naming_the_slug(OWN_FINDING))) + assert acted == [] + assert remainder == [shorter] + + unnamed = {key: value for key, value in OWN_FINDING.items() + if key != "slug"} + acted, remainder = story_inspection.acted_on_by( + [unnamed], passing_with(naming_the_slug(OWN_FINDING))) + assert acted == [] + assert remainder == [unnamed] + + +# -------------------------------------------------------------------------- +# What a verdict that acts on nothing leaves behind +# -------------------------------------------------------------------------- + + +def test_a_passing_verdict_acting_on_nothing_files_every_routed_finding( + driven, filings): + """The criterion this story exists for — the assertion attempt 1 of + story-151 would have failed. + + Two findings routed, a verdict that passes and names neither: the judge + saw an empty queue, the run ends with both in it, and the events.log line + written after the verdict says how many were routed, that none were acted + on and how many were filed. The verdict's own reading is untouched: the + run passes exactly as a run whose verifier declined a finding passed + before. + """ + run = driven(findings=[OWN_FINDING, BOTH_FINDING]) + + assert run.code == 0 + assert run.state["status"] == "completed" + assert run.runner.queue_at_judging == [[]] + assert routed_markers(run) == sorted( + [OWN_FINDING["body"], BOTH_FINDING["body"]]) + + recorded = accounting(run) + assert recorded["attempt"] == 1 + assert recorded["routed"] == 2 + assert recorded["acted_on"] == 0 + assert recorded["filed"] == 2 + assert recorded["dropped"] == 0 + + events = accounting_events(run) + assert len(events) == 1 + assert events[0]["routed"] == 2 + assert events[0]["acted_on"] == 0 + assert events[0]["filed"] == 2 + line = events[0]["message"] + assert "2 routed" in line + assert "0 acted on" in line + assert "2 filed" in line + assert line in run.messages + # Filed through the one call every producer files through, with both + # findings in one filing. + assert filings_of(filings, OWN_FINDING) == 1 + assert filings_of(filings, BOTH_FINDING) == 1 + + +def test_the_accounting_files_from_the_artifact_and_invokes_no_inspector( + driven): + """No second invocation to recover a finding: the Inspector ran once, and + the brief the accounting filed carries the finding as the artifact the + pre-stage inspection wrote holds it.""" + run = driven(findings=[OWN_FINDING]) + + assert len(run.inspector.invocations) == 1 + assert run.lines.count("inspected") == 1 + assert run.findings_record()["findings"] == [OWN_FINDING] + assert routed_markers(run) == [OWN_FINDING["body"]] + + +def test_the_accounting_called_alone_reaches_no_model( + target, harness, tmp_path, no_model): + """The same, asked of the entry point directly with the guard the autouse + fixture installed still in place: being called at all is the failure, and + the finding reaches the queue anyway from the artifact alone.""" + run_dir = tmp_path / "accounted-run" + run_dir.mkdir() + story_inspection.write_findings(run_dir, FINDINGS_ARTIFACT, 1, STORY_ID, + ran=True, findings=[OWN_FINDING]) + config = harness_config.load_config(target) + + answered = story_inspection.account_after_verdict( + run_dir, target, config, harness, STORY_ID, + artifact=FINDINGS_ARTIFACT, attempt=1, verdict=PASS) + + assert answered is None + assert no_model.calls == 0 + entries = [json.dumps(json.loads(path.read_text(encoding="utf-8"))) + for path in outbox.entry_files(outbox.queue_dir(target))] + assert any(OWN_FINDING["body"] in entry for entry in entries), entries + recorded = json.loads((run_dir / story_inspection.accounting_artifact_file( + FINDINGS_ARTIFACT, 1)).read_text(encoding="utf-8")) + assert (recorded["routed"], recorded["acted_on"], recorded["filed"]) == \ + (1, 0, 1) + assert any("1 filed" in line for line in + (run_dir / "events.log").read_text(encoding="utf-8").splitlines()) + + +# -------------------------------------------------------------------------- +# Named by slug, or by file alone +# -------------------------------------------------------------------------- + + +def test_a_routed_finding_the_verdict_names_by_slug_is_not_filed( + driven, filings): + """A passing verdict raising a correctable finding from the routed one and + naming its slug: acted on, and filed by nobody. The correction pass then + runs and the run completes, so the queue as the run left it is the queue + as the judge saw it.""" + run = driven(findings=[OWN_FINDING], + verdicts=[passing_with(naming_the_slug(OWN_FINDING)), PASS]) + + assert run.code == 0 + recorded = accounting(run) + assert (recorded["routed"], recorded["acted_on"], recorded["filed"]) == \ + (1, 1, 0) + assert recorded["acted_on_slugs"] == [OWN_FINDING["slug"]] + assert routed_markers(run) == [] + assert filings_of(filings, OWN_FINDING) == 0 + + +def test_a_routed_finding_the_verdict_names_by_file_alone_is_filed( + driven, filings): + """The control the absence above needs: the same run with the correctable + finding naming the file and no slug — the fixture's own — and the routed + finding is filed after the first verdict, so the judge's second entry + already sees it in the queue.""" + run = driven(findings=[OWN_FINDING], + verdicts=[passing_with(CORRECTABLE_FINDING), PASS]) + + assert run.code == 0 + recorded = accounting(run) + assert (recorded["routed"], recorded["acted_on"], recorded["filed"]) == \ + (1, 0, 1) + assert routed_markers(run) == [OWN_FINDING["body"]] + assert filings_of(filings, OWN_FINDING) == 1 + seen = [json.dumps(entry) for entry in run.runner.queue_at_judging[1]] + assert any(OWN_FINDING["body"] in entry for entry in seen) + + +def test_a_blocking_issue_naming_the_slug_is_acting_on_the_finding(driven): + """The other kind of entry the rule reads. A failed verdict whose blocking + issue names the routed finding's slug acts on it on attempt 1, so nothing + is filed then; attempt 2 inspects again and its passing verdict names + nothing, so the finding is filed under attempt 2's accounting.""" + run = driven(findings=[OWN_FINDING], + verdicts=[failing_naming(OWN_FINDING), PASS]) + + assert run.code == 0 + assert run.state["retry_count"] == 1 + first, second = accounting(run, 1), accounting(run, 2) + assert (first["routed"], first["acted_on"], first["filed"]) == (1, 1, 0) + assert (second["routed"], second["acted_on"], second["filed"]) == (1, 0, 1) + # The judge's second entry, on attempt 2, saw the queue still empty. + assert run.runner.queue_at_judging == [[], []] + assert routed_markers(run) == [OWN_FINDING["body"]] + + +# -------------------------------------------------------------------------- +# The two lines agree with the run directory and the outbox +# -------------------------------------------------------------------------- + + +def test_the_inspection_line_and_the_accounting_line_agree_with_the_run( + driven): + """The inspection-time line reports the routed count in words true when + written, the post-verdict line reports the split, and the figures on the + two lines are the figures the findings artifact and the outbox hold.""" + run = driven(findings=[OWN_FINDING, BOTH_FINDING, SIBLING_FINDING]) + + summaries = [line for line in run.messages if "finding(s)" in line] + assert len(summaries) == 1, run.messages + inspected = summaries[0] + routed = len(run.findings_record()["findings"]) + assert routed == 2 + assert f"routed to this story: {routed}" in inspected + assert "3 finding(s)" in inspected + assert "1 filed" in inspected + assert "answered by the story" not in inspected + assert not any("answered by the story" in line for line in run.messages) + + events = accounting_events(run) + assert len(events) == 1 + assert events[0]["routed"] == routed + assert events[0]["acted_on"] == 0 + assert events[0]["filed"] == len(routed_markers(run)) == 2 + assert f"{routed} routed" in events[0]["message"] + assert "2 filed" in events[0]["message"] + # The outbox holds every finding the inspection made: one filed at + # inspection time and two after the verdict, with no third place. + assert filed_markers(run) == sorted( + [OWN_FINDING["body"], BOTH_FINDING["body"], SIBLING_FINDING["body"]]) + assert filed_markers(run, at_judging=True) == [SIBLING_FINDING["body"]] + + +def test_a_run_that_routed_nothing_accounts_nothing(harness, tmp_path): + """The control for the accounting's presence: a run whose one finding was + filed at inspection time writes no accounting artifact, no accounting + event and no accounting record, so each of those is a fact about the + findings that were routed rather than a fixed addition to every run.""" + run = drive(build_target(tmp_path / "nothing-routed"), harness, tmp_path, + findings=[SIBLING_FINDING], name="nothing-routed-journal") + + assert run.code == 0 + assert run.findings_record()["findings"] == [] + assert accounting_files(run) == [] + assert accounting_events(run) == [] + assert accounting_records(run) == [] + # And the run beside it, which routed one, has each of the three. + routed = drive(build_target(tmp_path / "one-routed"), harness, tmp_path, + findings=[OWN_FINDING], name="one-routed-journal") + assert accounting_files(routed) == [accounting_path(routed).name] + assert len(accounting_events(routed)) == 1 + assert len(accounting_records(routed)) == 1 + + +# -------------------------------------------------------------------------- +# Once per attempt +# -------------------------------------------------------------------------- + + +def test_one_attempt_is_accounted_once_across_a_correction_pass( + driven, filings): + """The judging stage is entered twice inside attempt 1 — once with a + verdict that routes a correction pass and once with the clean verdict it + returns to — and the routed finding is filed exactly once, at the first + verdict rather than waiting on the second, with one accounting artifact, + one event and one cross-run record for the attempt.""" + run = driven(findings=[OWN_FINDING], + verdicts=[passing_with(CORRECTABLE_FINDING), PASS]) + + assert run.code == 0 + assert run.runner.calls.count(JUDGING) == 2 + assert run.state["retry_count"] == 0 + assert filings_of(filings, OWN_FINDING) == 1 + assert accounting_files(run) == [accounting_path(run, 1).name] + assert len(accounting_events(run)) == 1 + assert accounting(run, 1)["attempt"] == 1 + assert len(accounting_records(run)) == 1 + # Filed at the first verdict: the judge's second entry saw it queued. + seen = [json.dumps(entry) for entry in run.runner.queue_at_judging[1]] + assert any(OWN_FINDING["body"] in entry for entry in seen) + + +def test_a_retry_is_accounted_again_under_the_next_attempts_number( + driven, filings): + """The control the case above needs: a retry inspects again and is + accounted again, under a name of its own — two artifacts, two events and + two records, one per attempt. + + The first verdict acts on the finding by slug, so attempt 1 files nothing + and attempt 2 re-finds the same finding rather than seeing it already + queued; attempt 2's clean verdict then files it, so the one filing is + attempt 2's. + """ + run = driven(findings=[OWN_FINDING], + verdicts=[failing_naming(OWN_FINDING), PASS]) + + assert run.code == 0 + assert run.state["retry_count"] == 1 + assert run.findings_record(2)["findings"] == [OWN_FINDING] + assert accounting_files(run) == sorted( + [accounting_path(run, 1).name, accounting_path(run, 2).name]) + assert accounting(run, 1)["attempt"] == 1 + assert accounting(run, 2)["attempt"] == 2 + assert [one["filed"] for one in (accounting(run, 1), accounting(run, 2))] \ + == [0, 1] + assert filings_of(filings, OWN_FINDING) == 1 + assert len(accounting_events(run)) == 2 + assert len(accounting_records(run)) == 2 + + +def test_a_retry_re_finding_a_finding_the_last_attempt_filed_files_it_once( + driven, filings): + """What a retry after a filing looks like: attempt 1's verdict names the + file alone, so the finding is filed then; attempt 2's inspection re-finds + it, drops it as already queued before the partition — the dedupe every + producer files through — and routes nothing, so attempt 2 is not + accounted and the queue holds the finding once.""" + run = driven(findings=[OWN_FINDING], + verdicts=[failing_into(RETRY_CATEGORY), PASS]) + + assert run.code == 0 + assert run.state["retry_count"] == 1 + assert accounting(run, 1)["filed"] == 1 + assert run.findings_record(2)["findings"] == [] + assert any(inspection.ALREADY_QUEUED in line for line in run.messages), \ + run.messages + assert accounting_files(run) == [accounting_path(run, 1).name] + assert filings_of(filings, OWN_FINDING) == 1 + assert routed_markers(run) == [OWN_FINDING["body"]] + assert sum(1 for entry in run.queue + if OWN_FINDING["body"] in json.dumps(entry)) == 1 + + +def test_the_accounting_name_is_keyed_by_the_attempt_and_is_not_a_findings_name(): + """Two attempts cannot land on one name, and the accounting's name is not + matched by the findings artifact's own attempt-wildcarded glob — so a + reader listing an attempt's findings records does not count its accounting + among them.""" + import fnmatch + + names = {story_inspection.accounting_artifact_file(FINDINGS_ARTIFACT, n) + for n in (1, 2, 3)} + assert len(names) == 3 + findings_glob = story_inspection.findings_artifact_file( + FINDINGS_ARTIFACT, "*") + assert not any(fnmatch.fnmatch(name, findings_glob) for name in names) + # The control: the findings names themselves are matched by that glob. + assert fnmatch.fnmatch( + story_inspection.findings_artifact_file(FINDINGS_ARTIFACT, 1), + findings_glob) + + +def test_a_resume_carries_the_accounting_with_the_findings_record(): + """An accounting artifact left behind would have the next entry's attempt + 1 find it and file nothing for the findings it routed, so it travels with + the findings record it accounts for.""" + import tempfile + + with tempfile.TemporaryDirectory() as temporary: + run_dir = Path(temporary) + for name in (story_inspection.findings_artifact_file(FINDINGS_ARTIFACT, 1), + story_inspection.accounting_artifact_file( + FINDINGS_ARTIFACT, 1)): + (run_dir / name).write_text("{}", encoding="utf-8") + + carried = story_coordinator.inspection_artifacts( + run_dir, WORKFLOW["stages"], "*") + assert story_inspection.accounting_artifact_file( + FINDINGS_ARTIFACT, 1) in carried + assert set(carried) <= set( + story_coordinator.entry_artifacts(run_dir, WORKFLOW["stages"])) + undeclared = [{"name": name} for name in STAGE_NAMES] + assert story_coordinator.inspection_artifacts( + run_dir, undeclared, "*") == [] + + +# -------------------------------------------------------------------------- +# What the re-entry events say +# -------------------------------------------------------------------------- + + +ROUTED_CLAUSE = "findings routed to this story" + + +def the_one(run: Run, kind: str) -> str: + events = events_of_kind(run, kind) + assert len(events) == 1, events + return events[0]["message"] + + +def test_the_correction_pass_event_says_how_many_it_carries_were_routed( + harness, tmp_path): + """A reader of events.log can tell a re-entry spent on the Inspector's + reading from one spent on the verifier's own: the correctable finding + naming the routed slug counts, and the one naming the file alone does + not.""" + named = drive(build_target(tmp_path / "named"), harness, tmp_path, + findings=[OWN_FINDING], + verdicts=[passing_with(naming_the_slug(OWN_FINDING)), PASS], + name="named-journal") + message = the_one(named, "correction-pass-routed") + assert f"1 of them are among the 1 {ROUTED_CLAUSE}" in message, message + + unnamed = drive(build_target(tmp_path / "unnamed"), harness, tmp_path, + findings=[OWN_FINDING], + verdicts=[passing_with(CORRECTABLE_FINDING), PASS], + name="unnamed-journal") + message = the_one(unnamed, "correction-pass-routed") + assert f"0 of them are among the 1 {ROUTED_CLAUSE}" in message, message + + +def test_the_retry_event_says_how_many_blocking_issues_were_routed( + harness, tmp_path): + """The same of the verification-failed event, over its blocking issues.""" + named = drive(build_target(tmp_path / "named"), harness, tmp_path, + findings=[OWN_FINDING], + verdicts=[failing_naming(OWN_FINDING), PASS], + name="named-journal") + message = the_one(named, "verification-failed") + assert f"1 of them are among the 1 {ROUTED_CLAUSE}" in message, message + + unnamed = drive(build_target(tmp_path / "unnamed"), harness, tmp_path, + findings=[OWN_FINDING], + verdicts=[failing_into(RETRY_CATEGORY), PASS], + name="unnamed-journal") + message = the_one(unnamed, "verification-failed") + assert f"0 of them are among the 1 {ROUTED_CLAUSE}" in message, message + + +@pytest.mark.parametrize("kind, verdict", [ + ("correction-pass-routed", passing_with(CORRECTABLE_FINDING)), + ("verification-failed", failing_into(RETRY_CATEGORY)), +], ids=["correction pass", "retry"]) +def test_a_re_entry_event_says_nothing_about_routing_when_nothing_was_routed( + target, harness, tmp_path, kind, verdict): + """Only when the attempt routed any findings. Two controls for the clause + above: the same event under the same workflow with every finding filed at + inspection time carries no such clause, and under a workflow declaring no + inspection the event is byte-for-byte what it was — read as the one under + the declaring workflow, where nothing was routed, is.""" + declared = drive(build_target(tmp_path / "declared"), harness, tmp_path, + findings=[SIBLING_FINDING], verdicts=[verdict, PASS], + name="declared-journal") + assert declared.code == 0 + message = the_one(declared, kind) + assert ROUTED_CLAUSE not in message + # The clause's own phrasing, not the bare token "routed": the retry + # event's accepted wording says "rerouted to ", which is not a + # claim about the Inspector's findings. + assert "of them are among the" not in message + + undeclared_target = build_target(tmp_path / "undeclared") + undeclared = drive(undeclared_target, + without_the_declaration(tmp_path, undeclared_target), + tmp_path, findings=[SIBLING_FINDING], + verdicts=[verdict, PASS], name="undeclared-journal") + assert undeclared.code == 0 + assert the_one(undeclared, kind) == message + + +# -------------------------------------------------------------------------- +# The cross-run log +# -------------------------------------------------------------------------- + + +CROSS_RUN_SCHEMA = schema_validator.load_schema( + story_coordinator.CROSS_RUN_HISTORY_SCHEMA) +INSPECTION_LOG = next( + name for name, declared in CROSS_RUN_SCHEMA["properties"].items() + if story_inspection.INSPECTION_EVENT + in declared["items"]["properties"]["event"]["enum"]) + + +def test_the_cross_run_schema_declares_the_accounting_beside_the_inspection(): + """The accounting kind is routed to the log the inspection's own kind is, + and to no other, so a reader of that one log can tell the two apart and + no other log gains a line.""" + for name, declared in CROSS_RUN_SCHEMA["properties"].items(): + kinds = declared["items"]["properties"]["event"]["enum"] + assert (story_inspection.ACCOUNTING_EVENT in kinds) == \ + (name == INSPECTION_LOG), name + fields = CROSS_RUN_SCHEMA["properties"][INSPECTION_LOG]["items"]["properties"] + for field in ("routed", "acted_on", "filed", "dropped"): + assert field in fields, field + + +def test_the_log_receives_one_accounting_record_per_accounted_attempt(driven): + """One record per attempt, carrying the four counts and no cost, no + invocation count, no mode and no dedupe verdict — it made no invocation + and asked no query — and conforming to the declaration that routes it.""" + run = driven(findings=[OWN_FINDING], + verdicts=[failing_naming(OWN_FINDING), PASS]) + + records = accounting_records(run) + assert len(records) == 2 + assert len(run.history_records) == 2, "the inspection's own, one per attempt" + for record in records: + assert record["routed"] == 1 + assert record["dropped"] == 0 + for absent in ("cost_usd", "invocations", "mode", "dedupe_ran"): + assert absent not in record, record + # Attempt 1's verdict acted on it; attempt 2's did not, so it was filed — + # which is what a reader of this log can now tell apart. + assert [(one["acted_on"], one["filed"]) for one in records] == \ + [(1, 0), (0, 1)] + assert schema_validator.validate( + records, CROSS_RUN_SCHEMA["properties"][INSPECTION_LOG]) == [] + # And the inspection's own records carry what the accounting's do not, + # which is how the declaration says the two kinds are told apart. + for record in run.history_records: + assert "routed" not in record + assert "invocations" in record + + +# -------------------------------------------------------------------------- +# The guarantee: total, and decided on by nothing +# -------------------------------------------------------------------------- + + +def test_the_accounting_offers_no_way_to_stop_a_run(): + """Its signature in the terms the pre-stage entry point's own assertion + uses: everything the post-story sibling takes, what this one needs to name + its artifact and read its verdict, and no parameter by which a caller + could be told to stop.""" + parameters = inspect_module.signature( + story_inspection.account_after_verdict).parameters + sibling = inspect_module.signature( + story_inspection.inspect_after_story).parameters + assert list(parameters) == [ + "run_dir", "target_root", "config", "harness_root", "story_id", + "artifact", "attempt", "verdict"] + # No stage list, because it inspects nothing, and no runner, because it + # invokes nothing — the two the post-story sibling has that this does not. + assert set(sibling) - set(parameters) == {"stages", "runner"} + assert set(parameters) - set(sibling) == {"artifact", "attempt", "verdict"} + + +@pytest.mark.parametrize("text", ["", "not json", "[1, 2]"], + ids=["absent", "unreadable", "not a record"]) +def test_an_artifact_that_cannot_be_read_accounts_nothing_and_raises_nothing( + target, harness, tmp_path, text): + """A failure inside it is not the run's: an artifact that is absent, + unreadable or not a record leaves the call answering None, nothing + accounted, nothing filed and nothing raised.""" + run_dir = tmp_path / "unreadable-run" + run_dir.mkdir() + if text: + (run_dir / story_inspection.findings_artifact_file( + FINDINGS_ARTIFACT, 1)).write_text(text, encoding="utf-8") + config = harness_config.load_config(target) + + answered = story_inspection.account_after_verdict( + run_dir, target, config, harness, STORY_ID, + artifact=FINDINGS_ARTIFACT, attempt=1, verdict=PASS) + + assert answered is None + assert not (run_dir / story_inspection.accounting_artifact_file( + FINDINGS_ARTIFACT, 1)).exists() + assert outbox.entry_files(outbox.queue_dir(target)) == [] + + +@pytest.mark.parametrize("verdict, retries", [ + (PASS, 0), (failing_into(RETRY_CATEGORY), 1), +], ids=["passing", "failing"]) +def test_a_failure_inside_the_accounting_changes_no_routing_and_no_status( + harness, tmp_path, monkeypatch, verdict, retries): + """The fault constructed inside the body — the queue gone, as far as the + accounting can tell — and the run indifferent to it: the same routing, the + same status and the same exit code as the run beside it where the + accounting worked, with the failure printed and recorded in events.log. + + The working run is driven first, so the fault is installed once and is + undone by the fixture's own teardown rather than by hand — an undo by hand + would also undo the autouse guard on the agent runner. + """ + working = drive(build_target(tmp_path / "working"), harness, tmp_path, + findings=[OWN_FINDING], verdicts=[verdict, PASS], + name="working-journal") + assert working.code == 0 + assert working.state["retry_count"] == retries + assert routed_markers(working) == [OWN_FINDING["body"]] + + def cannot_reach(*args, **keywords): + raise RuntimeError("MARKER-FAULT the queue cannot be reached") + + monkeypatch.setattr(story_inspection, "acted_on_by", cannot_reach) + faulted = drive(build_target(tmp_path / "faulted"), harness, tmp_path, + findings=[OWN_FINDING], verdicts=[verdict, PASS], + name="faulted-journal") + + assert faulted.code == working.code + assert faulted.state["status"] == working.state["status"] == "completed" + assert faulted.state["retry_count"] == working.state["retry_count"] + assert faulted.runner.calls == working.runner.calls + assert any("could not run" in line and "MARKER-FAULT" in line + for line in faulted.messages), faulted.messages + assert routed_markers(faulted) == [] + assert accounting_files(faulted) == [] + + +def test_the_coordinator_calls_the_accounting_once_and_reads_nothing_from_it(): + """One call site, a bare statement — so nothing it answers can become a + decision — and the slugs read back off its artifact branch on nothing in + the stage loop: they word two events and decide nothing.""" + calls = pre_stage_calls_in( + COORDINATOR_SOURCE, story_inspection.account_after_verdict) + assert len(calls) == 1 + assert [node for node in calls if not isinstance(node, ast.Expr)] == [] + + source = conftest.function_source(COORDINATOR_SOURCE, "run_story") + assert branches_on(source, "routed_slugs") == [] + planted = source + "\n if routed_slugs:\n return 1\n" + assert branches_on(planted, "routed_slugs") != [] + + +@pytest.mark.parametrize("planted", [ + " outcome = story_inspection.account_after_verdict(run_dir)\n", + " if story_inspection.account_after_verdict(run_dir):\n" + " return 1\n", +]) +def test_the_scan_reports_an_accounting_call_site_that_reads_its_answer( + planted): + """Control: the bare statement above is a fact about the coordinator's one + call site rather than about a scan that cannot see a use of the answer.""" + source = f"def a_function(run_dir):\n{planted}" + found = pre_stage_calls_in( + source, story_inspection.account_after_verdict) + assert len(found) == 1 + assert [node for node in found if not isinstance(node, ast.Expr)] != [] + + +# -------------------------------------------------------------------------- +# What the shipped judging prompts ask for +# -------------------------------------------------------------------------- + + +def shipped_judging_prompts() -> list[Path]: + """The prompt of every stage that declares an inspection, in every + workflow this repository ships. The subject here is what the harness + ships, so the shipped definitions are read — and the prompts are found + through their declarations rather than named.""" + found = [] + for path in sorted((conftest.HARNESS_ROOT / "workflows").glob("*.json")): + stages = json.loads(path.read_text(encoding="utf-8"))["stages"] + found += [conftest.HARNESS_ROOT / "prompts" / stage["prompt"] + for stage in stages if stage.get("inspection")] + return found + + +def test_a_shipped_workflow_declares_the_inspection_on_a_prompted_stage(): + """The premise of the prompt assertion: there is at least one such stage, + and each of its prompts renders the findings.""" + prompts = shipped_judging_prompts() + assert prompts + for prompt in prompts: + assert f"{{{{{FINDINGS_FIELD}}}}}" in prompt.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("prompt", shipped_judging_prompts(), + ids=lambda path: path.name) +def test_the_judging_prompt_asks_for_the_slug_and_says_what_an_unnamed_one_becomes( + prompt: Path): + """Where the inspection findings are rendered, the prompt asks that an + entry raised from a routed finding name the finding's slug in its + location, and says why: a routed finding the verdict does not name is filed + as a brief.""" + text = prompt.read_text(encoding="utf-8") + marker = f"{{{{{FINDINGS_FIELD}}}}}" + introducing = text[:text.index(marker)].rstrip().split("\n\n")[-1] + assert "`slug`" in introducing, introducing + assert "`location`" in introducing, introducing + assert "filed as a brief" in introducing, introducing