From 614752c7c77671ea5af683ce315b01abc4cc8c99 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 04:15:43 +0000 Subject: [PATCH] fix(outcomes): replay "."-marked rows; let shellcheck parse dispatch-runner Two defects in #622, reported after it merged: - audit-fix-outcomes.sh skipped every row with a non-empty `file`, but dispatch-runner.sh sends "." as the repo-root marker for repo-grain fixes, so those repo-scoped rows were dropped from the replay. Rows naming "", "." or "./" are now repo-scoped (one jq `repo_scoped` definition shared by the skip count and the replay filter). The test fixture's shell-quote row now carries file "."; without the fix, three checks fail. The E1 log has no "." rows, so E1 is unchanged. - A `# shellcheck disable` inside a case arm (SC1124) made shellcheck stop parsing dispatch-runner.sh at that line, hiding everything after it. Moved before the `case`; shellcheck findings for the file now match the pre-#622 baseline exactly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PpQVLrCbURpoXtrdvy8V3B --- .../experiments/E1-fix-outcome-replay-2026-10-20.adoc | 2 +- scripts/audit-fix-outcomes.sh | 11 ++++++++--- scripts/dispatch-runner.sh | 2 +- tests/outcome-verdict-test.sh | 4 +++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc b/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc index 65149291..bab1e2e7 100644 --- a/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc +++ b/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc @@ -109,7 +109,7 @@ They went in commit `8225e55f` (2026-02-27, "Auto-commit: Sync changes", owner). ** `recipe-annotate-believe-me` is the canonical id. `recipe-remove-believe-me` stays as a registry alias, listed under `deprecated_recipe_ids`, because Hypatia still emits it. `dispatch-runner.sh` reports outcomes under the canonical id, so a verified annotation never raises the removal recipe's confidence. ** The script's predicate measures what it actually does: un-annotated `believe_me` calls. E1 above measured the claimed effect (removal). * *Reporting.* `learning-monitor.sh` counted a `.success` field that no row has. It now reports per verdict, with pre-verdict rows shown as legacy. -* *Replay.* `scripts/audit-fix-outcomes.sh` reruns this experiment. It reads the rows of all three writers. It excludes the fixers' own third-party directories (`lib/third-party-excludes.sh`), and reports NA, not 0, when `git grep` cannot run. A row that names one `file` is skipped (and counted on stderr): the predicates are repo-wide, as the fixers run. Every replayed row here is repo-scoped. Rerun with those changes, it reproduces `E1-replay-2026-10-20.tsv` byte for byte. It measures `recipe-remove-believe-me` by removal and `recipe-annotate-believe-me` by annotation, as each claims. `E1-replay-2026-10-20.tsv` is its output for the rows above. The HEAD column moves as the repos do. +* *Replay.* `scripts/audit-fix-outcomes.sh` reruns this experiment. It reads the rows of all three writers. It excludes the fixers' own third-party directories (`lib/third-party-excludes.sh`), and reports NA, not 0, when `git grep` cannot run. A row that names one `file` is skipped (and counted on stderr): the predicates are repo-wide, as the fixers run. `.` (or `./`) is not a file but the repo-root marker `dispatch-runner.sh` sends for repo-grain fixes, so such rows are replayed. Every replayed row here is repo-scoped. Rerun with those changes, it reproduces `E1-replay-2026-10-20.tsv` byte for byte. It measures `recipe-remove-believe-me` by removal and `recipe-annotate-believe-me` by annotation, as each claims. `E1-replay-2026-10-20.tsv` is its output for the rows above. The HEAD column moves as the repos do. All of this sits behind the automation quarantine (exit 78), which this change does not lift. It is a precondition for lifting it. diff --git a/scripts/audit-fix-outcomes.sh b/scripts/audit-fix-outcomes.sh index be987cd0..0d09026d 100755 --- a/scripts/audit-fix-outcomes.sh +++ b/scripts/audit-fix-outcomes.sh @@ -21,7 +21,9 @@ # # Rows are replayed repo-wide, as the fixers run: a row naming one `file` is # a narrower claim than the predicate measures, so it is skipped (and counted -# on stderr) rather than refuted by a target in some other file. +# on stderr) rather than refuted by a target in some other file. "." (and +# "./") is not a file: dispatch-runner.sh passes it as the repo-root marker for +# repo-grain fixes, so those rows are repo-scoped and replayed. set -euo pipefail LOG=$1; shift # The fixers' own exclusion list (lib/third-party-excludes.sh), as glob @@ -32,6 +34,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib/third-party-excludes.sh" EX=() for d in "${THIRD_PARTY_SKIP_DIRS[@]}"; do EX+=(":(exclude,glob)**/$d/**"); done IDR_EX=("${EX[@]}" ':(exclude,glob)**/.pack/**') # fix-believe-me.sh also skips .pack +# jq: a row is repo-scoped when it names no file, or only the repo-root marker. +# shellcheck disable=SC2016 # $f is a jq variable, not a shell one +REPO_SCOPED='def repo_scoped: (.file // "") as $f | $f == "" or $f == "." or $f == "./";' # grep_hits : the git grep output, or exit 1 when # git grep itself failed (status > 1: no PCRE, unreadable revision). Status 1 # is "no match", which is a count of 0, not an error. @@ -63,9 +68,9 @@ pred() { # repo rev pattern -> count of remaining hits, or NA printf 'repo\tpattern\toutcome\tfixed_at\trev_at\thits_at\thits_plus7d\thits_head\n' for repo in "$@"; do name=$(basename "$repo"); head=$(git -C "$repo" rev-parse origin/main) - skipped=$(jq -c --arg r "$name" 'select(.repo==$r and (.file // "") != "")' "$LOG" | wc -l) + skipped=$(jq -c --arg r "$name" "$REPO_SCOPED"' select(.repo==$r and (repo_scoped | not))' "$LOG" | wc -l) ((skipped == 0)) || echo "audit: $name: $skipped file-scoped row(s) skipped" >&2 - jq -r --arg r "$name" 'select(.repo==$r and (.file // "") == "") | [.pattern // .recipe_id // .pattern_id, .outcome, .fixed_at // .timestamp] | select(.[2] != null) | @tsv' "$LOG" | sort -u | + jq -r --arg r "$name" "$REPO_SCOPED"' select(.repo==$r and repo_scoped) | [.pattern // .recipe_id // .pattern_id, .outcome, .fixed_at // .timestamp] | select(.[2] != null) | @tsv' "$LOG" | sort -u | while IFS=$'\t' read -r pat out ts; do at=$(git -C "$repo" rev-list -1 --before="$ts" origin/main || true) p7=$(git -C "$repo" rev-list -1 --before="$(date -u -d "$ts + 7 days" +%FT%TZ)" origin/main || true) diff --git a/scripts/dispatch-runner.sh b/scripts/dispatch-runner.sh index 1248d93a..28777917 100755 --- a/scripts/dispatch-runner.sh +++ b/scripts/dispatch-runner.sh @@ -115,8 +115,8 @@ fold_counts() { local name [[ -f "$PARALLEL_TMPDIR/tally" ]] || return 0 while IFS= read -r name; do + # shellcheck disable=SC2004 # $name names the counter; the $ is the indirection case "$name" in - # shellcheck disable=SC2004 # $name names the counter EXECUTED|SKIPPED|FAILED|SUCCEEDED) (( $name++ )) || true ;; esac done < "$PARALLEL_TMPDIR/tally" diff --git a/tests/outcome-verdict-test.sh b/tests/outcome-verdict-test.sh index dd931988..a7e43ada 100755 --- a/tests/outcome-verdict-test.sh +++ b/tests/outcome-verdict-test.sh @@ -193,7 +193,7 @@ GIT_COMMITTER_DATE=2026-01-01T00:00:00Z git -C "$ar" -c user.email=t@t -c user.n commit -q --date=2026-01-01T00:00:00Z -m fixture git -C "$ar" update-ref refs/remotes/origin/main HEAD printf '%s\n' \ - '{"repo":"ar","pattern":"recipe-shell-quote-vars","outcome":"success","fixed_at":"2026-02-01T00:00:00Z"}' \ + '{"repo":"ar","pattern":"recipe-shell-quote-vars","file":".","outcome":"success","fixed_at":"2026-02-01T00:00:00Z"}' \ '{"repo":"ar","pattern_id":"PA004","recipe_id":"recipe-annotate-believe-me","outcome":"success","timestamp":"2026-02-01T00:00:00Z"}' \ '{"repo":"ar","pattern":"recipe-remove-believe-me","file":"A.idr","outcome":"success","fixed_at":"2026-02-01T00:00:00Z"}' \ > "$TMP/audit/log.jsonl" @@ -207,6 +207,8 @@ check "replay predicate failure is NA" "recipe-annotate-believe-me=NA recipe-she "$(PATH="$TMP/shim-git:$PATH" audit)" # A row naming one file is a narrower claim than a repo-wide predicate: it is # skipped (above, no recipe-remove-believe-me row) and the skip is reported. +# The shell-quote row names "." (dispatch-runner.sh's repo-root marker), so +# it is repo-scoped and replayed, not skipped. check "file-scoped row is skipped, and reported" "audit: ar: 1 file-scoped row(s) skipped" \ "$("$ROOT/scripts/audit-fix-outcomes.sh" "$TMP/audit/log.jsonl" "$ar" 2>&1 >/dev/null)"