diff --git a/.github/workflows/k9-conformance.yml b/.github/workflows/k9-conformance.yml index 5cad4041..7584fc84 100644 --- a/.github/workflows/k9-conformance.yml +++ b/.github/workflows/k9-conformance.yml @@ -22,8 +22,10 @@ name: k9-conformance # the PR base: an entry the PR adds exempts nothing unless a commit message or # the PR body carries `Ratchet-exception: .machine_readable/k9-contract-debt.txt # โ€” ` (the governance exemption ratchet's form; it watches this ledger -# too). On a pull request the checkout is full so the changed-file list is the -# merge-base diff and the PR's commit messages are readable. +# too). The checkout is full so the changed-file list is the merge-base diff +# and the commit messages are readable; on a push the range is the push's own +# (`before`..`sha`), so a direct push to a ledgered contract, or a direct +# ledger addition, is ratcheted exactly as a pull request is. # # Outputs: the `k9-grid` artifact (report.json and badge.json as I-JSON/JCS, # report.md, the validator logs) and the `verdict` output. The caller posts @@ -49,7 +51,6 @@ on: paths: - '**.k9.ncl' - '**.k9' - - '**.k9.lock' - '1-formats/k9/**' - 'scripts/k9-grid.sh' - 'scripts/lib/k9-grid.jq' @@ -60,7 +61,6 @@ on: paths: - '**.k9.ncl' - '**.k9' - - '**.k9.lock' - '1-formats/k9/**' - 'scripts/k9-grid.sh' - 'scripts/lib/k9-grid.jq' @@ -112,10 +112,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - # Full history on a pull request: the changed-file list is the - # merge-base diff, and the ledger ratchet reads the PR's commit - # messages. Strings, not numbers: a bare 0 is falsy in an expression. - fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} + # Full history: the changed-file list is the merge-base diff and the + # ledger ratchet reads the commit messages, on a pull request (base + # ..head) and on a push (the push's before..sha) alike. + fetch-depth: 0 persist-credentials: false - name: checkout the K9 validator, contract and renderer at this workflow's commit @@ -186,17 +186,20 @@ jobs: echo "TYPECHECK_CODE=$?" >> "$GITHUB_ENV" cat "$RUNNER_TEMP/typecheck.log" - - name: Enumerate the committed K9 files, the PR's changed files and ledger growth + - name: Enumerate the committed K9 files, the changed files and ledger growth env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} + # A pull request compares with its base; a push with the commit the + # push replaced (all zeros when the ref is new: nothing to compare). + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} LEDGER: .machine_readable/k9-contract-debt.txt run: | set -euo pipefail # Tracked files only: an untracked file is not a committed contract. + # The two extensions K9-CONTRACT-SPEC.adoc reserves, nothing else. # A caller with no K9 file at all is a legitimate (grey) corpus, so # grep's no-match exit is not an error here. - git ls-files -z -- '*.k9.ncl' '*.k9' '*.k9.lock' \ + git ls-files -z -- '*.k9.ncl' '*.k9' \ | tr '\0' '\n' \ | { grep -v -E '^(\.k9-canon/|9-archive/|1-formats/k9/tools/fixtures/)' || [ $? -eq 1 ]; } \ | sort > "$RUNNER_TEMP/expected" @@ -204,16 +207,20 @@ jobs: : > "$RUNNER_TEMP/changed" : > "$RUNNER_TEMP/ledger-growth" echo "BASE_UNRESOLVED=" >> "$GITHUB_ENV" + case "$BASE_SHA" in *[!0]*) ;; *) BASE_SHA="" ;; esac [ -n "$BASE_SHA" ] || exit 0 # The PR's own changes are those since its merge base, not since the # base branch tip: a file the base branch changed after the PR - # branched is not this PR's edit. The full checkout already holds - # the base commit; the fetch is a fallback only, and runs without - # credentials (persist-credentials is off), so it serves public - # repositories. + # branched is not this PR's edit. A push compares with `before` + # itself, never a merge base: after a force push the fork point is + # not the state the push replaced, and a ledger entry the old tip + # had removed would come back unseen. The full checkout already + # holds the base commit; the fetch + # is a fallback only, and runs without credentials + # (persist-credentials is off), so it serves public repositories. if { git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null || git fetch --quiet origin "$BASE_SHA" 2>/dev/null; } \ - && merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA" 2>/dev/null) \ + && merge_base=$(if [ "$GITHUB_EVENT_NAME" = pull_request ]; then git merge-base "$BASE_SHA" "$HEAD_SHA"; else git rev-parse --verify "$BASE_SHA^{commit}"; fi 2>/dev/null) \ && git diff --name-only "$merge_base" "$HEAD_SHA" > "$RUNNER_TEMP/changed" 2>/dev/null; then echo "$(wc -l < "$RUNNER_TEMP/changed") file(s) changed since the merge base $merge_base" else @@ -260,7 +267,9 @@ jobs: worst=0 while IFS= read -r f; do [ -n "$f" ] || continue - bash "$CANON/1-formats/k9/tools/k9-validate.sh" --strict --layer L2 --json --quiet "$f" >> "$RUNNER_TEMP/results.jsonl" 2>> "$RUNNER_TEMP/corpus.log" + # `--` ends the options: a tracked file whose name starts with `-` + # is a path, not a flag. + bash "$CANON/1-formats/k9/tools/k9-validate.sh" --strict --layer L2 --json --quiet -- "$f" >> "$RUNNER_TEMP/results.jsonl" 2>> "$RUNNER_TEMP/corpus.log" rc=$? [ "$rc" -gt "$worst" ] && worst=$rc done < "$RUNNER_TEMP/expected" diff --git a/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc b/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc index 8bd7593a..644f2f47 100644 --- a/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc +++ b/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc @@ -82,7 +82,7 @@ bash 1-formats/k9/tools/k9-validate.sh --self-test > selftest.log; echo $? > sel nickel typecheck 1-formats/k9/spec/contract/k9_contract.ncl > typecheck.log; echo $? > typecheck.code # The same exclusions as the lane: the fixture corpus is deliberately invalid # and the archive is out of scope. -git ls-files -- "*.k9.ncl" "*.k9" "*.k9.lock" \ +git ls-files -- "*.k9.ncl" "*.k9" \ | { grep -v -E "^(9-archive/|1-formats/k9/tools/fixtures/)" || [ $? -eq 1 ]; } | sort > expected # The lane also records the validator's worst exit status over the corpus. corpus_code=0 diff --git a/1-formats/k9/tools/k9-validate.sh b/1-formats/k9/tools/k9-validate.sh index ff68747f..4e32a01a 100755 --- a/1-formats/k9/tools/k9-validate.sh +++ b/1-formats/k9/tools/k9-validate.sh @@ -663,14 +663,19 @@ check_l3() { validate_one() { F="$1"; DIALECT=""; HAS_MAGIC=0 ERRORS=0; WARNINGS=0; SKIPS=0; FINDINGS=() + # F is the logical name (JSON, messages); FP is the path the checks hand to + # cmp, head, sed, grep and dirname. A name that starts with `-` would be read + # as an option by those, so it is given a ./ prefix here and nowhere else. + local FP="$F" + case "$FP" in -*) FP="./$FP" ;; esac - if [ ! -f "$F" ]; then + if [ ! -f "$FP" ]; then err K9-E000 L0 "no such file" if [ "$JSON" -eq 1 ]; then emit_json "error"; else emit_findings; fi return 1 fi - check_l0 "$F" + check_l0 "$FP" local rc=$? if [ $rc -eq 2 ]; then # Routed out of scope: examined, and handed to the spec that governs it. @@ -683,9 +688,9 @@ validate_one() { return 1 fi - layer_reached L1 && check_l1 "$F" - layer_reached L2 && check_l2 "$F" - layer_reached L3 && check_l3 "$F" + layer_reached L1 && check_l1 "$FP" + layer_reached L2 && check_l2 "$FP" + layer_reached L3 && check_l3 "$FP" local verdict="pass" [ "$ERRORS" -gt 0 ] && verdict="fail" @@ -782,6 +787,19 @@ EOF t "schema_version stays on line 5" ' schema_version = "1.0.0",' "$(sed -n '5p' "$stripped")" rm -f "$tmp" "$facts" "$stripped" + echo "== a name that starts with '-' is a path, not an option ==" + # The checks hand file names to cmp, head, sed, grep and dirname. A tracked + # `-x.k9.ncl` must reach the same verdict as `x.k9.ncl`, and the JSON must + # carry the name as given. + local dashdir dashjson + dashdir="$(mktemp -d)" + cp "$SCRIPT_DIR/fixtures/valid/kennel-data.k9.ncl" "$dashdir/-dash.k9.ncl" + dashjson="$(cd "$dashdir" && LAYER=L1 JSON=1 QUIET=1 validate_one "-dash.k9.ncl" 2>/dev/null || true)" + t "dash-leading name: no L0 failure" "" "$(printf '%s' "$dashjson" | grep -o 'K9-E00[0-9]' | head -n 1)" + t "dash-leading name: verdict pass at L1" 1 "$(printf '%s' "$dashjson" | grep -c '"verdict":"pass"')" + t "dash-leading name: JSON keeps the logical name" 1 "$(printf '%s' "$dashjson" | grep -c '"file":"-dash.k9.ncl"')" + rm -rf "$dashdir" + echo "== L3: signature presence is not verification (ยง10) ==" # No fixture can make a verifier appear, so the three verdicts a host can # reach are asserted here with a stub verifier instead. What is under test is diff --git a/scripts/lib/k9-grid.jq b/scripts/lib/k9-grid.jq index 3017d625..04cab223 100644 --- a/scripts/lib/k9-grid.jq +++ b/scripts/lib/k9-grid.jq @@ -112,7 +112,8 @@ def corpus_items(results; expected; ledger; growth; changed; require_files): .rows += [item("validator output"; "indeterminate"; "Malformed JSON result.")] elif (($r.file | type) != "string") or ($expected[$r.file] | not) or (.seen[$r.file] // false) or (($r.verdict // "") | IN("pass", "fail", "error", "routed") | not) - or (($r.findings | type) != "array") then + or (($r.findings | type) != "array") + or (($r.findings | type) == "array" and ($r.findings | any(type != "object"))) then .rows += [item((if ($r.file | type) == "string" then $r.file else "validator output" end); "indeterminate"; "Unexpected, duplicate or invalid result.")] else .seen[$r.file] = true @@ -213,8 +214,15 @@ def state_label: {red: "๐Ÿ”ด failing", orange: "๐ŸŸ  indeterminate", yellow: " def colour: {red: "red", orange: "orange", yellow: "yellow", green: "brightgreen", grey: "lightgrey"}[.]; def word: {red: "failing", orange: "indeterminate", yellow: "advisory", green: "conformant", grey: "not_applicable"}[.]; # Escape text for grid cells and finding summaries, flattening line breaks. +# Ids and reasons carry contributor-controlled file names and validator text, +# and the comment is posted by the lane's own account: every character that +# could start Markdown or HTML syntax, or an @-mention, becomes an entity, so a file named +# `![x](https://e.invalid/i).k9.ncl` is shown, not rendered. Entities display as +# the character itself in a GitHub table cell. def esc: tostring | gsub("&"; "&") | gsub("<"; "<") | gsub(">"; ">") | gsub("\\|"; "|") - | gsub("`"; "`") | gsub("\r?\n"; " "); + | gsub("`"; "`") | gsub("\\\\"; "\") | gsub("\\["; "[") | gsub("\\]"; "]") + | gsub("\\("; "(") | gsub("\\)"; ")") | gsub("\\*"; "*") | gsub("_"; "_") + | gsub("!"; "!") | gsub("~"; "~") | gsub("@"; "@") | gsub("\r?\n"; " "); def grid_label: "K9 L0โ€“L2"; def badge_url(st): "https://img.shields.io/badge/\(grid_label | @uri)-\(st | word | @uri)-\(st | colour)"; diff --git a/scripts/tests/k9-grid-test.sh b/scripts/tests/k9-grid-test.sh index 393112ef..4bd55396 100644 --- a/scripts/tests/k9-grid-test.sh +++ b/scripts/tests/k9-grid-test.sh @@ -218,6 +218,28 @@ check "baseline with trailing junk is one value" 1 "$(bash "$GRID" baseline "$WO render "$WORK/o11l" --results "$WORK/r-clean" --expected "$WORK/expected" "${good_evidence[@]}" --sha 2222222222222222222222222222222222222222 --pr 7 --previous "$WORK/two-objects" check "a forged two-object baseline still renders" yes "$(grep -q 'no comparable baseline' "$WORK/o11l/report.md" && echo yes || echo no)" +# 13. A findings array that holds something other than objects is not a +# gradeable record (orange), never a jq error that leaves no report. +{ rec a.k9.ncl pass '[]'; jq -nc '{file: "b.k9.ncl", verdict: "pass", findings: ["bad"]}'; } > "$WORK/r-bad-finding" +render "$WORK/o13a" --results "$WORK/r-bad-finding" --expected "$WORK/expected" "${good_evidence[@]}" +check "a non-object finding element is orange" orange "$(state "$WORK/o13a")" +check "the non-object finding names its file" "Unexpected, duplicate or invalid result." "$(reason "$WORK/o13a" b.k9.ncl)" + +# 14. Contributor-controlled ids and validator text cannot write Markdown or +# HTML into the lane's own comment. +evil='![x](https://e.invalid/i)_[y]*.k9.ncl' +printf '%s\n' "$evil" > "$WORK/expected-evil" +rec "$evil" fail '[{"severity":"error","rule":"K9-S002","layer":"L1","message":"see here [z](https://e.invalid) \\ `x`"}]' > "$WORK/r-evil" +render "$WORK/o14" --results "$WORK/r-evil" --expected "$WORK/expected-evil" "${good_evidence[@]}" +check "markdown image syntax is escaped in the id" no "$(grep -q '!\[x\](' "$WORK/o14/report.md" && echo yes || echo no)" +check "the escaped id is still legible" yes "$(grep -q '![x](https://e.invalid/i)_[y]*.k9.ncl' "$WORK/o14/report.md" && echo yes || echo no)" +check "markdown link syntax is escaped in the reason" no "$(grep -q '\[z\](https' "$WORK/o14/report.md" && echo yes || echo no)" +check "html and backslashes are escaped in the reason" yes "$(grep -q '<b>here</b> [z](https://e.invalid) \ `x`' "$WORK/o14/report.md" && echo yes || echo no)" +rec '@hyperpolymath/owners.k9.ncl' fail '[{"severity":"error","rule":"K9-S002","layer":"L1","message":"ping @someone"}]' > "$WORK/r-mention" +printf '%s\n' '@hyperpolymath/owners.k9.ncl' > "$WORK/expected-mention" +render "$WORK/o14b" --results "$WORK/r-mention" --expected "$WORK/expected-mention" "${good_evidence[@]}" +check "an @-mention in an id or a reason is not live" no "$(grep -q '@hyperpolymath\|@someone' "$WORK/o14b/report.md" && echo yes || echo no)" + # 12. Usage errors exit 2. check "missing --results exits 2" 2 "$(bash "$GRID" render --out "$WORK/o12" >/dev/null 2>&1; echo $?)" check "unknown subcommand exits 2" 2 "$(bash "$GRID" frobnicate >/dev/null 2>&1; echo $?)"