diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 557e2780..eca7821a 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -13,7 +13,7 @@ Hypatia is the neurosymbolic CI/CD intelligence layer for the hyperpolymath ecos ## Architecture ``` -Hypatia # 141 Elixir modules total (`shopt -s globstar; ls lib/**/*.ex | wc -l`) +Hypatia # 142 Elixir modules total (`shopt -s globstar; ls lib/**/*.ex | wc -l`) ├── Elixir pipeline # core: pattern analysis, dispatch, learning ├── Merge orchestration # 12 modules — leases, ledger, ticker (largest subsystem) ├── Kin subsystem # 6 modules — contingency, arbiter, gate, watchdog diff --git a/.github/workflows/hypatia-scan.yml b/.github/workflows/hypatia-scan.yml index f647bbcc..4739c432 100644 --- a/.github/workflows/hypatia-scan.yml +++ b/.github/workflows/hypatia-scan.yml @@ -25,5 +25,6 @@ jobs: # Scanner pin: see governance.yml. Drop once `uses:` is re-pinned. with: hypatia-ref: aa34a4cc6ba39d873710ff39763fdb9cde715f9d + # Only the one secret the callee declares (standards#1108 §6b). secrets: HYPATIA_SCAN_PAT: ${{ secrets.HYPATIA_SCAN_PAT }} diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index e4baabce..20bc741d 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -42,6 +42,7 @@ permissions: jobs: mirror: uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + # Only the seven forge keys the callee declares (standards#1108 §6b). secrets: GITLAB_SSH_KEY: ${{ secrets.GITLAB_SSH_KEY }} BITBUCKET_SSH_KEY: ${{ secrets.BITBUCKET_SSH_KEY }} diff --git a/.gitleaksignore b/.gitleaksignore index 46f630f4..4a0185fe 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -1,9 +1,21 @@ # SPDX-License-Identifier: MPL-2.0 -# Historical false positives, scoped to these exact commit/path/rule/line findings. -# Prometheus recording-rule identifier for API latency, not a credential. -269dac03d57835a1dfe3fb336095ffac61567ccf:deploy/terraform/modules/monitoring/main.tf:generic-api-key:421 -# Kubernetes template values all decode to the literal CHANGE_ME_IN_PRODUCTION. +# +# Exact historical fingerprints only (commit:file:rule:line). The full-history +# gitleaks pass in hyperpolymath/standards secret-scanner-reusable.yml scans +# every commit; these five matches sit in two files that f2fe2df removed from +# the tree. Each entry names what the matched value is. None of them suppresses +# a rule in any new commit or location. +# +# deploy/k8s/secrets.yaml at 0d310fa is a Secret TEMPLATE. Each of the four +# values is base64 of the placeholder CHANGE_ME_IN_PRODUCTION, as the comment +# above each line says. Checked 2026-10-19 by comparing every value against +# the encoded placeholder. 0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:38 0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:40 0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:44 0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:50 +# +# deploy/terraform/modules/monitoring/main.tf at 269dac0, line 421, is a +# Prometheus recording-rule name (record = "cicd_hyper_a:api:"). It is +# a metric identifier, not a credential. +269dac03d57835a1dfe3fb336095ffac61567ccf:deploy/terraform/modules/monitoring/main.tf:generic-api-key:421 diff --git a/.machine_readable/descriptiles/AGENTIC.a2ml b/.machine_readable/descriptiles/AGENTIC.a2ml index d07ff88b..ecc2bc39 100644 --- a/.machine_readable/descriptiles/AGENTIC.a2ml +++ b/.machine_readable/descriptiles/AGENTIC.a2ml @@ -40,6 +40,23 @@ require-rerun-after-fix = true release-claim-requires-hard-pass = true completion-claim-requires-behavioural-tests = true +[workflow-pin-detection] +# Hypatia's half of the pin-propagation architecture +# hypatia (this: detection) -> gitbot-fleet (actuation) -> .git-private-farm (propagation) +# Every rule here has a fleet fix script that carries the SAME facts, so a +# detection without a repair, or a repair without a detection, is a drift: +# WF026 workflow_audit.ex SHA-pinned dtolnay/rust-toolchain with no toolchain: input -> fleet fix-rust-toolchain-input.sh +# (detected in block YAML and KYAML/flow style alike, Y-3; repaired in block style only — +# a KYAML, flow-step, anchored or aliased finding carries auto_fixable: false and routes to review) +# WF027 workflow_audit.ex caller pinned at a reusable revision MEASURED dead (table) -> fleet fix-stale-reusable-pin.sh (Window A) +# PIN-003 pr-automerge-policy.json standards revisions whose own actions.lock GitHub +# rejects at job setup (545 commits, 2026-08-19..2026-10-19; PI001/PA001 fire) -> fleet fix-stale-reusable-pin.sh (Window B) +# known_reusables (sha_bump_propagation.ex) now include hyperpolymath/cicd-suite's gates +may-extend-dead-revision-tables = true # only from a MEASURED walk (file content or lock record checked per commit), never from dates +may-name-a-replacement-revision = false # a human chooses it against standards/scripts/lock-selfcheck.sh; agents carry the choice into the table +must-mirror-fleet-tables = true # WF027's list == fleet stale-reusable-pins.tsv Window A; PIN-003 == Window B; regenerate together +must-not-promote-to-blocking = true # these rules report and route to the fleet; a gate that blocks on them is an owner decision + [automation-hooks] on-enter = [ "Read 0-AI-MANIFEST.a2ml or AI.a2ml first", diff --git a/.machine_readable/merge-orchestration/pr-automerge-policy.json b/.machine_readable/merge-orchestration/pr-automerge-policy.json index 89e2a09f..e8595a68 100644 --- a/.machine_readable/merge-orchestration/pr-automerge-policy.json +++ b/.machine_readable/merge-orchestration/pr-automerge-policy.json @@ -2,19 +2,24 @@ "$schema": "https://github.com/hyperpolymath/hypatia/blob/main/docs/design/merge-orchestration/schemas/pr-automerge-policy.schema.json", "$comment": "SPDX-License-Identifier: MPL-2.0 — Single source of truth for estate PR automerge + pin integrity. Read by BOTH the Elixir brain (Hypatia.Automerge.Policy) and the token-bearing actuator (scripts/estate/pr-automerge.sh). Two independent readers, one policy file: the actuator never trusts the brain's verdict, it re-derives it from the same data.", "version": "1.0.0", - "updated": "2026-09-26", + "updated": "2026-10-19", "owner": "hyperpolymath", "description": "Declarative policy for the 'safer core' of hypatia: unambiguous bumps/chores/pins get merged and their branches deleted; poisoned pins are excised or vetoed; everything else is flagged. Written after the 2026-09-26 finding that 108 workflow files across 84 hyperpolymath repos (and 23 in metadatastician) carry the codeql-action v4.38.1 commit labelled as v4.38.0.", - "pin_denylist": [ { "id": "PIN-001", "action": "github/codeql-action", "ecosystem": "github-actions", "severity": "critical", - "blocked_versions": ["4.38.1"], - "blocked_shas": ["1c5b675653bb5c22dbe9b12b556ec555138e09fd"], - "blocked_refs": ["v4.38.1"], + "blocked_versions": [ + "4.38.1" + ], + "blocked_shas": [ + "1c5b675653bb5c22dbe9b12b556ec555138e09fd" + ], + "blocked_refs": [ + "v4.38.1" + ], "reason": "GitHub rejects this commit at workflow start-up: `startup_failure`, zero jobs, no logs, wherever it is used. Kills CodeQL, Hypatia Security Scan and Scorecard on every repo it reaches.", "known_good_version": "4.38.0", "known_good_sha": "b96794f015dfd88f77b49b1c93e0fa7110f94c63", @@ -40,13 +45,582 @@ "known_good_version": null, "known_good_sha": null, "replacement": "unpin_moving_ref", - "evidence": ["hypatia/docs/DEBT-REGISTER.adoc CI-1 (run 31170993296, job 92842543720)"], + "evidence": [ + "hypatia/docs/DEBT-REGISTER.adoc CI-1 (run 31170993296, job 92842543720)" + ], "added": "2026-09-26", "added_by": "hypatia debt register CI-1", "notes": "Structural cure, not a re-pin: drop the entry from actions.lock rather than pinning a new SHA." + }, + { + "id": "PIN-003", + "action": "hyperpolymath/standards", + "ecosystem": "github-actions", + "severity": "critical", + "blocked_versions": [], + "blocked_shas": [ + "fae7f01010f44606a65f84fe53e613201a0950ac", + "e16a23170fa378bf3b1444c3ee94100413ae3ff7", + "b2cb7b685e17bd24e0122d19950a45de94ddcbc6", + "8ff9ce65f7e54f8d5c5aa94584e2ead0f2ba460f", + "ce768a6abea6fb0bca9b4c3eeb135c3a75a7f06e", + "3de430074148c49c8db8878653b3c66707c344e4", + "f7573a00ba48a2dc19c6f912d3096f95b38f7be5", + "813dcbaba53f11c06e09a0092c8466b4cbb778ee", + "f335165a8882abadc99a092f55ff950b392af2d8", + "a88f511b9bc06b430a4b4071efe75d91921cd9f7", + "0d9a814ac8d7b1d5c1f39d8510639e8fdd99f3e4", + "f78a36c9058cbab0b6610a675d3d5d7b024fd837", + "68242e650c0bd83bbf280f3b578fa949c0ab4c16", + "94572e612a7bf11f8e6343d19adadf17ef2a3de3", + "80014df42eb81f09396c948299febb258cec4719", + "95de3dfc3fcf09a1cf3325ab42723e3d81682b45", + "6a9251037760690d56022ad6fab2cfe9d887dd76", + "35cf3e2b05ab1efaecf72f8e0875ee7b6da46b88", + "ed04132e56c086795d3cc478cbdc145333ab5780", + "25824f323e76d481a3d37f22f28440ef77dc7905", + "934b879f6068295e42dac20d80e010d555ebec82", + "25d23cf3934e6c1fffd275ac78398c70aa9d58f3", + "8a39fa0bdb4d7787f7529111be417f984c85381e", + "192389bf1b40b70c594464937db7fdbf39a9a964", + "2ae085c076a4b14990bf67da0dd3821ce1c12944", + "c11c255e1b5546854e874d1486d61d6d5a4da29a", + "75064de100849f1864f8fe1c912e64c48d70a3ee", + "ae36881b5ff4b469fb904b66542ac226a15107f3", + "3daaad25e39d671dde8ad58e6427282309b02cce", + "b26d3fb548fa2c4a7920cc69128821811d4e3222", + "50cfbd55d44130399bf1215c04fd154a49277ee8", + "507b89f1b35453bf3de62622373cc6573245b664", + "bf21155871bfd8d4789f33543d0c30880b8403cd", + "e3d5107b6319dc1121e3edf12f497bed2f9fcd61", + "7d2041a2315572d8257341449257e6cfddc9aebc", + "48e0ed9138ade95a5357f76079fc56d8f6f91bb5", + "736f792321e24af85b0dfc44b7666538bf4a2ff1", + "312ff116d9acbfff3523da5e9f71ac3ca9c327d3", + "0b1898e2e6a0fc88348c0f2517970593d373ca10", + "4c101fbbc0b20e816c6beca9fc0c909994163443", + "9164d76efef88f5173fdb717b231618a2d3c701a", + "3108c1b01186741eea80c00da3f4beda912286f9", + "7712ae1568af5c872408bfa1ba27578dd2afecd0", + "8496854d8241c3a0e20abaecf67e9c70638aa3b1", + "dd58dd9659f425a3f2372b123dde524c27d045a4", + "9d629d112daa396636bbb32757213552a8fb6878", + "76773445d8b0c3a2558f33cc426bacc638206a6f", + "6d19ce0ebae047027dcfd594c7810bf35c642faa", + "d8848f8509ccd94e931f6768487c88c8423f6033", + "2e12b312b529c51f095ce3b22cd35968f5aba898", + "ca12cd467cb13d02072837f9a2a69ede429b0b09", + "900c42c70b968e11a2bca04bf0ce050be5c286dc", + "a695e37936bacff66add7ec354eb629f659477e5", + "bbcc722bb0a5969ca0b1ebb1178af2ebcd5e9c34", + "691ea2af89d4313c906672d880690f81219dbdc5", + "d8a90cf8093ab787e98b940cd7d7675b2c981ec2", + "e35431d5bce94895b675d5a135e3ff5af1ca4a63", + "a709d3a642bee6e8d22829a69d2d64e1e8656d57", + "c9dfc2dcb333c1672aa779fcc86671be037ee735", + "8639b851db282e89083351aa782d906eb537def2", + "2f9d0be57ecbb09fb5d93df411fa6c9758ab305c", + "0683ddf00237ae92c5756cf858b3da1cc706894e", + "85aa1934975a8df576c7a4912bacbb8160ab2d2d", + "69ea45cfefab795faa2122df468ce7d120ebdff2", + "f77fc28ad400f281d26431b5e6a204dfc7a2ea22", + "63d2bbf64020b5f87bc546973d21881c19e79c65", + "8453510aa0da5027dabe54743e06ed7e7648fbe2", + "d7b85cac57eb16edf51508d6f30806e86d63c9d3", + "d5a7cfa4a9e9b77e1ec45b828ea32e3bdba4634f", + "9f68971b9fa254a80559432d9fcb7652b72e941d", + "b9e78cd426a83d87408d1f5db9e798b9fb8a3bbe", + "8b5dfd0646c1232174e680ae7772250610bfa2b8", + "7d51f30f992b51fe2b33119761901c594e266e82", + "89d778d9dc5343ad3dc178656c60e9caf235588b", + "043496c2d1efb9ba48f69ec533b5444ab61bbb5d", + "ebfcd98ba6c162df2eeca854254e725095fc85d2", + "239759976b272a426a3b7d34d0d48c0009bddcfb", + "79df53059d638e699b317951f717ee9710d2618d", + "640d86efeddb1e47da230515f5af9f4c936bba80", + "ad637e5134bc9f02ddf65431827d515b26bd58ee", + "bae58d9398f6deff6187a18c59620c984b6e1c47", + "223a9da337e327246d17ad49800cf567744a3d5e", + "617f3bcb83d33f5a0e3793c41d26a3c7937dfd16", + "0ae8b773b5a0f16ca1c5389442c907ceacf75e00", + "eafcfbb5933dde8b42fdbd61483794bb75fb4c97", + "5a0891e9e58b785181a92e141505ca96ca987b42", + "8464935a8028dbf0b06eb3d2b9bc7c7a7a35b4c6", + "b935e63937b1f1b5cbaeea7e375947d7b924038f", + "44a0de481c1b10843b358e6c5c278efbfd4eaad9", + "cfc3a92e55f6e6feeed33412d9fad94812d330ed", + "78a0a61f481bbe394214002f51f41a92325c0fff", + "ce92a8cdb60b60ea4b8cd5040a14c5ec2eec9f0e", + "5751b97ed597e560573242f56f1287db159edd00", + "a5ff08d7a12399fe21fa9ed72841a9c5749d086b", + "8556163f6bb9f06e2ed9beb67a91a335aa1f8913", + "ef7e6c4cc801d1ebf5504b940ff70f72061334d0", + "0cb69026a45f756d6610d8b67e298c4c3d6acc95", + "67e148fe710381c732ee1a6a8b2898688ad13110", + "80b3a1abccfe39b16d8af8ead874e09e9529737b", + "5b56aec07917394f48d51cc95c459b12dc9c179e", + "32d37263e96600ee81ec27f343bef812c4670968", + "3bd25bd0fa9bb8f5a50d9d8288ad6c615a83fbda", + "bee32592ef4952e7e0dce8035e66f827c79839b5", + "44ca61e2deca34d8fa68a31922b1efbe7f13f241", + "8ffcaf6251be5b05fe8ae4b3b88efbc5acc203bb", + "5693b8b28b402355ab4516db4db3246b0b1019b4", + "6075712d2138d0bdcb1192d473ba50fba9501e0f", + "aa1c4dd591fa2d5ecf0d65d53bb127eab6b95486", + "c33b22b1037d6eabceb7673975e50c78030059d7", + "a41f3692d3e17193f14ede5cd7f79172f339085b", + "65d9af0140d3761b4b450a5f475d5490b31ec5e2", + "4d60617725a5eb899cc5ebba85e72d598e66560c", + "4e13946f6ff22657c7ce1592e42100222da4e554", + "1f099de983ed1ec26b8fbfdf176b69950291ead3", + "177be894c56f68c3f3a3eb428b4993a8996da892", + "e59aad7153dd7bc87c056039fcd71ba0567f37af", + "93f0c3fdc0c2dbf24d41cb888668adc4e34c6352", + "2f8c3eebad6418c04ed8dcc35d0b1ac4bf3954d4", + "b3075e041f52cb6f7e3f9bc610e22000ce388548", + "5c300a80f82d94d6faf02de0da7a1ab25700461f", + "a9a3281d620bdd195369c9a6f0bc1d661910965a", + "0ad178797da63e826742c20600a89e50c47c20ec", + "e2e0f6d61c2d2a36ef516849dc1ca9223563479f", + "c55031473e743591b649da28cc16a4f095712082", + "804b780093b3c48842913ee472e9191afe5f8329", + "f14575da71758f21a437fc875ec99d68399d7931", + "f5f48a4605aff2622330356f45cf720871aa6c54", + "aa8fd9e81585ced5ce0fa580092b21e1f84d0fe1", + "202a215d01297eb825c0d45495fcbc46a82a5e30", + "907bbfe535d21e0709eb4a67fbea1e304beec22b", + "11ba299dca221b6d4e970eeb58dd17ccbb825a9a", + "56b767ff3f70ea5427900c37659b0ea94eb49628", + "c780e55efec3ef2f765cb0223cd154102a85a1a3", + "e4b64b00cf7698a624ea8a0a8bfba551d4674dc0", + "215df8e96590dda0539d168eba000a579ab52517", + "71bb615e234133d22f04106c812aad9fcb269aa5", + "721e4a9e1beb2392ee096d38ead44c0ece962eb3", + "10e0b6ffb979f0a27fe58206512055133e67460a", + "39ee4ec5e026dbdae084989c856a655e1e342629", + "9719570729342a13fac0ead93a68d4516c5dab79", + "df20618b3ce9cefc19d603d5300952550074a3fe", + "13b872c19a9459e21a7f91aaebaf2a29e99684c9", + "0187ccd95604614009b37bb23a55f2e634cdc404", + "89813bf8e4cb806eb01232b396410c28f30ca742", + "1b6e19eaef9e81ec19d6e826837e7e1a09f968cd", + "2c06ddf3d4f0d682c70ae75643db307df4fd36dd", + "7de6a8a7965950448a3abd0fe16653fe5af21e82", + "7243deb443c38ff4a47a840ae01546c8106de245", + "9f0a8bef3c4839c14751d6223a69b03e4f33fcea", + "3c5bf1e1cefc9f3ecc2cb3b07ea5981933df9fd9", + "658ded3ffea6353492de3f082e61aa489c728fa3", + "3fbfa71e126a95a884f74e9eeff8d189902d378a", + "bd9c319f9f10d9dc3c3d94fd490d92dd4270ba40", + "8cfad82a72b94d1b362c20c5be95fbf677cc31f1", + "3238461b0930ade9172a2a449497af12aad56e1a", + "74d2f66f575246cf6e313ae7775f44df6e097ff2", + "4b2a34740425b4bc124804edd8e37c51d303c135", + "2ee32bc65b7e0d2ca8aa05a6b952817fd87f93f1", + "5fb9ad197f5bf57cf86c5f8df0c061ccb0bcf6a2", + "5f82b635c5da5c3df44d5a0caa8983d9b95e0db9", + "d0dc04d9c315b2a658bcabb3b970994365034c6d", + "51eeca7ec47cf217f8bf296f6d1b2190584d0398", + "13baaa8df7dc7ba1c8e41351b08caacfd1f2244c", + "3a845452177a9910e21ce553a3c53d9f5e5763c0", + "bd9313a6fca4a08b4f0f0763f0079f6a24262726", + "d8b9ad9a680729df98bfa947c7c1538adab02b00", + "f428562bf63be8d5004bc871a6260d91a6a20d7c", + "6e0d69c42ee3b83080a8ec82bf76cb8fe558c24a", + "6e98f4bf810afb9b03d363b4c2c0c0adc1ac4953", + "fcf6e8d66463ddcb23a0abc1ba6fa9732ea595bd", + "60fa96c63967305b9784cbd8f685824281f67bea", + "7b2060bc871878d55adadb012f300fe3f00cac16", + "2479cf769ed5f0481ccf64860a2ab954514c2b59", + "53b7a0716e36654f041e922e1c41843d7a04ff4f", + "ebe60f2fc0f71a215cf0fd7fa848769f446f7fbf", + "c56f4ecc791676a2f8ce8f4e47aaf0b1c5b3763a", + "2c91e9e30567447d038f6bbe37308e9185087ea7", + "db10ad3fa281dfbd5f682a76b808193b8e179752", + "64ad98645b13a5746996af0ac964c19512fe6472", + "f56da4fbb342e7452dfbe9fd42430615833048d0", + "520dcf63a8c6d0f194de3ed4aa72f78fa573721f", + "30815fa6e9132605d8f0e8cfdce855638d2eb553", + "65de939f110068b1761504dcbed341383ab9819b", + "779d8a9e62cf9e5661dc3abeecf62e49ca64c15a", + "0f2d9b5447d3d6e56b41fe69d54e834849d7c932", + "682fc60a5381744863443a719dfbe6b9bac93ab7", + "392e84d63946e6337901baca316c0e106d3bb7ea", + "0f13f51fafe9d2b670252aa2a18993223bffdece", + "a613156836c24bfb1b94ea014273c0b711db6e69", + "991b13e8006d44c66a609ece7a3433f67112f06a", + "faed4dd1957a6b60ecc131ddd6aa41f6fdc3f0bc", + "48dc0add6170c4b6441554449ed21644230d7165", + "a55935e103729fc834780386b77176d6b7b7b285", + "73e84b8fd767310142604b67b67e41f7fea9cc6f", + "e3b9929cdc57149874e2211e6c8934145d245855", + "58aa824437eb20147acdf72dcabb4e271874a243", + "ed5e3f651305dd1ce0d0b5d2d08b97a963634632", + "d1bd7f42fcacd3fe29328b32be361964a915788e", + "f95130bbcbc8917bfec8f8875fa810ad93b08871", + "de6810309a6165547443f9970ee311af0bc7688c", + "edac0d6a533281e3bb2eb729dcd09d1b40023bbe", + "79991c75d1419fcf76e72eddb9b0aea14016e438", + "7b931ef7f9dbb8d2386fc9170895708c0acae95c", + "68acee77cebda300e1f36fb6279f19d53d5b5cda", + "2eb9bb19d43b6285466ca36b92d1324a09e7a207", + "417094b2224ea7cc130997f8d9c36847b2efb302", + "436a89b30cc988fd5505c9cd8f5dd81110a8606b", + "8aea869bd93421128db2b4c9d4fd0b936e2a3a99", + "12d76e891ffe5eda6265c79b438a07f5dce126d3", + "051f98ec6ab2479cd46e405432811d2bb129e4e6", + "3a7c90aeacbd418f2d29d2522027e5621060dbf7", + "38d211f19a4bcb9b8b684cdb3fe1f0459f05b73d", + "9c256b67486b4b30c757730e1b65b2c2d2af935b", + "be6c9580237ec7a4f86c3d98228e4db569d6dca4", + "e977cc67456de8349dcc08143b92ce5608083f67", + "7b05a32346a40789271f401afbbb7fb3310b281f", + "a2ff696ab8f5e0b38823ad3b5cdff72772093c6e", + "5c5b894a67b967484f7c34a6be0d076fda20e246", + "4f7f02ca528212c578fd56379a202d219d01abe0", + "8fd6c915c96073c818bfc42089a95936e80f265c", + "b1a3b7b7c9c8cef5e8c8d44771426e7764b9521e", + "a521ed701f0e58cc5e7effd239b10b360908365a", + "b77c53c85e5734756a793497845735485c9dc933", + "56723d0cf7cd8ff07c5f08bf9bc4575b4e2ed895", + "07f85ad7a273e9f7b761265e4a1edf4ef1d45252", + "fb15752f38f89920dc533e20572e1f6980fbe831", + "eb0561c9c4e2d7d8a334b9d58075aeac8393eec8", + "9379fb71b66a2e1412cab7c5b5a5ff9f04632552", + "d06afc106ce644f2deb5530a36aa7878bb0fe9f7", + "a8305e82e93d03322a704cb09d3caf09d14bad1d", + "c5b5450dab4e2fbef695e16856ec44228688c9ae", + "2b0709d61427694b5c21140021b61a0d402e9c21", + "6615f70c912e4af82f2149ae08a0dfa8e0ed1a2c", + "d1ca0e8c4774c776703106c7bd1efbd22ef44bcf", + "08f77c1486bf2028cea594d13cc1c1a7a3a8ca75", + "f86fc748ae0c1744484609f65abdfd9af59d258d", + "45b4c981e5f3fbb804b711787e4d5ac98f5942a9", + "6e8770f0d1a953d23ed475b47c299cd44a9fe7ee", + "bf41f3ac094a68854f2af8ab92d7752435a6e2af", + "2b77b4130f0de2e3e3846f8c51ece58720b0bad9", + "84a438ed9baa75c499ac51e67f1ee831cbacde57", + "59a3f1806f2535298003582703b4d43d52474999", + "8d7652776678320027c8f13a3af826e28423ae2b", + "3e388f6c6905dcb158fa1969b8a23ce3638bea6e", + "2aa452611d542482c8f854b565b5311aad3542d3", + "115a28148283de8ea6314aa468212541b50906aa", + "bcf9dfa654964e72b32b3ac74a34aaef2b5ec3c0", + "a087ce57288aab8945cdf4aca4ba554eb46fec94", + "69074330666ab52f4386e0ef3fcff622d0d63c93", + "e13e2ea3dbbc9c7815e39c3749b4480514f555a6", + "82e4206fdc6e045f1c6f0f372995a5c943f15237", + "588bc92c33dfad516c730819a763175cc1b1fccf", + "000b9f6baeb60071512616d345c355a5c1ff5de3", + "6e97d0dde7947a2e2d42fc8069c4d2c0d4c8817c", + "097ac91926e57cfdcf05f76144946ebe664123e7", + "7acfded6f6171adcf2466ce83d20c4cb27e74a62", + "4885cffc1b986dfbf29184c7ec7603f2cbb8447d", + "c9257ae75101d9731483ac39d5771f382e6da39c", + "4e6ffe5568d7864f540d99c7d4ea9b9250ed30f2", + "1fcf0271646951e9b89c7303a9c3519fcf4c08ea", + "9f68279ef0e04b184bd133ddb395b3e8b13c41ba", + "699c47fa4b5c9ec4056b59b351761ccb6402072a", + "8438e359ffa7be7ba0b375404b54ce36687241ff", + "92ae1834e8ff26f9d9b15b4afb9ea36556503c2d", + "a8a2bded92d86f6ef4f16e92e303896198dc92b5", + "950bbbca54a0bd723341882fcc354a16568ec4ac", + "c5f469cf2f823889aaa53fbf90a1c41f4f4bb891", + "6054f0fd118073a08752807758629bab61cf258a", + "408d3a678a8b72d6f20e52ca5df34834fb8bc7c6", + "4e0e123f5012a4154bbe4c20194c7c021be69c77", + "4c29ea1bcf0cffc114de0f49c9162fce894f4a4f", + "71d0b9f08ab6bbdd4fe0234b0486c52dd45498e7", + "0974c3717fb71650a25898f4067eaf861d52f93f", + "d79abd10f12bd15f7be6a4b737f3c5d891a245c4", + "816905345646edb682078451ec7e1fc6c549b68b", + "598d6a5005c06d15e74ae621fa93e08dfe5884ff", + "b3b3190d0ea0a8b8e00a9378892f23d3af362b61", + "b6d5842c64f903b63cd61b79212e9c4b2a21a9a5", + "5574d3c433b99861ca3452a60be6cff1060e6a0b", + "f5ba975e5d1f9d862bb1d43621da76c14b6403e8", + "482c1feb328e8e83edf55f6053fc592b09a8b9cd", + "418d5768ee3a81f5f6c711e967f5bb2aa70ed2f6", + "7298334213411e218b7bc15aa99dcaa80e0910ca", + "61362e7b629e018ccf41293eca0b6fe382162e09", + "81a19667ebd56b604c7a158702b270b517fdf0a4", + "92e25e207bddd35facb23ee9d15ded30b753aca0", + "3cdaa69bca7bcc1fae6013095a979a95708dfba4", + "cfbb662982caf238e9006810045d70e1285e33b6", + "7521a1572358a038a77bdc3d4c4c19c8e6f0549c", + "d7cf0ba471fbc278aad355b829b442c77aad7368", + "cf7d262d74679cd874d143a245a7bf3337383dc5", + "176d1f794a0779a5c22f786828602ac4d8063a65", + "37395b3ebf25b2680159526750f9e370b8ed134f", + "8ec31edb38353f95bb9cac14a73d7ee861d22390", + "0afcfd4c3c7edd3c039057cb0ea1d8c58bee6c62", + "b584759103b9fa3d1b2688394bfee2ccb7246094", + "cdec5738cfc2c09b51c44d0958222c63b64f5606", + "56e217b077abb33045bfb2dc88dbb0f3de3f8eac", + "c2191a3fb10641d904107c54e08e34778b4abc01", + "efaec62cd396d3f71a4a522b3a7ae4948e335a12", + "4ef3f49b137ddda009973360e7b5118b535ed919", + "33ec4a7d1f46de9c0ff62c966b332689a983c008", + "41a6d7baa54df23e06b1b7513c9218d12b11cdbe", + "bbecaeeaaf38425d9085d759ded5ad845e74406f", + "613408509f896c68616a2df6c9c6d3fa6662f519", + "e9ec616d2f10323199073af1b25a586f1619ecb9", + "1af4a5ecd0c05f722ca04dfd287d892e11677b5c", + "eb01951c9598f15b2bf35b1c653aedbe87c49f73", + "f02def8ae7d7fcff0deb967f2f11d88e9d099b4d", + "b7fd0d56b2e89e4624ccd81f1dd50a2cc8c2e091", + "0db83dd13da549be953a316ba32826d974eed8c2", + "f69b35c8cff251e82fa33b82acb48c60ea05bf3f", + "ea75b9f21956341d325fdce4e38b6db41603f09c", + "245a86c80a2b2e1515a80a217aa7afdc139a8d9c", + "cf0fc461ac82eb8e9e2ca121a3cb1865533283aa", + "ddf1d2a883276c1575606fb9794df53fa0e0b400", + "31d58a849958079807963ee428d211200538d66e", + "59b4daee8f3db173f6a170b455ed840ac80679da", + "26ded1686b33f7ed2ed5d73f95eceb740b8d4387", + "f8dae005ddb31b54275ec4333adb317902a4f9f5", + "6d3cb50f1d58a0c816ee9c187c0408e6462f1f1c", + "6bbb4283c62e81f7f2788d468bbaa88685a3306b", + "4257362a3e91d29a34ccf1d27f5e5d645c468cd7", + "131f3254e45c071d618b500e06dff22f444dcf48", + "1723e19174c7ef3aa13e3b368eae0f979b26d790", + "1fbeb72867dc079fc4857de481fa1220e8362853", + "1f9128710b2dd8fb396ddf70c2b0cc6b36014271", + "ac7ba305e8216a9aa115bf63f9358915f1306b4f", + "a9cd1ddef78f4bf5476bf0c46be548bcd794ca3f", + "7c50f4f5e5f2799f05c2137453c34cdfd3ee150c", + "32b26bb692e612f1d0264b1ed3ac8a38c9dc4ac7", + "a423e48a52edc03febf258af9a86d383be728842", + "2ef97728fcc962749f5b2f4c20e772d84f342c82", + "b1987eba6fa6f5c40f987504a2fe2aaa1a34bde8", + "84ae82df0065de23110e705e4d09bb06f38bc291", + "a9ff271af5500fcd4b1609b964f80aad8105d4b9", + "4c9225a17aad6cc3e32d747b6960f4bc9697242b", + "3d516202c3259dbd9b4783c26072a0029e405e53", + "5fe2cca931aa907e46855b5f6a1c5863b6b2e762", + "abd97578e14c62c3e7127b70c46153cbd9421ed6", + "ee04eb7c0f1896611fec9f054c82be45b8246a21", + "3b4b717030770423ddb1f9da18e454203d318cc0", + "0b7c72d9a5b2dda6ceeefbbe23220d19a5076414", + "af1e59d8cb5d8eaaebe73a11132cdadad902886f", + "71439735b8e1e17d8bfa5c0ee91aa539c949c6bc", + "317101e03b8fe642589498f4bdb84541ab466062", + "a310095a5e5a0b085b5685300a9d1b757b839cf2", + "28849f5da2a875129ba4cdfd46c6ddb1f750a9a7", + "3305a76b4be86519655b5d27ea24627b3239c9c6", + "2ddb42eb26d6b19c66fcb75a626931bf5f4eb46f", + "32c69eda9af620dc6c3cf293546bd78226fcaf2e", + "c27611fffa3c59dddacc0152ce767207b3773a27", + "145618fb84bf448299e91704c3b82a851bbe87e0", + "6eaf5d4cadaf5fcbca4b9e63cbe8c9f2de95917c", + "0925f99493413ffa512b2f9278d12f0f81d43797", + "40e5c1a4dc420d35698d8d90c1daa6dc2eaff7ab", + "5331019bef5906209c7661f24adefca29e2fe9b4", + "210f14e753c80064ec1bcae72f1d654dd9b0e687", + "0e6e9ad98496ace25ae4b4c3bf1754f718061548", + "3a829c5bde408289cddac5ce23e1dc687cfbb154", + "c78f9148873c2e0f8e6dd9248fe3d0c4d93015f5", + "853424a7406f3c85d8090d697f6964cd5e9ed674", + "b2b01e6c5ca5e9d11c58a4bd7517c106affd53a2", + "593a08c8b28f8dca79230774c2b5158549976303", + "ad7ed8aa8e3acfe3d0e2efa8a0592c1119a54133", + "26aaa4f2533324ffee8f70e2aecffaf44a7a1d4e", + "a06a278fb2599177b063bfc56e1abf0591a2fde9", + "7100c159f552c6262c82954e3626ffeabeb4f421", + "c57c5efb6380da3ebc408113f836aa079d3638ee", + "7f0357977e3bfcf7484ca7032dbb0146d48f3a14", + "0cb2c4eca79737ef1cdc0d00899a519b5b630551", + "db3ebdaab2b66808a2063c4e732003606fdf8adc", + "c0058d9452a45c5192c2b47cf80d71e75c191d78", + "8ebeb45c666f938fec64fb7d0b5b7259c6cbe608", + "1730741cc09f9724a53bb23476ab0aa55828a249", + "cc88e9cc38c44719aebcf16f4333fab0e75ab60c", + "da549f59cd2113ac8d1ca133e80ecedfeb4db6bb", + "a24c4d8320cf5aecfc3797a91f3b9c76a5a7233e", + "83d8d49757658d7a742526ba5c72250244fd57d9", + "2cea69eb79fe358f63223e4a83058981461bbc77", + "73118049bb38a2f854c8c1280188a96839a1663e", + "1729091101160b4f1a78b60f86e2a8f9eb7cc9e3", + "42599e7dfd21a23dc24f18dc650066cd69a6a9cd", + "7eb8c94a909187eb08219891742e8eb60bc265fc", + "d073cb1f08ba1cc193331e6b0778b25be3397fc9", + "4a64f9a844e0af8cab93cda2f1b03bde083008f9", + "74d10cab153ab6d08f9b8c1cfa7f0a3ba78ae15e", + "27d8e086071759e4a642195e06fd952da6de2928", + "ec59d8731a967cb33044ba90d725565cf85cc07f", + "635bccec53fd04a22367ed387295f2ef38a50b79", + "5ff8eb6998ac8fa894f8db5a4f32e2e4fbed72f9", + "4394a3bc03e92c2a7c99f6584b86de85e41a96f1", + "fed2736a78077642a73256045c6e67985a2a8522", + "1d95d26d7c15aaadded91503dd461a034aae16d4", + "9d5437bc483e714468f9fa343e34ea098c40fb6d", + "3900059f7cbd83537475837d9e4a2243853cbb40", + "e22f91d9229bee5cac23b11a212315ad07088927", + "e9ef0092c9e97a415e677e0568f57a76100516bf", + "e12248c0e1996e5c2f1c618fbb47da4f6cb3ff43", + "49bf1a22fd3a906c6dde59819a404095b64834dc", + "aeb2fe1c274be79898e4a0ce1345cd2d260c48ad", + "aa5cce1e6c279364fba5a63d94d48835214dec3d", + "313c0031bbeded98d1bdbf2ed8ff785e73615d38", + "da2c748aad55c1a1dcba00b60fe4a35017bc6540", + "591cea1b074b56367f25ec0084a0c4ab66b42a7e", + "e9fa12b965897f485f8ec143e048a424b52f689a", + "874ffe5839095bd9b181eb465ec20fb473cda7c5", + "febccbdc19e0fc80761797dc52811dc056c0a39e", + "62e38a72a41d072f70ad78baa402d5ca4bb13022", + "460c375ff6889a2ad7eeffbdc246eedf767b6136", + "af24e419f48e543ced58dbca99ab39dc48ab6b11", + "5ab3a2849d68a074ebc70d0ac3aa4525804cea0f", + "486dda64980163cf4ab642b00514fcca169dd148", + "c94044ef540811ec9546606c8f641ce8c7fb7495", + "cc5f7693398ffd9844316ef7c16233f106bee8d7", + "8e50188e183222ffeb44f9729dde056a33dded38", + "953756fcbc88d5c17943faee243bd3efe9a5815c", + "0ec476f32193c33d024ec99d26e2cea0d74168db", + "d746d4625b2cb306f2c7427c5c97efc6d25fcb34", + "f69ab255a58896e066a754aeea1e4381c4e858b0", + "73856a817dcaa91a87cca97f32f47fb0edb8b9a1", + "7f89241892e2385aa1442c2afed93a5542018b45", + "1f3eef621985b2dd0e7e2d3ac265c0f0ba6b0a78", + "841eaea8885f568c5f195f22ff3c8738769f2fe6", + "704d27e3eac9b0ae42e600559f763cacd9a55959", + "9766128dbc8b3b47f154d1112e063183df0f5021", + "619713b3aa1bbaaca174387537a9b089f728f9b9", + "f90201b532678aeadd5933ad31893a5c720ef0a7", + "db5c90a92b6a3218f340af2847934c6549645082", + "5ec48f1be265535c91d4044048bb363fb17893bd", + "cf0af730bf0023484db82d4cc337d2f68541cb6e", + "60837d59cfad2b8adaba1aaf7b459171e4588792", + "14ed3374e4e70ffdbdfef740a6272e92e4a7300c", + "d8aba486f57f013a337f4a48774f161b7cc594ee", + "cd2ab490341f080607792ec86b6c4e61cfb760d3", + "ba8dbd364dd18a16396bf10f722a156241fffcf6", + "3c616d0c3b519cff42ee675fa113bd86c05a77a0", + "4f85a1b7fd2a6dda80872e28ccd1c426ece01b36", + "8f2ee50841e216cd8c192eeb68953118190f105c", + "be39d4eac8653672120843ad9bc715bad8fca05c", + "fdd54cf0d1643a1060147159e1d9ced46402a1a1", + "83a79230dc2a0e30ed0dfc03121a0ae2f9ee6de6", + "5483e909b32272ee294e3518036479498af37902", + "257869d3061d5a8ed1529bf34225d90a2416d51a", + "9e9513b11fff182b08cf8cd6c9c2272460636586", + "d200ddcafb7f7336b0ae0353ea1504153876d6f2", + "fad242d35291de1898242d6737ba02b74a59a2f2", + "d32478ac7c6cb7fdad4dc7a8fdc54f7f171a22fe", + "a8f7249c3c5aef1ed6dfa16e637f13dfde815464", + "fcb566cfb8a86cea2d3666bf65a4f177a49b1313", + "a26cc7479b14cdf70433919bfc228625f1c5c6f2", + "092dedada188f56c5915f74a5fd40aac093742c3", + "a3308ea8d0cc513727c8ad149b1132317a886c98", + "76117d7031e856687c5bd9f5f99c79b94b1316d7", + "9637782beb62abfc9a53a59ab762d39f6ae6fb29", + "51662dfb31fe27ffd77d375397c539f31ed5a648", + "a4ab5378e4cc20b30ef94848cca5700af46449fe", + "3e85179a5601bf322bff7697cfc40d5212249707", + "516bd724d2f4820c822d94af2b3b523dd0a87ae3", + "5db75ff7f6a7c6bb5ae534dd5c618c4e787e7ff3", + "48c78d90a3625cae8716bb20091bda775378b075", + "67e2663dd470950473d6183be345f6d8b1269165", + "aed079ec655046f6a876654d7c36f388682b777f", + "62f33637281ffb45ca8fa91b9411e1cd73e06dab", + "2da539ab426f347c58640a45f5428a80b4ca930f", + "25ca9884a186e07efd9c6ad0c26ab53552db2f11", + "4dd1f01c26e4b24e691075f1b78bcbf90f7f19b1", + "6036065906a8bff074116960cd7dd39db9887b56", + "ba9506eb36a78cd081ea10ffcc543e0f1abd840b", + "e083dfe79605163283d635989662937bd34dbe79", + "055c65edc75d5c6e95b96d2094aadda49b44c965", + "5ba80ae9a40991b8f01c69debd791ffdad5ceb2d", + "969a93e25e377c6990508e8d0bc86aa526812f59", + "bebca33f522165afbdd23f95118715ff6637a731", + "afd733ebdb9260f143953f817a3ce31855ba8812", + "d64ae050c7936eb0a5d00144900f892fc826707c", + "174c264c0a59e84e3c4d2a283e94d6981606a8be", + "4703b76221f1cc7194e11e8af2b9d5959c672a55", + "c59f24f76e02a8d4ecd2aea4309d185d7ed13128", + "5dda3276064d871b8372501271cd62264efb8d89", + "9b171118ea9a3207fe3992d56104582c5fb75be5", + "c51fb9761ebdbf9f6fb46300e047635b1c8afcd3", + "33c8d5f15521969a09ea245c076e541d768b5070", + "0ef5917b3a6742dc69b6758b3e40a7beb8b97d04", + "f8cc276bb0add12debc9eddce79e2a842bac5d3e", + "b5e96eef9e61e53c947e3ba407d4025d71375af9", + "571cc734cd69fb846032ec77a662aa8ee4fc32cd", + "6b38eb50104901e2fec80f9455a972bc3eced813", + "b1048c6b106073c8e51dd61d3104200dfbe7f807", + "4d104d325b96c348d685956c6678139640c15c4b", + "f83f03f59b00d9ed0efc0f372c9a153bcd5dc56d", + "28f7a2cba34c51ebccbc4e99acd4cb7cbe07c71a", + "541214b84dea7f2a94e5a5ca9506c4a0aadcda28", + "26112958664f295cd0ffa28936fc940aa001acda", + "4a329372d9257c59929624e8d05d67124d3b84a3", + "f192f08d8f7b27bbb61ad905764255c83d684cdf", + "0a70a726ce116e2d51c65a2b7282ca2134325759", + "0ecfa6323d4bcc400f1a32c6693ef11691c26448", + "77eaea773f952715656a1734e24f274d41e751a7", + "18d883be86974f1836675db59f277861de57c750", + "4e74daad2f74bd836eec04ff2f4202bee973107c", + "24a12d6f9278a4ab2928475e8ce06a0f59ae73d4", + "59c181fb6e48f7c58e7fe12a1dda10d2183d0e2b", + "cc02f9afca03970a97c2ddea6d36df3b864e7e18", + "e4326a39e36a0b72b15adc95c91671e6a6dcce66", + "1441c354eec2ab2d25319e1d96b33ef347123f47", + "3007f0538458a697749520100901aa80885aeae2", + "de3a4883c3197a29abafab5e87a0a8bead421fbe", + "84355587cb2a1f86e6882de83514a32db2646e7a", + "e9b9c9c34e06822ca1b643d0fdccea3efb4d2899", + "cf064b91ac6d89d5cc6014365d3c0d4753cd6378", + "1d74c6cc47df5cbf75b0b71151c80262459af8d8", + "6cb237c5f1c11c10377ab406385617ee783707ad", + "53eb5671945452618ad3288e6f939717b8b446ef", + "685341c6544a6ed2ce488028184430456ca07005", + "9f916c1709074435f18c6bdc6ee968078cb9cd43", + "9de5f9213b29c2469410048b94f783697734d56e", + "a63b2761ba0b97575c4f964ffc0833afb4d39986", + "4ac10c837490a0d2b68e589643b9c0e36f539297", + "e23bb022327c32995eb415991e7fefb5b26a6ec4", + "d5fe075a50ab3ce4f41614d66ed77f152fda134f", + "d67def6716c4ec2a650e7a8d18264fa22aff2750", + "11886177fe1b02839bd4f6d5f2ac12e03d651a7d", + "beb62aaaad0c1701fbce6f3f21777837a424d737", + "6fd77606f4ead7b81d4aa948a88b4064ebd50b97", + "3de9c413664d81786e789a379e9d8ce9b31402d5", + "bbfae30d95cf72ec632a5914265c3f4ab50a9ec8", + "210dc96d7d514abb943a8b314603397c5577ffff", + "14c162416ebe76d1eb4d542af75f876fc2237c1d", + "07d7b63f09b6685278032e189e9f6d52dccb0939", + "8c702cea303c5efd4348d1b6cad58a5a9b545121", + "63f31133a69fb15322546633a334864d6636834d", + "23187bb333d50fdeddba88a9eb769f4cc6df1127", + "f3139b7e8104dbe3da75ab238b86695c75525a3f", + "f451a2bb3874ca0a286a5f09cd15f7fe6470ddda", + "70cdad0e95bb2366a9b2ae9789c0e377fef6e3ef", + "c65070f9cd162346bb724d7bf4f2fa20cde8f1cf", + "a22a7abf2f6d6dda8e2ad2e41aeaeff9d7b2d02b", + "19bccaf083d56fce700beb7550414231e562b4ea", + "6f45838f3cca3ec8f25b0ee62d1d4259b0e85c2a", + "321873914f47d759dce3fc87bb18f77f8455bf78", + "6e612b12d7fee334726651eb156c57151c64ab7f", + "dfe8b4600c2f80fddba6835b21a6091939882a5e", + "8d0ae097adba4f90a2fad325abc5585ad8b9966e", + "5b5dab9fb89c0ab7187ddc334d8995bd24392e41" + ], + "blocked_refs": [], + "reason": "Since ~2026-10-15 GitHub validates a called reusable workflow against the CALLEE repository's actions.lock at the called SHA and rejects a dependency record whose ref is the bare commit SHA (`Unable to resolve action actions/checkout@3d3c42e5…: invalid lockfile pin`). Every standards commit listed here carries such a record, so every reusable of standards at these revisions dies at `Set up job` for every caller: the run reports `failure` (not startup_failure), the job that dies is the reusable's first one, and the message names an action the caller never wrote.", + "known_good_version": null, + "known_good_sha": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "replacement": "pin_to_known_good", + "evidence": [ + "hyperpolymath/cicd-squabbler run 38846539059 (2026-10-16) — first red, byte-identical lock and workflows to the last green run 60e8f7a (2026-10-15)", + "hyperpolymath/hypatia#929 — five callers re-pinned to 75bd599, jobs created again", + "hyperpolymath/cicd-squabbler#135 — seven callers re-pinned, lock closure rebuilt from standards' own lock at 75bd599", + "standards/scripts/lock-selfcheck.sh rates these SELF-CONSISTENT: it checks that every ref is KEYED, not that a key's ref is a tag — this list was produced by a walk of every commit reachable from standards main (1324 commits, 545 with the rejected record shape, 2026-08-19..2026-10-19)" + ], + "added": "2026-10-19", + "added_by": "cicd-suite#54 follow-through (session 01Xn9nqHVbUTRLgBdedoXAc8)", + "notes": "Keyed on SHA only: standards reusables are pinned by commit, never by tag. The list is a measured window, not a policy about age — a standards commit older than 2026-08-19 has no actions.lock and is NOT in it. Mirrored by gitbot-fleet scripts/lib/stale-reusable-pins.tsv (Window B), which is what fix-stale-reusable-pin.sh rewrites from; regenerate both together when standards fixes or re-breaks a window. 75bd599 is the first commit on standards main whose lock carries no self-SHA action record; the toolchain-input window of June 2026 (rust-ci-reusable only) is WF027 in workflow_audit.ex, not here, because it kills one reusable, not all of them." } ], - "moving_refs_never_lockable": [ "stable", "beta", @@ -56,7 +630,6 @@ "master", "HEAD" ], - "safe_change_classes": { "bump_ci_patch_minor": { "description": "Dependabot/renovate bump of a github-actions dependency within the same major, touching only `uses:` lines.", @@ -64,7 +637,12 @@ "method": "squash", "pool": "P2", "safety": "arm_auto", - "requires": ["author_is_dependency_bot", "semver_patch_or_minor", "no_denylisted_pin", "delta_is_pin_lines_only"] + "requires": [ + "author_is_dependency_bot", + "semver_patch_or_minor", + "no_denylisted_pin", + "delta_is_pin_lines_only" + ] }, "bump_lockfile_only": { "description": "Dependency bump whose diff is confined to lockfiles (Cargo.lock, mix.lock, package-lock.json, go.sum, flake.lock).", @@ -72,7 +650,11 @@ "method": "squash", "pool": "P2", "safety": "arm_auto", - "requires": ["author_is_dependency_bot", "no_denylisted_pin", "all_files_are_lockfiles"] + "requires": [ + "author_is_dependency_bot", + "no_denylisted_pin", + "all_files_are_lockfiles" + ] }, "chore_meta_only": { "description": "Non-code chore: docs, licence headers, linguist exclusions, CODEOWNERS, issue templates, .gitattributes. No runtime path.", @@ -80,7 +662,11 @@ "method": "squash", "pool": "P3", "safety": "arm_auto", - "requires": ["no_denylisted_pin", "no_code_path_change", "no_workflow_semantic_change"] + "requires": [ + "no_denylisted_pin", + "no_code_path_change", + "no_workflow_semantic_change" + ] }, "pin_rollback_denylisted": { "description": "A pin currently on the denylist is replaced by its known-good SHA, in place, with no other edit.", @@ -88,10 +674,12 @@ "method": "squash", "pool": "P1", "safety": "arm_auto", - "requires": ["every_changed_line_is_a_denylisted_pin_site", "replacement_equals_known_good"] + "requires": [ + "every_changed_line_is_a_denylisted_pin_site", + "replacement_equals_known_good" + ] } }, - "never_auto": [ { "id": "NA-001", @@ -124,7 +712,6 @@ "reason": "Automerge is for machine-authored chores. A human PR keeps its human reviewer." } ], - "meta_guard": { "comment": "Change-level is `meta` (and therefore safety=flag) whenever the delta reaches CI, the oracle, or policy. The single exemption is a pure pin substitution — the actuator must re-prove `pin_only` from the diff itself before honouring it.", "meta_paths": [ @@ -136,7 +723,11 @@ ".machine_readable/merge-orchestration/", "stdlib/" ], - "meta_repos": ["standards", "rsr-template-repo", "hypatia"], + "meta_repos": [ + "standards", + "rsr-template-repo", + "hypatia" + ], "exemptions": [ { "id": "MGX-001", @@ -146,7 +737,6 @@ } ] }, - "lockfile_names": [ "Cargo.lock", "mix.lock", @@ -162,7 +752,6 @@ "Gemfile.lock", "composer.lock" ], - "dependency_bots": [ "dependabot[bot]", "renovate[bot]", @@ -170,7 +759,6 @@ "app/dependabot", "app/renovate" ], - "absence_rules": [ { "id": "ABS-001", @@ -197,7 +785,11 @@ "id": "ABS-003", "field": "chrome", "test": "missing_required_file", - "required_files": ["LICENSE", "README.adoc", "SECURITY.adoc"], + "required_files": [ + "LICENSE", + "README.adoc", + "SECURITY.adoc" + ], "severity": "medium", "auto_fixable": "seed_from_template", "why": "Standard project chrome. A seed from rsr-template-repo is unambiguous; a missing licence *choice* is not, so LICENSES/ is seeded but never invented.", @@ -205,7 +797,6 @@ "issue_body_template": "Missing from the repository root: {missing_list}\n\nRSR template chrome is seeded, not invented: `hypatia` can copy the template skeleton, but the licence *choice* and the security *contact* need a human.\n\nFiled by `hypatia` (ABS-003) with the absence-intake run of {date}. Deduplicated by this marker: ``" } ], - "intake_limits": { "comment": "The 2026-09-26 complaint was that issues had become unreadable. These caps are the cure: intake is capped per run, deduplicated by an in-body marker, and upgradeable — a repeat finding comments on the existing issue instead of filing a new one.", "max_new_issues_per_run": 25, @@ -214,7 +805,6 @@ "comment_on_existing_instead_of_new": true, "roll_up_when_class_exceeds": 20 }, - "stats_thresholds": { "comment": "Live thresholds for the private-farm dashboard. A metric crosses into 'recent/live tracking' when it exceeds these; below them it is reported as a headline number only.", "failing_tests_warn": 1, @@ -232,7 +822,6 @@ "open_issue_count_warn": 250, "open_issue_count_critical": 500 }, - "stats_output": { "comment": "Where the dashboard artifact goes. The .git-private-farm repo is not readable or writable from the public estate — this is the contract it consumes.", "artifact_name": "estate-stats.json", diff --git a/Cargo.toml b/Cargo.toml index edc5d904..703051e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,11 @@ members = [ "integration", "tools/cii-registrar", ] +# standards' governance "JS runtimes" job checks its js-runtimes crate out at +# .standards-js/ inside this checkout and `cargo install`s it from there. +# Without this, cargo treats that crate as an undeclared member of this +# workspace and refuses to build it. +exclude = [".standards-js"] [workspace.package] version = "0.1.0" diff --git a/README.adoc b/README.adoc index 4e3d76d4..0feac51a 100644 --- a/README.adoc +++ b/README.adoc @@ -104,7 +104,7 @@ The escript exposes five commands: [source,bash] ---- -hypatia scan [--format json|text|github|sarif] [--exit-zero] +hypatia scan [--format json|text|github|sarif] [--exit-zero] [--advisory] hypatia report hypatia pr-eligibility --owner --repo --pr hypatia version diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index db17e4d0..ee731f27 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -89,7 +89,13 @@ Every examined PR gets one line in `pr-decisions.jsonl`: | `blocked_by` | `awaiting_required_checks`, `introduces_denylisted_pin`, `introduces_denylisted_pin_and_major_bumps`, - `introduces_denylisted_pin_alongside_wanted_updates`, `major_version_delta`, + `introduces_denylisted_pin_alongside_wanted_updates` (armed only when what + remains after the excision would arm on its own), + `introduces_denylisted_pin_alongside_unmergeable_changes` (it would not; + `remainder_blocked_by` names why), + `denylisted_pin_context_unknown` (the hunk shows the pin among script text; + brain only — the shell actuator does not yet read hunk context), + `major_version_delta`, `version_claims_conflict`, `pin_delta_unresolvable`, `dependency_manifest_not_lockfile`, `touches_licence_requires_owner_review`, `repository_is_archived_cannot_merge`, `not_a_dependency_bot`, @@ -127,11 +133,19 @@ Find every denylisted pin in an estate; with `--rewrite`, produce the substituted file trees ready for a branch and a PR. * Flags: `--org` (repeatable), `--policy`, `--out DIR`, `--via search|clone`, - `--rewrite`, `--max-repos N`. + `--rewrite`, `--max-repos N`, `--sample N`. * Artifacts under `--out`: `pin-findings.jsonl`, `pin-repair-plan.jsonl`, `patched//` trees; one squash PR is planned per repository - (`hypatia/pin-integrity-known-good`). -* Exit: `0` clean, `1` error, `2` findings, `3` nothing to do. + (`hypatia/pin-integrity-known-good`), and only for a repository whose every + confirmed file is repairable and whose every candidate file was read. A + repository with a flagged sibling (an `actions.lock` entry, a fixture) or an + unread candidate (unreadable, or left out by `--sample`) is held for owner + review: repairing the workflow alone would arm a merge on a repository that + still carries the pin. A search that failed or came back partial may hide a + sibling anywhere, so then every repository is held and no plan is armed. +* Exit: `0` clean, `1` error, `2` findings or an `INCOMPLETE` run (failed or + partial searches, listings or clones, unread candidates, or the limits + `--max-repos` and `--sample` impose), `3` nothing to do. * A repair that cannot be proved is a finding, not an action: the substitution refuses when the policy carries no `known_good_sha`, and it refuses when the rewrite would drop a `uses:` site. diff --git a/docs/wiki-pages/Operations.md b/docs/wiki-pages/Operations.md index 8de53245..a0af81c6 100644 --- a/docs/wiki-pages/Operations.md +++ b/docs/wiki-pages/Operations.md @@ -62,6 +62,7 @@ The supervision tree starts Bandit on port 9090 (9099 under `MIX_ENV=test`): | `HYPATIA_ALERT_WEBHOOK_URL` | Webhook for watcher alerts | | `HYPATIA_ALERT_LOG_FILE` | File sink for watcher alerts | | `HYPATIA_EXIT_ZERO` | Make the CLI exit 0 even when findings are present | +| `HYPATIA_ADVISORY` | Also report advisory findings (WF026, WF027) below the severity threshold; they never set the exit code | ## CLI @@ -69,7 +70,7 @@ The escript supports exactly five commands — `scan`, `report`, `pr-eligibility `help`. Anything else exits with "unknown command". ```bash -hypatia scan [--format json|text|github|sarif] [--exit-zero] +hypatia scan [--format json|text|github|sarif] [--exit-zero] [--advisory] hypatia report hypatia pr-eligibility --owner --repo --pr ``` diff --git a/lib/hypatia/cli.ex b/lib/hypatia/cli.ex index 0d06a4f8..64bceb15 100644 --- a/lib/hypatia/cli.ex +++ b/lib/hypatia/cli.ex @@ -33,11 +33,14 @@ defmodule Hypatia.CLI do --format Output format: json (default), text, github, sarif --severity Minimum severity to report: critical, high, medium (default), low, info --path Path to scan (alternative to positional argument) + --advisory Also report advisory findings below the severity threshold + (they never count toward the exit code) ## Environment Variables HYPATIA_FORMAT Override --format HYPATIA_SEVERITY Override --severity + HYPATIA_ADVISORY Truthy: behaves as --advisory """ @version "2.0.0" @@ -90,6 +93,8 @@ defmodule Hypatia.CLI do * `0` -- success (no findings at/above the severity threshold), or `--exit-zero` / `HYPATIA_EXIT_ZERO` was set and the scan ran cleanly. * `1` -- findings exist at/above the threshold (default behaviour). + An advisory finding (`advisory: true`) never counts: see + `report_sets/2`. * `2` -- error (bad arguments, scan failed, etc.). Findings are always written to stdout in the requested format. A @@ -107,6 +112,7 @@ defmodule Hypatia.CLI do help: :boolean, version: :boolean, exit_zero: :boolean, + advisory: :boolean, # pr-eligibility flags owner: :string, repo: :string, @@ -126,13 +132,15 @@ defmodule Hypatia.CLI do format = opts[:format] || System.get_env("HYPATIA_FORMAT") || "json" severity = opts[:severity] || System.get_env("HYPATIA_SEVERITY") || "medium" exit_zero = opts[:exit_zero] || env_flag?("HYPATIA_EXIT_ZERO") + advisory = opts[:advisory] || env_flag?("HYPATIA_ADVISORY") config = %{ format: format, severity: severity, rules: parse_rules(opts[:rules]), path: opts[:path], - exit_zero: exit_zero + exit_zero: exit_zero, + advisory: advisory } case args do @@ -191,26 +199,52 @@ defmodule Hypatia.CLI do severity_floor: config.severity ) - # Filter by severity threshold - filtered = - findings - |> Enum.filter(fn f -> - sev = Map.get(f, :severity, "medium") |> to_string() - rank = Map.get(@severity_order, sev, 5) - threshold = Map.get(@severity_order, config.severity, 3) - rank <= threshold - end) + {shown, blocking} = report_sets(findings, config) - output(filtered, config.format) - emit_finding_summary(filtered, config) + output(shown, config.format) + emit_finding_summary(blocking, config, length(shown) - length(blocking)) cond do - length(filtered) == 0 -> :ok + blocking == [] -> :ok config.exit_zero -> :ok true -> System.halt(1) end end + @doc """ + Splits a scan's findings into `{shown, blocking}`. + + `shown` is what is written to stdout: every finding at or above the + `config.severity` threshold, plus, when `config.advisory` is set + (`--advisory` / `HYPATIA_ADVISORY`), every advisory finding below it. + `blocking` decides exit 1: the shown findings at or above the threshold + that are not advisory. + + An advisory finding carries `advisory: true`. It names a repair a fleet + bot can route (WF026, WF027), and its rule must never fail a gate + (AGENTIC `must-not-promote-to-blocking`). Before this split the only way + to see one was to lower `--severity` to its level, which also made it, and + everything else at that level, fail the scan. Advisory output is opt-in + rather than always on because a gate that rejects every unbaselined + finding it is shown (standards' governance baseline gate runs at threshold + `info` over a default scan) would otherwise start failing on them. + """ + @spec report_sets([map()], map()) :: {[map()], [map()]} + def report_sets(findings, config) do + threshold = Map.get(@severity_order, config.severity, 3) + advisory? = Map.get(config, :advisory, false) + + within? = fn f -> + sev = Map.get(f, :severity, "medium") |> to_string() + Map.get(@severity_order, sev, 5) <= threshold + end + + shown = Enum.filter(findings, &(within?.(&1) or (advisory? and advisory_finding?(&1)))) + {shown, Enum.filter(shown, &(within?.(&1) and not advisory_finding?(&1)))} + end + + defp advisory_finding?(f), do: Map.get(f, :advisory) == true + # ─── Report command ─────────────────────────────────────────────────── defp run_report(path, config) do @@ -222,22 +256,14 @@ defmodule Hypatia.CLI do end findings = collect_findings(abs_path, config.rules) - - filtered = - findings - |> Enum.filter(fn f -> - sev = Map.get(f, :severity, "medium") |> to_string() - rank = Map.get(@severity_order, sev, 5) - threshold = Map.get(@severity_order, config.severity, 3) - rank <= threshold - end) + {shown, blocking} = report_sets(findings, config) # Report always uses text format with extra detail - output_report(filtered, abs_path) - emit_finding_summary(filtered, config) + output_report(shown, abs_path) + emit_finding_summary(blocking, config, length(shown) - length(blocking)) cond do - length(filtered) == 0 -> :ok + blocking == [] -> :ok config.exit_zero -> :ok true -> System.halt(1) end @@ -266,7 +292,7 @@ defmodule Hypatia.CLI do # signal that includes severity counts and the exit code about to be # returned. See hypatia#213. - defp emit_finding_summary(filtered, config) do + defp emit_finding_summary(filtered, config, advisory_count) do counts = Enum.reduce(filtered, %{}, fn f, acc -> sev = Map.get(f, :severity, "medium") |> to_string() @@ -294,6 +320,17 @@ defmodule Hypatia.CLI do if(config.exit_zero and total > 0, do: " (--exit-zero suppressed exit 1)", else: "") ) + # Advisory findings are written to stdout but counted neither above nor + # toward the exit code (report_sets/2); their own line keeps that line's + # shape. + if advisory_count > 0 do + IO.puts( + :stderr, + "[hypatia] advisory: #{advisory_count} finding(s) reported, " <> + "not counted toward the exit code" + ) + end + # A separate line, so the summary above keeps its shape for any caller # that pattern-matches it. Levels are derived (Hypatia.Verdict), never # stored by a rule. @@ -486,7 +523,23 @@ defmodule Hypatia.CLI do # structural_drift normalizer's `Map.take` above. Before this, # 7 `fix_recipe:` and 7 `recipe_id:` declarations in # workflow_audit.ex were read by nothing at all. - |> Map.merge(Map.take(f, [:fix_recipe, :recipe_id, :job])) + # WF026/WF027 anchor to a line and name their repair script and + # the SHAs involved; keep those so SARIF points at the line and + # the fleet receives the advertised script and replacement. + |> Map.merge( + Map.take(f, [ + :fix_recipe, + :recipe_id, + :job, + :line, + :auto_fixable, + :fix_script, + :current_sha, + :replacement_sha, + :reusable, + :advisory + ]) + ) end) results ++ normalized @@ -1544,11 +1597,16 @@ defmodule Hypatia.CLI do --exit-zero Always exit 0 after a successful scan, even when findings exist. Use in CI when a downstream step gates on severity counts. (See HYPATIA_EXIT_ZERO.) + --advisory Also report advisory findings (advisory: true, + e.g. WF026/WF027) below the --severity threshold. + They never count toward exit 1. + (See HYPATIA_ADVISORY.) EXIT CODES: 0 No findings at or above the configured severity threshold, OR --exit-zero / HYPATIA_EXIT_ZERO was set and the scan ran cleanly. - 1 Findings exist at/above the threshold (default behaviour). + 1 Non-advisory findings exist at/above the threshold (default + behaviour). Advisory findings never cause exit 1. 2 Error (bad arguments, scan failed, etc.). ENVIRONMENT: @@ -1556,12 +1614,14 @@ defmodule Hypatia.CLI do HYPATIA_SEVERITY Override --severity HYPATIA_EXIT_ZERO If set to a truthy value (1, true, anything non-empty/non-zero), behaves as --exit-zero. + HYPATIA_ADVISORY If set to a truthy value, behaves as --advisory. EXAMPLES: hypatia scan . hypatia scan ~/repos/my-project --format text hypatia scan . --rules root_hygiene,code_safety --severity high hypatia scan . --exit-zero # CI: emit findings, never fail step + hypatia scan . --advisory # also report routable advisory findings hypatia report . > report.txt HYPATIA_FORMAT=json hypatia scan . """) diff --git a/lib/rules/actions_lock.ex b/lib/rules/actions_lock.ex index e1fe0ad6..3a4f8cee 100644 --- a/lib/rules/actions_lock.ex +++ b/lib/rules/actions_lock.ex @@ -106,6 +106,15 @@ defmodule Hypatia.Rules.ActionsLock do def pinned?(_, _, _), do: false + @doc """ + The refs the lock associates with `workflow_path` (`owner/repo@ref` keys), + or nil when its `workflows:` section lists no such workflow. Basenames + resolve under `.github/workflows/`, as for `pinned?/3`. + """ + @spec workflow_refs(t(), String.t()) :: MapSet.t(String.t()) | nil + def workflow_refs(%__MODULE__{} = lock, workflow_path) when is_binary(workflow_path), + do: Map.get(lock.workflows, normalise_workflow_path(workflow_path)) + @doc """ Return dependency refs from a fully validated lock. diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index c7cd6727..3124d43f 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -53,47 +53,118 @@ defmodule Hypatia.Rules.PinIntegrity do or to the owner as `flag` when the repair is not a substitution. """ - # A `uses:` pin site, in block form (`- uses: a/b@ref # v1`) or KYAML form - # (`uses: "a/b@ref", # v1` and one-line `{ uses: "a/b@ref" },`). The key - # must open the line — after indentation and an optional `-` or `{` — so a - # `uses:` inside a `run:` string never counts. The value may be quoted; the - # quote, trailing comma and closing brace are not part of the ref. - @uses_regex ~r/^\s*(?:-\s*|\{\s*)?uses:\s*(?["']?)(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}]+)\k\s*\}?\s*,?\s*(?:#\s*(?.*?))?\s*$/ + alias Hypatia.Rules.PinSites @doc """ Parse every `uses:` pin site in a workflow file. + A site is what `Hypatia.Rules.PinSites` reads as one — block form + (`- uses: a/b@ref # v1`), KYAML form (`uses: "a/b@ref", # v1`), and every + step of a compact flow sequence or mapping (`steps: [{ uses: a/b@x }]`, + `- { name: x, uses: a/b@x }`) — so the file scan sees the pins the diff + rules (PA001-PA004) see. A `uses:` inside a quoted `run:` string, a + comment or script text is not one, nor is one on a line of a `run: |` + block or of a quoted scalar continued from an earlier line: the file is + read through `PinSites.mask_lines/1`, which carries that state from line + to line. A line inside a flow collection opened on an earlier line is + judged against the brackets still open (`PinSites.flow_prefixes/1`), so a + step's `uses:` after a sibling (`name: "hello` then `world", uses: a/b@x + }]`) is a site and one in an input's mapping is not. A `uses:` whose reference YAML continues below it (`- uses:` + then ` a/b@x`, blank and comment-only lines allowed between) is a site + on the value's line (`PinSites.continuations/1`), and a `uses: *name` is a + site on the line of the anchor it names, where its reference stands + (`PinSites.alias_sites/1`). + Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first - two path segments, which is what an allow/deny list keys on), - `:ref`, and `:comment`. + two path segments, which is what an allow/deny list keys on), `:ref`, + `:ref_at` (the ref's byte offset in its line), `:ref_len` (its length as + written — a value written with escapes, `"a/b@\\x31…"`, is read decoded, + so the ref's text is longer than the ref) and `:comment` — the + line's comment, trimmed, when the site is the line's only one, else `""`: + `[ { uses: a/b@x }, { uses: c/d@y } ] # v4` labels neither pin. """ @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do - content - |> String.split("\n") - |> Enum.with_index(1) - |> Enum.flat_map(fn {line, number} -> - case Regex.named_captures(@uses_regex, line) do - %{"action" => action, "ref" => ref} = caps -> - [ - %{ - line: number, - action: action, - action_base: action_base(action), - ref: ref, - comment: Map.get(caps, "comment", "") || "" - } - ] + masked = PinSites.mask_lines(content) + prefixes = PinSites.flow_prefixes(content) + numbered = masked |> Enum.zip(prefixes) |> Enum.with_index(1) + # A block `uses:` key the indentation above it puts under no step or job + # (an input of that name, `with:` then `uses: ...`) holds data: no site + # on its line, below it, or through its alias. + data = PinSites.data_uses_lines(content) + + # A `uses:` whose reference continues below it — past any blank or + # comment-only lines — is a site on the value's line. + continued = + for {key, number, site} <- PinSites.continuations(masked), key not in data do + next = Enum.at(masked, number - 1) + site_entry(site, number, next |> PinSites.comment() |> String.trim()) + end - _ -> - [] + # A `uses: *name` runs the reference the anchor holds: its site is the + # anchor's value, where a repair rewrites it (every alias with it). + aliased = + for %{line: alias_line, anchor_line: number, site: site} <- + PinSites.alias_sites(masked, prefixes), + alias_line not in data do + site_entry( + site, + number, + masked |> Enum.at(number - 1) |> PinSites.comment() |> String.trim() + ) end - end) + + # A step or job given whole as an alias (`- *step`) runs the `uses:` of + # the mapping its anchor holds — a site even where that mapping sits in + # data (a matrix entry), where a repair rewrites it. + node_aliased = + for %{anchor_line: number, site: site} <- PinSites.node_alias_sites(content) do + site_entry( + site, + number, + masked |> Enum.at(number - 1) |> PinSites.comment() |> String.trim() + ) + end + + numbered + |> Enum.reject(fn {_, number} -> number in data end) + |> Enum.flat_map(fn {{line, prefix}, number} -> line_sites(line, number, prefix) end) + |> Kernel.++(continued) + |> Kernel.++(aliased) + |> Kernel.++(node_aliased) + |> Enum.uniq_by(&{&1.line, &1.ref_at}) + |> Enum.sort_by(&{&1.line, &1.ref_at}) end def pin_sites(_), do: [] + defp line_sites(line, number, flow_prefix) do + sites = PinSites.sites(line, flow_prefix) + comment = if length(sites) == 1, do: line |> PinSites.comment() |> String.trim(), else: "" + + for site <- sites, do: site_entry(site, number, comment) + end + + defp site_entry(site, number, comment) do + %{ + line: number, + action: site.action, + action_base: action_base(site.action), + ref: site.ref, + ref_at: site.ref_at, + ref_len: Map.get(site, :ref_len, byte_size(site.ref)), + comment: comment + } + end + + # GitHub resolves `GitHub/CodeQL-Action` and `github/codeql-action` to one + # repository, so a table keyed on one spelling must match the other. + defp same_action?(a, b) when is_binary(a) and is_binary(b), + do: String.downcase(a) == String.downcase(b) + + defp same_action?(_, _), do: false + @doc """ The first two segments of an action path. @@ -125,7 +196,7 @@ defmodule Hypatia.Rules.PinIntegrity do policy |> Map.get("pin_denylist", []) |> Enum.find(fn entry -> - Map.get(entry, "action") == base and + same_action?(Map.get(entry, "action"), base) and (normalised in normalise_all(Map.get(entry, "blocked_shas", [])) or normalised in normalise_all(Map.get(entry, "blocked_versions", [])) or normalised in normalise_all(Map.get(entry, "blocked_refs", []))) @@ -138,13 +209,13 @@ defmodule Hypatia.Rules.PinIntegrity do defp normalise_all(list), do: Enum.map(list, &normalise_ref/1) defp normalise_ref(ref) do - ref - |> String.trim() - |> String.trim_leading("v") - |> case do - # `v4.38.1` and `4.38.1` are the same claim about the same artefact. - other -> other - end + trimmed = ref |> String.trim() |> String.replace_prefix("v", "") + + # `v4.38.1` and `4.38.1` are the same claim about the same artefact — one + # `v` only: `vv4.38.1` is another tag, not the blocked release; a + # commit ID in upper-case hex is the same object as the lower-case one + # the table carries, so both sides compare in lower case. + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, trimmed), do: String.downcase(trimmed), else: trimmed end @doc """ @@ -154,7 +225,10 @@ defmodule Hypatia.Rules.PinIntegrity do def known_good(policy, action) do base = action_base(action) - case Enum.find(Map.get(policy, "pin_denylist", []), &(Map.get(&1, "action") == base)) do + case Enum.find( + Map.get(policy, "pin_denylist", []), + &same_action?(Map.get(&1, "action"), base) + ) do %{"known_good_sha" => sha} = entry when is_binary(sha) -> %{sha: sha, version: Map.get(entry, "known_good_version"), rule: Map.get(entry, "id")} @@ -364,15 +438,18 @@ defmodule Hypatia.Rules.PinIntegrity do end @doc """ - Extract `action@sha` pairs from a `gh actions-lock` file (TOML-ish), - keyed by the resolved SHA or ref. Used by PI003 and PI004. + Extract `action@sha` pairs from a `gh actions-lock` file (TOML-ish). + Used by PI003 and PI004. Reads the first single-quoted `action@ref` on each line and returns - `%{ref => {action_base, line_number}}`, with one-based line numbers and - action paths reduced to their first two segments. Later entries replace - earlier ones with the same ref; no matches produce an empty map. + `[{ref, {action_base, line_number}}]` in file order, with one-based line + numbers and action paths reduced to their first two segments. One entry + per action and ref: two actions locked at the same ref + (`actions/checkout@v4`, `github/codeql-action@v4`) are two entries, and + the same pair met again (a workflow's edge and its dependency record) + keeps the line of its last mention. No matches produce `[]`. """ - @spec locked_refs(String.t()) :: %{String.t() => {String.t(), integer()}} + @spec locked_refs(String.t()) :: [{String.t(), {String.t(), integer()}}] def locked_refs(content) do content |> String.split("\n") @@ -388,7 +465,30 @@ defmodule Hypatia.Rules.PinIntegrity do [{ref, {action_base(action), number}}] end end) - |> Map.new() + |> Enum.reverse() + |> Enum.uniq_by(fn {ref, {action, _}} -> {String.downcase(action), ref_identity(ref)} end) + |> Enum.reverse() + end + + # The refs a parsed lock associates with workflow `path`, in the shape + # `locked_refs/1` returns them, or nil when the lock does not parse or + # lists no such workflow. + defp scoped_locked_refs(lock_content, path) do + with {:ok, lock} <- Hypatia.Rules.ActionsLock.parse(lock_content), + %MapSet{} = refs <- Hypatia.Rules.ActionsLock.workflow_refs(lock, path) do + for key <- refs, [action, ref] <- [String.split(key, "@", parts: 2)] do + {ref, {action_base(action), 0}} + end + else + _ -> nil + end + end + + # What a ref names, for comparing two spellings of it: a commit ID is one + # object in either case (`1C5B…` is `1c5b…`); a symbolic ref (`main`, + # `v4`) is a name and compares exactly — `v4` and `4` are two refs. + defp ref_identity(ref) do + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, ref), do: String.downcase(ref), else: ref end # ─── PI004 · lock/spec divergence ───────────────────────────────────── @@ -407,21 +507,30 @@ defmodule Hypatia.Rules.PinIntegrity do def pi004_lock_divergence(workflow_content, lock_content, opts \\ []) do path = Keyword.get(opts, :path, "") repo = Keyword.get(opts, :repo, "") - locked = locked_refs(lock_content) + + # A ref locked for another workflow does not vouch for this one: with the + # workflow's path, the comparison is that workflow's own `workflows:` + # entry. Without a path, or a lock that does not parse or name the + # workflow, every ref the lock holds is the comparison, as before. + locked = + (Keyword.has_key?(opts, :path) && scoped_locked_refs(lock_content, path)) || + locked_refs(lock_content) workflow_content |> pin_sites() |> Enum.flat_map(fn site -> locked_refs_for_action = locked - |> Enum.filter(fn {_ref, {action, _line}} -> action == site.action_base end) + |> Enum.filter(fn {_ref, {action, _line}} -> same_action?(action, site.action_base) end) |> Enum.map(fn {ref, _} -> ref end) + |> Enum.uniq_by(&ref_identity/1) cond do locked_refs_for_action == [] -> [] - site.ref in locked_refs_for_action -> + # The same commit in another case is agreement, not divergence. + ref_identity(site.ref) in Enum.map(locked_refs_for_action, &ref_identity/1) -> [] true -> @@ -486,53 +595,87 @@ defmodule Hypatia.Rules.PinIntegrity do {:error, "no known_good_sha for a denylisted action in this file"} true -> - {new_content, excisions} = - Enum.reduce(hits, {content, []}, fn site, {acc, log} -> - original = Enum.at(lines, site.line - 1) || "" - %{sha: sha, version: version} = known_good(policy, site.action) - - rewritten = - original - |> String.replace("@" <> site.ref, "@" <> sha) - |> relabel_line(version) - - excision = %{ - line: site.line, - action: site.action, - from_ref: site.ref, - to_ref: sha, - to_version: version, - comment_before: site.comment, - comment_after: comment_of(rewritten) - } - - # Whole-line replacement, so two identical pin lines are treated - # identically and neither is touched twice. - {String.replace(acc, original, rewritten), log ++ [excision]} + {new_lines, excisions} = + lines + |> Enum.with_index(1) + |> Enum.map_reduce([], fn {original, number}, log -> + case Enum.filter(hits, &(&1.line == number)) do + [] -> {original, log} + line_hits -> rewrite_line(original, line_hits, sites, policy, log) + end end) - if length(pin_sites(new_content)) == length(sites) do - {:ok, new_content, excisions} - else - {:error, "substitution would remove a pin site — refusing"} + new_content = Enum.join(new_lines, "\n") + new_sites = pin_sites(new_content) + + # The rewrite is proved, not assumed: as many sites as before, and + # none of them denylisted — a `known_good_sha` that is itself + # blocked (by this entry or another for the same action) repairs + # nothing, and is refused rather than logged as an excision. + cond do + length(new_sites) != length(sites) -> + {:error, "substitution would remove a pin site — refusing"} + + Enum.any?(new_sites, &denylisted?(policy, &1.action, &1.ref)) -> + {:error, "substitution leaves a denylisted pin — refusing"} + + true -> + {:ok, new_content, excisions} end end end - defp comment_of(line) do - case String.split(line, "#", parts: 2) do - [_head, comment] -> comment - _ -> "" - end + # Each denylisted ref is spliced at its own offset, right to left so the + # offsets still to come stay valid: a line that carries the same ref twice, + # or the ref's text again in a `name:`, changes only at its sites. The + # comment is relabelled only on a line whose one site was the hit — on a + # multi-site line it labels no pin in particular. + defp rewrite_line(original, line_hits, sites, policy, log) do + rewritten = + line_hits + |> Enum.sort_by(& &1.ref_at, :desc) + |> Enum.reduce(original, fn site, acc -> + %{sha: sha} = known_good(policy, site.action) + tail_at = site.ref_at + site.ref_len + + binary_part(acc, 0, site.ref_at) <> + sha <> binary_part(acc, tail_at, byte_size(acc) - tail_at) + end) + + rewritten = + case {line_hits, Enum.count(sites, &(&1.line == hd(line_hits).line))} do + {[site], 1} -> relabel_line(rewritten, known_good(policy, site.action).version) + _ -> rewritten + end + + excisions = + for site <- line_hits do + %{sha: sha, version: version} = known_good(policy, site.action) + + %{ + line: site.line, + action: site.action, + from_ref: site.ref, + to_ref: sha, + to_version: version, + comment_before: site.comment, + comment_after: PinSites.comment(rewritten) + } + end + + {rewritten, log ++ excisions} end + # The comment is the text after the `#` PinSites finds outside quoted + # scalars, so `name: "C# build"` is not cut in two. defp relabel_line(line, version) when is_binary(version) do - case String.split(line, "#", parts: 2) do - # Mirror estate-pin-integrity.sh: the head keeps no `#`, the comment - # is handed over WITH it, and relabel/2 returns it with it. - [head, comment] -> head <> relabel("#" <> comment, version) - _ -> line - end + head = PinSites.head(line) + + # Mirror estate-pin-integrity.sh: the head keeps no `#`, the comment + # is handed over WITH it, and relabel/2 returns it with it. + if byte_size(head) < byte_size(line), + do: head <> relabel("#" <> PinSites.comment(line), version), + else: line end defp relabel_line(line, _version), do: line @@ -610,8 +753,8 @@ defmodule Hypatia.Rules.PinIntegrity do """ @spec pin_only_edit?([String.t()], [String.t()], map()) :: boolean() def pin_only_edit?(removed, added, policy) do - removed_sites = lines_to_sites(removed) - added_sites = lines_to_sites(added) + removed_sites = Enum.flat_map(removed, &line_sites(&1, 0, "")) + added_sites = Enum.flat_map(added, &line_sites(&1, 0, "")) pinned_removed = removed_sites |> Enum.map(& &1.ref) |> Enum.sort() pinned_added = added_sites |> Enum.map(& &1.ref) |> Enum.sort() @@ -620,12 +763,12 @@ defmodule Hypatia.Rules.PinIntegrity do removed == [] or added == [] -> false - length(removed_sites) != length(removed) or length(added_sites) != length(added) -> + not Enum.all?(removed ++ added, &PinSites.pin_line?/1) -> # At least one changed line is not a `uses:` line. false - Enum.sort(Enum.map(removed_sites, & &1.action)) != - Enum.sort(Enum.map(added_sites, & &1.action)) -> + Enum.sort(Enum.map(removed_sites, &String.downcase(&1.action))) != + Enum.sort(Enum.map(added_sites, &String.downcase(&1.action))) -> # An action was swapped for a different action. false @@ -642,25 +785,6 @@ defmodule Hypatia.Rules.PinIntegrity do end end - defp lines_to_sites(lines) do - lines - |> Enum.map(&Regex.named_captures(@uses_regex, &1)) - |> Enum.flat_map(fn - %{"action" => action, "ref" => ref} = caps -> - [ - %{ - action: action, - action_base: action_base(action), - ref: ref, - comment: Map.get(caps, "comment", "") || "" - } - ] - - _ -> - [] - end) - end - # ─── PI005 · stale pin (advisory) ───────────────────────────────────── @doc """ diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex new file mode 100644 index 00000000..7324b83b --- /dev/null +++ b/lib/rules/pin_sites.ex @@ -0,0 +1,1441 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.Rules.PinSites do + @moduledoc """ + Structural `uses:` pin sites on one line of workflow YAML. + + One reader for every rule that judges a pin by the line it sits on — + PI001/PI002 (`PinIntegrity`) over a file, PA001-PA004 (`PrAutomerge`) over + a diff — so a pin shape one of them learns, the others see too. + + A site is a `uses:` key the YAML structure makes a pin: + + * the line's own leading `uses:` key (`uses:`, `- uses:`, `- { uses:`, + quoted or not); + * an entry of a flow mapping that is a step or a job — an anonymous + `{ ... }` at the line's level or in a `[` that `steps:` (or nothing on + the line) opens, or a mapping a job key opens inside `jobs: { ... }`. + + Text is not a site: a pin inside a quoted `name:` or `run:` scalar, after + `#`, inside an input's mapping (`with: { uses: ... }`), or after free text + (`echo { uses: ... }` in a script). + + A double-quoted value is read as YAML decodes it — KYAML (YAML-POLICY + Y-3) writes every value so, and an escape (`"a/b@\\x31…"`) is the + reference GitHub runs, not its spelling (`decode_dq/1`). A composite + action's steps are its root `runs:` steps in flow form too (`{ runs: { + steps: [ { uses: ... } ] } }`), as they are in block form. + + Every position this module returns is a byte offset into the line as + given: the mask keeps the line's bytes where they were. + """ + + # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is + # not one. Matched against the MASKED line. The value may carry YAML node + # properties before it (`uses: &rust a/b@x`, `uses: !!str a/b@x`): the + # anchor or tag names the node, the scalar is still the pin. + @site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ + + @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*$/ + + # A block scalar's content line that is a whole reference: no quotes (they + # would be part of the value) and nothing after it. + @block_ref_re ~r/^\s*([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)\s*$/ + + # Root keys a workflow or an action defines whose values are data: no step + # or job sits under any of them (`block_rows/1`). + @data_root_keys ~w(name run-name on permissions env defaults concurrency description author inputs outputs branding) + + @doc """ + The structural sites of `line`, in order, as + `%{action: a, ref: r, action_at: byte, ref_at: byte}`. + + `flow_prefix` is, for a line inside a flow collection opened on an earlier + line, the structure of the lines since it opened (`flow_prefixes/1`): a + `uses:` there is a site when the brackets still open before it make it a + step's or a job's own key, however the line itself begins (`name: + "hello` then `world", uses: a/b@x }]`). Without it the line is judged on + its own. + """ + @spec sites(String.t(), String.t()) :: [map()] + def sites(line, flow_prefix \\ "") when is_binary(line) and is_binary(flow_prefix) do + masked = mask(line) + escaped = escaped_values(masked, flow_prefix) + + masked + |> scan(flow_prefix) + |> Enum.reject(fn [_full, {a_at, _} | _] -> + Enum.any?(escaped, &(a_at >= &1.from and a_at <= &1.to)) + end) + |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> + %{ + action: binary_part(masked, a_at, a_len), + ref: binary_part(masked, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + end) + |> Kernel.++(for %{site: %{} = site} <- escaped, do: site) + |> Enum.sort_by(& &1.action_at) + end + + # The double-quoted `uses` values of a masked line that hold an escape + # (`"github/codeql-action/init@\x31c5b…"`), each as `%{from, to, site}`: + # the body's byte span and the site its value decodes to (`decoded_site/2`), + # or nil — a value that is no structural site, that the line does not + # close (a double-quoted scalar continued over lines is not read here), or + # that does not decode to a reference. Such a value is read only through + # its decoding: GitHub runs the decoded reference, never the text. + defp escaped_values(masked, flow_prefix) do + ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*"/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [{at, len}] -> + from = at + len + + {body, close, _tail} = + take_quoted(binary_part(masked, from, byte_size(masked) - from), ?", "") + + prefix = binary_part(masked, 0, from) + + structural? = + if flow_prefix == "", + do: structural?(prefix), + else: structural_in_flow?(flow_prefix <> prefix) + + cond do + not String.contains?(body, "\\") -> + [] + + structural? and close != "" -> + [%{from: from, to: from + byte_size(body), site: decoded_site(body, from)}] + + true -> + [%{from: from, to: from + byte_size(body), site: nil}] + end + end) + end + + # The site a double-quoted body (starting at byte `from` of its line) + # decodes to: the action and ref as YAML reads them, `action_at`/`ref_at` + # the source bytes they start at, `ref_len` the source length of the ref + # as written (a repair replaces exactly those bytes), `escaped: true`. + defp decoded_site(body, from) do + with {:ok, decoded, spans} <- decode_dq(body), + [_, {a_at, a_len}, {r_at, r_len}] <- + Regex.run(~r/\A([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/, decoded, + return: :index + ) do + spans = List.to_tuple(spans) + {action_from, _} = elem(spans, a_at) + {ref_from, _} = elem(spans, r_at) + {_, ref_to} = elem(spans, r_at + r_len - 1) + + %{ + action: binary_part(decoded, a_at, a_len), + ref: binary_part(decoded, r_at, r_len), + action_at: from + action_from, + ref_at: from + ref_from, + ref_len: ref_to - ref_from, + escaped: true + } + else + _ -> nil + end + end + + @doc """ + The body of a YAML double-quoted scalar (the text between its quotes, on + one line) as YAML decodes it: `{:ok, decoded, spans}`, `spans` the source + byte span `{from, to}` of every decoded byte, or `:error` for an escape + YAML does not define. The escapes are YAML 1.2's: `\\0 \\a \\b \\t \\n \\v + \\f \\r \\e \\ \\" \\/ \\\\ \\N \\_ \\L \\P`, `\\xXX`, `\\uXXXX`, `\\UXXXXXXXX` — the + JSON-style subset is what `yq -o kyaml` writes. + """ + @spec decode_dq(String.t()) :: + {:ok, String.t(), [{non_neg_integer(), non_neg_integer()}]} | :error + def decode_dq(body) when is_binary(body), do: decode_dq(body, 0, [], []) + + defp decode_dq(<<>>, _at, out, spans), + do: {:ok, out |> Enum.reverse() |> IO.iodata_to_binary(), Enum.reverse(spans)} + + defp decode_dq(<>, at, out, spans) do + case escape(rest) do + {:ok, bytes, used} -> + span = {at, at + 1 + used} + spans = Enum.reduce(1..byte_size(bytes)//1, spans, fn _, acc -> [span | acc] end) + + decode_dq( + binary_part(rest, used, byte_size(rest) - used), + at + 1 + used, + [bytes | out], + spans + ) + + :error -> + :error + end + end + + defp decode_dq(<>, at, out, spans), + do: decode_dq(rest, at + 1, [<> | out], [{at, at + 1} | spans]) + + @simple_escapes %{ + ?0 => <<0>>, + ?a => <<7>>, + ?b => <<8>>, + ?t => <<9>>, + ?\t => <<9>>, + ?n => <<10>>, + ?v => <<11>>, + ?f => <<12>>, + ?r => <<13>>, + ?e => <<27>>, + ?\s => " ", + ?" => "\"", + ?/ => "/", + ?\\ => "\\", + ?N => <<0x85::utf8>>, + ?_ => <<0xA0::utf8>>, + ?L => <<0x2028::utf8>>, + ?P => <<0x2029::utf8>> + } + + # The escape after a `\` in `rest`: `{:ok, bytes, bytes of rest used}`. + defp escape(<>) when is_map_key(@simple_escapes, c), + do: {:ok, Map.fetch!(@simple_escapes, c), 1} + + defp escape(<>) when c in [?x, ?u, ?U] do + n = %{?x => 2, ?u => 4, ?U => 8}[c] + + with <> <- rest, + true <- Regex.match?(~r/\A[0-9A-Fa-f]+\z/, hex), + cp = String.to_integer(hex, 16), + true <- cp <= 0x10FFFF and cp not in 0xD800..0xDFFF do + {:ok, <>, 1 + n} + else + _ -> :error + end + end + + defp escape(_), do: :error + + @doc """ + The structural site matches of an already masked line, each as + `[full, action, ref]` `{byte, length}` pairs (`Regex.scan/3` with + `return: :index`). + """ + @spec scan(String.t(), String.t()) :: [[{non_neg_integer(), non_neg_integer()}]] + def scan(masked, flow_prefix \\ "") when is_binary(masked) do + @site_re + |> Regex.scan(masked, return: :index, capture: :all) + |> Enum.filter(fn [_full, {action_at, _} | _] -> + prefix = binary_part(masked, 0, action_at) + + if flow_prefix == "", + do: structural?(prefix), + else: structural_in_flow?(flow_prefix <> prefix) + end) + end + + @doc """ + The lines of a whole file as their structure reads: `mask_lines/1`, each + then masked (`mask/1`, so cut at its comment), with a quoted scalar the + line leaves open blanked from its opening quote. Joined, they read as one + text of keys, brackets and commas whatever scalars span them: a `#` that + continues a quoted scalar (`name: "hello` then `# world",`) cuts nothing, + and no quote is left open. + """ + @spec structure_lines(String.t()) :: [String.t()] + def structure_lines(content) when is_binary(content), + do: content |> mask_lines() |> Enum.map(&do_mask(&1, "", true)) + + @doc """ + For each line of `content`, the structure of the lines since the flow + collection it sits in was opened (`structure_lines/1`, joined, ending in a + newline), or `""` for a line at flow depth 0 (`line_depths/1`): what + `sites/2` reads a line inside a multi-line flow collection against. + """ + @spec flow_prefixes(String.t()) :: [String.t()] + def flow_prefixes(content) when is_binary(content) do + content + |> structure_lines() + |> Enum.zip(line_depths(content)) + |> Enum.map_reduce("", fn {text, depth}, carry -> + prefix = if depth > 0, do: carry <> "\n", else: "" + {prefix, if(depth > 0, do: carry <> "\n" <> text, else: text)} + end) + |> elem(0) + end + + @doc """ + The line cut at its comment, with every quoted scalar that is neither a + key nor the value of a `uses` key blanked to `x` byte for byte. A `uses` + value and a quoted key are kept as written, so text read from the masked + line at a site is the pin's own text, and an offset into it is an offset + into the line. + """ + @spec mask(String.t()) :: String.t() + def mask(line) when is_binary(line), do: do_mask(line, "") + + @doc """ + Is the line nothing but pin sites? Every structural site removed (with its + value's closing quote), what is left of the masked line may only be + indentation, list dashes, flow brackets and commas — so `- uses: a/b@x`, + `{ uses: "a/b@x" },` and `[ { uses: a/b@x }, { uses: c/d@y } ]` are pin + lines, and `steps: [ { uses: a/b@x } ]`, `- { name: x, uses: a/b@x }` + and `uses: a/b@x extra` are not. The comment is never part of it. Nor + is a line whose `uses` value is written with escapes (`"a/b@\\x31…"`): its + reference is read decoded (`sites/2`), but it is no plain re-pin. + """ + @spec pin_line?(String.t()) :: boolean() + def pin_line?(line) when is_binary(line) do + masked = mask(line) + sites = if escaped_values(masked, "") == [], do: scan(masked), else: [] + + remaining = + sites + |> Enum.reverse() + |> Enum.reduce(masked, fn [{at, len}, {action_at, _}, _], rest -> + # The site match includes an opening value quote but ends at the ref. + # Remove its matching closing quote along with the pin. + quote = binary_part(masked, action_at - 1, 1) + closing = binary_part(rest, at + len, min(1, byte_size(rest) - at - len)) + len = if quote in ["\"", "'"] and closing == quote, do: len + 1, else: len + binary_part(rest, 0, at) <> binary_part(rest, at + len, byte_size(rest) - at - len) + end) + + sites != [] and Regex.match?(~r/^[\s\-\[\]{},]*$/, remaining) + end + + @doc """ + The text after the line's comment `#`, as written (untrimmed), or `""`. + The `#` is found outside quoted scalars, so `name: "C# build"` has none. + """ + @spec comment(String.t()) :: String.t() + def comment(line) when is_binary(line) do + cut = byte_size(mask(line)) + + if cut < byte_size(line), + do: binary_part(line, cut + 1, byte_size(line) - cut - 1), + else: "" + end + + @doc "The line up to (not including) its comment `#`." + @spec head(String.t()) :: String.t() + def head(line) when is_binary(line), do: binary_part(line, 0, byte_size(mask(line))) + + @doc """ + The site a `uses:` key with nothing after it on its line holds on the next + line with content, where YAML continues its scalar (`- uses:` then + ` github/codeql-action/init@`; `continuations/1` steps over the + blank and comment-only lines between), as `sites/1` returns one, its + offsets into `next_line`; nil otherwise. Both lines are as `mask_lines/1` + returns them. + + The key is a block key (at the line's start or after its list dash), or an + entry of a flow mapping in a step or job position as `sites/1` reads one + (`steps: [ { uses:`, `- { name: x, uses:`). A block key's value sits deeper + than the key and holds nothing but the reference, quoted or plain, and a + comment; a flow entry's value is the reference up to the delimiter that + ends it (`,`, `}` or `]`), wherever the line is indented — inside a flow + collection the scalar continues on any line. A block key may also hold a + block scalar (`- uses: >-`): its one content line, a whole reference + (`line_states/1`, `:value`), is the value; in masked lines any other + content is blanked, so a scalar that folds more lines names no site. + """ + @spec continued_site(String.t(), String.t()) :: map() | nil + def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do + key = head(key_line) + + # Each form's value: the reference it may be, and what may follow it. + form = + cond do + m = Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) -> + if indent_of(next_line) > byte_size(Enum.at(m, 1)), + do: + {~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, + ~r/\A\s*(?:#.*)?\z/} + + m = + Regex.run( + ~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s+[|>][-+0-9]*\s*$/, + key + ) -> + if indent_of(next_line) > byte_size(Enum.at(m, 1)), do: {@block_ref_re, nil} + + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) and + structural?(String.trim_trailing(key) <> " ") -> + {~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/, + ~r/\A\s*(?:[,}\]]|#|\z)/} + + true -> + nil + end + + with {value_re, tail_re} <- form, + :plain <- escaped_value(next_line, tail_re), + [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next_line, return: :index) do + %{ + action: binary_part(next_line, a_at, a_len), + ref: binary_part(next_line, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + else + {:escaped, site} -> site + _ -> nil + end + end + + # A continued value written double-quoted with an escape (`"a/b@\x31…"`) + # is read decoded: `{:escaped, site}` — nil when it decodes to no + # reference or more than `tail_re` allows follows its closing quote — or + # :plain for any other value (a block scalar's, `tail_re` nil, included). + defp escaped_value(_line, nil), do: :plain + + defp escaped_value(line, tail_re) do + with [{0, from}] <- Regex.run(~r/\A\s*(?:[&!]\S*\s+)*"/, line, return: :index), + {body, close, tail} = + take_quoted(binary_part(line, from, byte_size(line) - from), ?", ""), + true <- String.contains?(body, "\\") do + {:escaped, if(close != "" and Regex.match?(tail_re, tail), do: decoded_site(body, from))} + else + _ -> :plain + end + end + + @doc """ + Every `uses:` whose value is an alias (`- uses: *checkout`, a block key or a + step/job flow entry as `sites/1` reads one), resolved to the site the + anchor it names holds: `%{line, at, len, anchor_line, site}` — the alias' + line, its byte offset and length, the anchor's line, and the site as + `sites/1` returns one, its offsets into the anchor's line. YAML resolves an + alias to the latest `&name` before it, so a later definition does not + count and an earlier one is shadowed — by any value, including one that is + no `action@ref` scalar, which names no site. Lines as `mask_lines/1` + returns them; `flow_prefixes` (`flow_prefixes/1` of the same file) lets a + line inside a multi-line flow collection be judged against the brackets + the lines above opened, as `sites/2` judges a literal site: a `uses: + *name` in an `env: {` spread over lines is data, not a step's action. + """ + @spec alias_sites([String.t()], [String.t()]) :: [map()] + def alias_sites(lines, flow_prefixes \\ []) when is_list(lines) do + prefixes = flow_prefixes ++ List.duplicate("", max(length(lines) - length(flow_prefixes), 0)) + numbered = lines |> Enum.zip(prefixes) |> Enum.with_index(1) + + defs = resolved_defs(numbered) + + for {{line, prefix}, n} <- numbered, + {name, at, len} <- uses_aliases(line, prefix), + %{site: %{} = site} = d <- [latest_anchor(defs, name, n, at)] do + %{line: n, at: at, len: len, anchor_line: Map.get(d, :site_line, d.line), site: site} + end + end + + # Every anchor the numbered lines define, each with the site its value is + # (`anchor_defs/2`, `continued_anchor/3`) and its `:line`. + defp resolved_defs(numbered) do + for {{line, prefix}, n} <- numbered, d <- anchor_defs(line, prefix) do + below = for {{l, _}, m} <- Enum.drop(numbered, n), do: {l, m} + d |> Map.put(:line, n) |> continued_anchor(line, below) + end + end + + # The site the alias `*name` at byte `at` of line `n` runs, as + # `[{line, site}]` (the line its reference stands on), or `[]`. + defp alias_value(defs, name, n, at) do + case latest_anchor(defs, name, n, at) do + %{site: %{} = site} = d -> [{Map.get(d, :site_line, d.line), site}] + _ -> [] + end + end + + # An anchor with nothing after it on its line but a tag or a comment + # (`BAD: &bad`) names the scalar YAML continues on the next line with + # content (` github/codeql-action/init@`): its site is there, + # `:site_line` that line. In block context the scalar must sit deeper than + # the key, and is all the line holds; inside a flow collection it may + # stand at any indentation, even column 0, up to the `,`, `}` or `]` that + # ends it. An anchored block scalar (`BAD: &bad >-`) names the reference + # its one content line holds (`line_states/1`, `:value`). + defp continued_anchor(%{site: nil, rest: rest} = d, line, below) do + block? = not d.flow and Regex.match?(~r/^\s*(?:![^\s]*\s+)*[|>][-+0-9]*\s*(?:#.*)?$/, rest) + + value_re = + cond do + block? -> + @block_ref_re + + d.flow -> + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/ + + true -> + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + end + + with true <- block? or Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, rest), + {next, n} <- Enum.find(below, fn {l, _} -> String.trim(head(l)) != "" end), + true <- d.flow or indent_of(next) > indent_of(line), + [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next, return: :index) do + site = %{ + action: binary_part(next, a_at, a_len), + ref: binary_part(next, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + + %{d | site: site} |> Map.put(:site_line, n) + else + _ -> d + end + end + + defp continued_anchor(d, _line, _below), do: d + + # The anchors `line` defines where a node starts (outside quotes and the + # comment), each `%{name, at, site, rest, flow}`: the site its scalar value + # is, or nil; the text after it; whether it sits inside a flow collection. + # A name runs to whitespace or a flow indicator (`&rust/tool`). A node + # starts at the line's start or after its list dashes, after `key: `, and + # — inside a flow collection only (one the line opens, or `flow_prefix` + # says it sits in) — after `[`, `{` or `,`, or directly after the colon of + # a quoted key (`{"BAD":&bad ...}`, JSON-style), each past any tag. + # `NOTE: a, &x b` in block context is one plain scalar: no anchor. + defp anchor_defs(line, flow_prefix) do + masked = mask(line) + + ~r/&([^\s,\[\]{}]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.filter(fn [{at, _}, _] -> + before = binary_part(masked, 0, at) + + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or + ((flow_prefix != "" or String.contains?(before, ["[", "{"])) and + Regex.match?(~r/(?:[\[{,]|["']:)\s*(?:![^\s]*\s+)*$/, before)) + end) + |> Enum.map(fn [{at, len}, {n_at, n_len}] -> + rest_at = at + len + rest = binary_part(line, rest_at, byte_size(line) - rest_at) + + site = + case Regex.run( + ~r/^\s+(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?(?=[\s,}\]#]|$)/, + rest, + return: :index + ) do + [_, {a_at, a_len}, {r_at, r_len}] -> + %{ + action: binary_part(rest, a_at, a_len), + ref: binary_part(rest, r_at, r_len), + action_at: rest_at + a_at, + ref_at: rest_at + r_at + } + + _ -> + nil + end + + before = binary_part(masked, 0, at) + open = length(Regex.scan(~r/[\[{]/, before)) - length(Regex.scan(~r/[\]}]/, before)) + + %{ + name: binary_part(masked, n_at, n_len), + at: at, + site: site, + rest: rest, + flow: flow_prefix != "" or open > 0 + } + end) + end + + # The `uses:` keys of `line` whose value is an alias, at a site position, + # as `{name, offset of the "*", length of "*name"}`. + defp uses_aliases(line, flow_prefix) do + masked = mask(line) + + ~r/["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([^\s,\[\]{}]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [{at, len}, {n_at, n_len}] -> + star = n_at - 1 + prefix = binary_part(masked, 0, at) + head = binary_part(masked, 0, star) + + structural? = + if flow_prefix == "", + do: structural?(head), + else: structural_in_flow?(flow_prefix <> head) + + if structural? and + not Regex.match?(~r/[A-Za-z0-9_.\/-]$/, prefix), + do: [{binary_part(masked, n_at, n_len), star, at + len - star}], + else: [] + end) + end + + defp latest_anchor(defs, name, line, at) do + defs + |> Enum.filter(&(&1.name == name and (&1.line < line or (&1.line == line and &1.at < at)))) + |> Enum.max_by(&{&1.line, &1.at}, fn -> nil end) + end + + @doc """ + Every step or job given whole as an alias — `- *step` in a job's + `steps:`, `*step` as an entry of a `steps: [ ... ]`, `rust: *job` under + `jobs:` — resolved to the `uses:` of the mapping its anchor holds: + `%{line, key_line, anchor_line, site}`, the alias' line, the line of that + `uses:` key, the line its reference stands on, and its site as `sites/1` + returns one (offsets into that line). The reference is the key's value as + YAML resolves it: written on the key's line, continued below it + (`continued_site/2`, a block scalar included), or an alias (`uses: *bad`, + on the line of `&bad`'s value). + YAML resolves the alias to that very node, so its `uses:` is the step's + or the job's action even where the anchor sits in data (`include: - + &step { uses: ... }` in a matrix). The anchored mapping is read as a flow + mapping on the anchor's own line (`&step { uses: ... }`) or in block form + (`- &step`, `key: &step`, its keys on the deeper lines below); any other + shape names no site here. A `- *step` whose list owner the lines above + do not show is judged a step, as a lone `- uses:` line is. + """ + @spec node_alias_sites(String.t()) :: [map()] + def node_alias_sites(content) when is_binary(content) do + lines = mask_lines(content) + numbered = lines |> Enum.zip(flow_prefixes(content)) |> Enum.with_index(1) + texts = List.to_tuple(lines) + + rows = block_rows(content) + + defs = resolved_defs(numbered) + + for {{line, prefix}, n} <- numbered, + {name, at} <- node_aliases(line, prefix, rows, n - 1), + %{} = d <- [latest_anchor(defs, name, n, at)], + {key_line, site_line, site} <- anchored_uses(d, texts, rows, defs) do + %{line: n, key_line: key_line, anchor_line: site_line, site: site} + end + end + + # The aliases on `line` that stand for a whole step or job, as `{name, + # offset of the "*"}`: a block list item that is nothing but the alias, in + # a sequence `steps:` holds (or one whose owner is not shown); a block key + # of `jobs:` whose value is the alias; or a flow entry whose brackets make + # it an item of a job's step sequence, or a job's value in `jobs: { ... }`. + defp node_aliases(line, prefix, rows, i) do + masked = mask(line) + row = elem(rows, i) + + item = Regex.run(~r/^(\s*-\s+)\*([^\s,\[\]{}]+)\s*$/, masked) + value = Regex.run(~r/^(\s*["']?[^"':\s]+["']?\s*:\s+)\*([^\s,\[\]{}]+)\s*$/, masked) + + block = + case {row, item, value} do + {%{dash: d}, [_, lead, name], _} when d != nil -> + if sequence_owner(rows, seq_owner(rows, i, d)) != :data, + do: [{name, byte_size(lead)}], + else: [] + + {%{dash: nil}, _, [_, lead, name]} -> + if match?({:value, _, "jobs"}, mapping_of(rows, i)), + do: [{name, byte_size(lead)}], + else: [] + + _ -> + [] + end + + flow = + ~r/([\[{,:])\s*\*([^\s,\[\]{}]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [_, {c_at, _}, {n_at, n_len}] -> + star = n_at - 1 + stack = flow_stack(prefix <> binary_part(masked, 0, star)) + + node? = + if binary_part(masked, c_at, 1) == ":", + do: match?([{"{", "jobs"} | _], stack), + else: alias_step_stack?(stack) + + if node?, do: [{binary_part(masked, n_at, n_len), star}], else: [] + end) + + block ++ flow + end + + # An entry directly in a `[` that `steps:` opens: the whole stack the text + # shows (a block job's `steps: [*step]`), or one that goes on through a + # job key into `jobs`. + defp alias_step_stack?([{"[", "steps"}]), do: true + + defp alias_step_stack?([{"[", "steps"}, {"{", job}, {"{", "jobs"} | _]) when is_binary(job), + do: true + + defp alias_step_stack?(_), do: false + + # The `uses:` of the mapping the anchor `d` holds, as `[{key_line, + # site_line, site}]`: a flow mapping opening on the anchor's line, read + # through the lines below to the brace that closes it, or a block mapping + # on the lines below, read to its `uses:` key. + defp anchored_uses(d, texts, rows, defs) do + line = elem(texts, d.line - 1) + rest_at = byte_size(line) - byte_size(d.rest) + + cond do + m = Regex.run(~r/^\s*(?:![^\s]*\s+)*\{/, d.rest, return: :index) -> + [{0, len}] = m + flow_mapping_uses(texts, d.line, rest_at + len - 1, defs) + + Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, d.rest) -> + block_uses(d.line, texts, rows, defs) + + true -> + [] + end + end + + # The top-level `uses` entry of the flow mapping whose `{` is at byte + # `open` of line `n`, as `[{key_line, site_line, site}]` (an alias value + # resolved to its anchor's), or `[]` when the mapping closes without one. + # The mapping is read through the lines below to the brace that closes + # it: an entry starts after its `{` or one of its commas — on the same + # line, or as the last thing a line above held — with no other bracket + # open between. Lines are read masked (`mask/1`), so brackets and + # commas inside quoted text and comments count for nothing. + defp flow_mapping_uses(texts, n, open, defs) do + n..tuple_size(texts)//1 + |> Enum.reduce_while({:open, 0, nil}, fn k, {:open, depth, last} -> + from = if k == n, do: open, else: 0 + + case mapping_line_uses(mask(elem(texts, k - 1)), from, depth, last) do + {:found, {:site, site}} -> + {:halt, [{k, k, site}]} + + {:found, {:alias, name, at}} -> + {:halt, for({line, site} <- alias_value(defs, name, k, at), do: {k, line, site})} + + :closed -> + {:halt, []} + + open_state -> + {:cont, open_state} + end + end) + |> case do + found when is_list(found) -> found + _ -> [] + end + end + + defp mapping_line_uses(m, from, depth, last) do + m + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.drop(from) + |> Enum.reduce_while({:open, depth, last}, fn {c, i}, {:open, d, l} -> + cond do + c in [?{, ?[] -> + {:cont, {:open, d + 1, c}} + + c in [?}, ?]] and d <= 1 -> + {:halt, :closed} + + c in [?}, ?]] -> + {:cont, {:open, d - 1, c}} + + c in [?\s, ?\t, ?,] -> + {:cont, {:open, d, if(c == ?,, do: c, else: l)}} + + d == 1 and l in [?{, ?,] -> + case entry_uses(m, i) do + nil -> {:cont, {:open, d, c}} + site -> {:halt, site} + end + + true -> + {:cont, {:open, d, c}} + end + end) + |> case do + {:open, _, _} = state -> state + :closed -> :closed + found -> {:found, found} + end + end + + # The `uses` entry starting at byte `i` of masked line `m`: `{:site, + # site}` for a reference written there, `{:alias, name, offset of the + # "*"}` for an alias, or nil. + defp entry_uses(m, i) do + rest = binary_part(m, i, byte_size(m) - i) + + case Regex.run( + ~r/\A["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*(?:["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)|\*([^\s,\[\]{}]+))/, + rest, + return: :index + ) do + [_, {a_at, a_len}, {r_at, r_len}] when a_at >= 0 -> + {:site, + %{ + action: binary_part(rest, a_at, a_len), + ref: binary_part(rest, r_at, r_len), + action_at: i + a_at, + ref_at: i + r_at + }} + + [_, _, _, {n_at, n_len}] -> + {:alias, binary_part(rest, n_at, n_len), i + n_at - 1} + + _ -> + nil + end + end + + # A block mapping anchored with nothing after the anchor (`- &step`, + # `step: &step`) on row `n`: its keys are the rows below deeper than the + # dash or the key, the first of them setting their column; the one named + # `uses` holds the site(s) (`key_value/3`). + defp block_uses(n, texts, rows, defs) do + case elem(rows, n - 1) do + %{} = row -> + parent = if row.key == nil, do: row.dash || row.indent, else: row.key_col + + children = + n..(tuple_size(rows) - 1)//1 + |> Enum.map(&{&1, elem(rows, &1)}) + |> Enum.reject(fn {_, r} -> r == :skip end) + |> Enum.take_while(fn {_, r} -> r.indent > parent end) + + case children do + [] -> + [] + + [{_, first} | _] -> + for {j, %{indent: ind, dash: nil, key: "uses"}} <- children, + ind == first.indent, + {line, site} <- key_value(texts, j + 1, defs), + do: {j + 1, line, site} + end + + _ -> + [] + end + end + + # The site(s) the block `uses:` key on line `k` holds, as `[{line, site}]`: + # written on its line, an alias (`uses: *bad`) resolved to the anchor's + # value, or a reference continued below it (`continued_site/2`: the next + # line with content, a block scalar's included). + defp key_value(texts, k, defs) do + text = elem(texts, k - 1) + masked = mask(text) + + alias_re = ~r/^\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([^\s,\[\]{}]+)\s*$/ + + case {sites(text), Regex.run(alias_re, masked, return: :index)} do + {[_ | _] = found, _} -> + for site <- found, do: {k, site} + + {[], [_, {n_at, n_len}]} -> + alias_value(defs, binary_part(masked, n_at, n_len), k, n_at - 1) + + {[], nil} -> + with v when is_integer(v) <- + Enum.find( + (k + 1)..tuple_size(texts)//1, + &(String.trim(head(elem(texts, &1 - 1))) != "") + ), + %{} = site <- continued_site(text, elem(texts, v - 1)) do + [{v, site}] + else + _ -> [] + end + end + end + + @doc """ + The one-based numbers of the lines of `content` whose leading block `uses:` + key the indentation above it proves is neither a step's nor a job's own: + an input of that name (`with:` then `uses: a/b@x`), an entry of a sequence + some key other than `steps:` holds, a key of a `steps:` that is no + job's, or one under a root key that holds no job (`env:` then `uses: + a/b@x` in a workflow). Such a key holds data, not a pin. A line whose + ancestry the lines above do not show far enough — a diff hunk cut short, + a fragment with no `jobs:` — is not listed: it is judged on its own line, + as before. + """ + @spec data_uses_lines(String.t()) :: MapSet.t(pos_integer()) + def data_uses_lines(content) when is_binary(content) do + rows = block_rows(content) + + for i <- 0..(tuple_size(rows) - 1)//1, + match?(%{key: "uses"}, elem(rows, i)), + uses_owner(rows, i) == :data, + into: MapSet.new(), + do: i + 1 + end + + # The lines of `content` as block structure (`block_row/2`), a tuple. A key + # at column 0 is a root key; a root key that is neither `jobs:` nor + # `runs:` (`env:`, `on:`, `inputs:`) holds no step or job, so it is marked + # `data_root?` — when the text shows the root (a `jobs:` or `runs:` at + # column 0), or the key is one a workflow or an action defines for data, + # so a hunk that shows `env:` at column 0 reads it the same. A fragment + # with neither (`build:` then `uses: ...`) is judged as before. + defp block_rows(content) do + rows = content |> line_states() |> Enum.zip_with(line_depths(content), &block_row/2) + + shown? = + Enum.any?(rows, &match?(%{indent: 0, dash: nil, key: k} when k in ["jobs", "runs"], &1)) + + rows + |> Enum.map(fn + %{indent: 0, dash: nil, key: k} = row when is_binary(k) and k not in ["jobs", "runs"] -> + Map.put(row, :data_root?, shown? or k in @data_root_keys) + + row -> + row + end) + |> List.to_tuple() + end + + # One line as block structure: its indentation, the column of its last + # list dash (nil without one), the column and name of the key after the + # dashes, whether that key's value is on the lines below (`key:` alone, + # past any node properties), and whether the line is a bare dash. Blank, + # comment-only and scalar lines, and lines inside a flow collection, are + # :skip: nothing there is a block parent. + defp block_row({text, :yaml}, 0) do + h = head(text) + + if String.trim(h) == "" do + :skip + else + [_, ind, dashes, rest] = Regex.run(~r/^( *)((?:-(?:\s+|$))*)(.*)$/, h) + + {key, open?} = + case Regex.run(~r/^["']?([^"':\s][^"':]*?)["']?\s*:(?:\s+(.*))?$/, rest) do + [_, k | value] -> + {k, Regex.match?(~r/^(?:[&!]\S*\s*)*$/, String.trim(Enum.join(value)))} + + nil -> + {nil, false} + end + + dash = + case :binary.matches(dashes, "-") do + [] -> nil + found -> byte_size(ind) + elem(List.last(found), 0) + end + + %{ + indent: byte_size(ind), + dash: dash, + key_col: byte_size(ind) + byte_size(dashes), + key: key, + open?: open?, + bare?: String.trim(rest) == "" + } + end + end + + defp block_row(_, _), do: :skip + + # :site, :data or :unknown for the `uses` key on row i: a step's when its + # mapping is an item of a job's `steps:`, a job's when its mapping is the + # value of a key of `jobs:`. + defp uses_owner(rows, i) do + case mapping_of(rows, i) do + {:item, j, d} -> sequence_owner(rows, seq_owner(rows, j, d)) + {:value, j, _key} -> job_key(rows, j) + :root -> root_owner(rows, i) + _ -> :unknown + end + end + + # A key of the root mapping (`block_rows/1`): :data when it is a data root, + # else :unknown — the text shows no more of where it sits. + defp root_owner(rows, j), + do: if(match?(%{data_root?: true}, elem(rows, j)), do: :data, else: :unknown) + + defp sequence_owner(rows, {:value, s, "steps"}), do: steps_owner(rows, s) + defp sequence_owner(_rows, {:value, _, _}), do: :data + defp sequence_owner(_rows, _), do: :unknown + + # `steps:` is a job's, or a composite action's under `runs:`; inside a + # list item or another key's mapping it is data (`with:` then `steps:`). + defp steps_owner(rows, s) do + case mapping_of(rows, s) do + {:value, _p, "runs"} -> :site + {:value, p, _job} -> job_key(rows, p) + {:item, _, _} -> :data + :root -> root_owner(rows, s) + _ -> :unknown + end + end + + # Is the key on row j a job: its mapping the value of `jobs:`? A key + # inside a list item (a step's `with:`) or under any other key is not. + defp job_key(rows, j) do + case mapping_of(rows, j) do + {:value, _, "jobs"} -> :site + {:value, _, _} -> :data + {:item, _, _} -> :data + :root -> root_owner(rows, j) + _ -> :unknown + end + end + + # The mapping row j's key sits in: the list item its own dash opens, or + # the one the rows above give it. + defp mapping_of(rows, j) do + case elem(rows, j) do + %{dash: d} when d != nil -> {:item, j, d} + %{key_col: col} -> container(rows, j, col) + end + end + + # The mapping a key at column `col` of row i belongs to, from the nearest + # row above it that sits shallower: a list item whose first key is at + # `col` (`- name: x` above ` uses:`) or a bare dash, `{:item, row, dash}`; + # a key whose value is below it, `{:value, row, key}`. Rows as deep or + # deeper are its siblings and their contents. :root when no row is + # shallower; :unknown when the shallower row is neither. + defp container(rows, i, col) do + Enum.reduce_while((i - 1)..0//-1, :root, fn j, acc -> + case elem(rows, j) do + :skip -> {:cont, acc} + %{indent: ind} when ind >= col -> {:cont, acc} + %{dash: d, key_col: ^col} when d != nil -> {:halt, {:item, j, d}} + %{dash: d, bare?: true} when d != nil -> {:halt, {:item, j, d}} + %{key: k, open?: true, key_col: kc} when k != nil and kc < col -> {:halt, {:value, j, k}} + _ -> {:halt, :unknown} + end + end) + end + + # The key that holds the sequence whose item's dash is at column `d` of + # row j: the nearest row above that is not a sibling item or deeper — + # `steps:` above it, or at its own column (`steps:` then `- uses:`). + # :unknown when no row above shows it. + defp seq_owner(rows, j, d) do + Enum.reduce_while((j - 1)..0//-1, :unknown, fn k, acc -> + case elem(rows, k) do + :skip -> + {:cont, acc} + + %{indent: ind} when ind > d -> + {:cont, acc} + + %{indent: ^d, dash: dd} when dd != nil -> + {:cont, acc} + + %{key: key, open?: true, key_col: kc} when key != nil and kc <= d -> + {:halt, {:value, k, key}} + + _ -> + {:halt, :unknown} + end + end) + end + + @doc """ + The flow depth at the start of each line of `content` — how many `[` and + `{` the lines above left open, as the walk of `line_states/1` counts them + (outside quoted scalars, comments and block-scalar content). A line at + depth 0 is block structure; one deeper sits inside a flow collection, even + when nothing on the line itself shows it (`steps: [`, then `{` alone, then + `uses: ...`). + """ + @spec line_depths(String.t()) :: [non_neg_integer()] + def line_depths(content) when is_binary(content) do + lines = String.split(content, "\n") + + lines + |> Enum.with_index() + |> Enum.map_reduce(initial_state(lines), fn + {line, n}, st -> + {_, next} = mask_file_line(line, n, st) + {st.depth, next} + end) + |> elem(0) + end + + @doc """ + Every `uses:` key in `lines` (as `mask_lines/1` returns them) whose + reference YAML continues below it, as `{key_line, value_line, site}`: + one-based line numbers and the `continued_site/2` site on the value line. + The value is the next line that holds anything but a comment — YAML lets + blank and comment-only lines stand between a key and its scalar + (`- uses:`, ` # rationale`, ` github/codeql-action/init@`). + """ + @spec continuations([String.t()]) :: [{pos_integer(), pos_integer(), map()}] + def continuations(lines) when is_list(lines) do + content = + lines + |> Enum.with_index(1) + |> Enum.reject(fn {line, _} -> String.trim(head(line)) == "" end) + + content + |> Enum.zip(Enum.drop(content, 1)) + |> Enum.flat_map(fn {{key, k}, {next, v}} -> + case continued_site(key, next) do + nil -> [] + site -> [{k, v, site}] + end + end) + end + + @doc """ + The lines of a whole file, each with the bytes that belong to a scalar + opened on an earlier line blanked to `x`, byte for byte: every content + line of a block scalar (`run: |`) — but the reference one a `uses:` key or + an anchor holds is whole (`:value`, `line_states/1`) — and a multi-line + quoted scalar's continuation up to and including its closing quote. A line read through + `sites/1` one at a time cannot see either, so a `uses:` written in a + `run: |` script reads as a key; read from these lines it does not. + + A quote opens a scalar only where YAML lets one start: after `key: `, a + list dash, or, inside a flow collection, `[`, `{` or `,`; at the start of + a line only when the line above left a value pending (`key:`). `echo + it's` stays a plain scalar. A quoted scalar still open at the end of the + file was misread, and the lines after its opening quote are returned as + written, each judged on its own as before: a misread quote never hides a + pin. + """ + @spec mask_lines(String.t()) :: [String.t()] + def mask_lines(content) when is_binary(content), + do: content |> line_states() |> Enum.map(&elem(&1, 0)) + + @doc """ + The lines of `mask_lines/1`, each with what the reader made of it: + `:yaml` (structure, as written), `:text` (inside a scalar an earlier line + opened: a block scalar's content, or a quoted scalar's continuation — at + least up to its closing quote), `:value` (the one content line of a block + scalar a `uses:` key or an anchor holds, when that line is a whole + reference — `- uses: >-` then ` a/b@`: the scalar's value, kept as + written so the key's site can be read there) or `:open` (after the + opening quote of a quoted scalar the text never closes, returned as + written: what it is cannot be said from these lines alone). A diff hunk, + which shows a few lines of a file, is read this way too: a hunk that cuts + a quoted scalar off leaves its lines `:open`. + """ + @spec line_states(String.t()) :: [{String.t(), :yaml | :text | :value | :open}] + def line_states(content) when is_binary(content) do + lines = String.split(content, "\n") + + {masked, st} = + lines + |> Enum.with_index() + |> Enum.map_reduce(initial_state(lines), fn + {line, n}, st -> mask_file_line(line, n, st) + end) + + if st.q do + Enum.take(masked, st.opened + 1) ++ + Enum.map(Enum.drop(lines, st.opened + 1), &{&1, :open}) + else + masked + end + end + + defp initial_state(lines), + do: %{ + q: nil, + opened: nil, + block: nil, + depth: 0, + pending: false, + keep: nil, + lines: List.to_tuple(lines) + } + + # Inside a block scalar: a blank line or one deeper than the key is its + # content; anything else ends it. The one line that is a `uses:` key's or + # an anchor's whole reference (`keep`, `block_value/4`) is kept as written. + defp mask_file_line(line, n, %{block: col} = st) when is_integer(col) do + cond do + n == st.keep -> + {{line, :value}, st} + + String.trim(line) == "" or indent_of(line) > col -> + {{String.duplicate("x", byte_size(line)), :text}, st} + + true -> + mask_file_line(line, n, %{st | block: nil, keep: nil}) + end + end + + defp mask_file_line(line, n, st) do + {cont, st} = + case st.q && close_at(line, st.q, 0) do + nil -> {0, st} + :open -> {byte_size(line), st} + at -> {at + 1, %{st | q: nil, opened: nil}} + end + + rest = binary_part(line, cont, byte_size(line) - cont) + view = String.duplicate("x", cont) <> rest + st = if st.q, do: st, else: walk(rest, String.duplicate("x", cont), n, st) + {{view, if(cont > 0, do: :text, else: :yaml)}, st} + end + + # The byte offset of the quote that closes a `q` scalar in `line` from + # `at`, or :open. `\` escapes inside double quotes; `''` is a literal `'`. + defp close_at(line, _q, at) when at >= byte_size(line), do: :open + + defp close_at(line, ?", at) do + case :binary.at(line, at) do + ?\\ -> close_at(line, ?", at + 2) + ?" -> at + _ -> close_at(line, ?", at + 1) + end + end + + defp close_at(line, ?', at) do + cond do + :binary.at(line, at) != ?' -> close_at(line, ?', at + 1) + at + 1 < byte_size(line) and :binary.at(line, at + 1) == ?' -> close_at(line, ?', at + 2) + true -> at + end + end + + # One line outside any scalar, `seen` being what precedes `rest` on it + # (quote bodies as `x`). Tracks flow depth, opens a quote that does not + # close on the line, and at its end a block scalar or a pending value. + defp walk(<<>>, seen, n, st) do + trimmed = String.trim_trailing(seen) + + cond do + String.trim(seen) == "" -> + st + + st.depth == 0 and + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*[|>][-+0-9]*$/, trimmed) -> + col = block_parent(seen) + %{st | block: col, pending: false, keep: block_value(st.lines, n, col, trimmed)} + + true -> + # A key or dash followed only by node properties (`- run: &cmd`, + # `key: !!str`) leaves its value pending as surely as a bare one. + pending = + Regex.match?(~r/(?::|^\s*-)(?:\s+[&!]\S*)*$/, trimmed) or + (st.depth > 0 and Regex.match?(~r/[\[{,]$/, trimmed)) + + %{st | pending: pending} + end + end + + defp walk(<>, seen, n, st) + when seen == "" or binary_part(seen, byte_size(seen) - 1, 1) in [" ", "\t"], + do: walk(<<>>, seen, n, st) + + defp walk(<>, seen, n, st) when q in [?", ?'] do + if scalar_start?(seen, st) do + case close_at(tail, q, 0) do + :open -> + %{st | q: q, opened: n} + + at -> + rest = binary_part(tail, at + 1, byte_size(tail) - at - 1) + walk(rest, seen <> String.duplicate("x", at + 2), n, st) + end + else + walk(tail, seen <> <>, n, st) + end + end + + defp walk(<>, seen, n, st) when c in [?[, ?{] do + st = if st.depth > 0 or value_start?(seen), do: %{st | depth: st.depth + 1}, else: st + walk(tail, seen <> <>, n, st) + end + + defp walk(<>, seen, n, %{depth: d} = st) when c in [?], ?}] and d > 0, + do: walk(tail, seen <> <>, n, %{st | depth: d - 1}) + + defp walk(<>, seen, n, st), do: walk(tail, seen <> <>, n, st) + + # Where a value starts: after `key: ` or a list dash, past any anchor or + # tag (`key: &a [`, `- !!str "`). + defp value_start?(seen), do: Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*$/, seen) + + # Inside a flow collection a scalar also starts after `[`, `{` or `,`, and + # directly after a `:`, JSON-style (`{"run":"echo ...`). Every start may + # carry an anchor or tag before the quote (`"run": &cmd "echo`). + defp scalar_start?(seen, st) do + cond do + Regex.match?(~r/^\s*(?:[&!]\S*\s+)*$/, seen) -> st.pending or st.depth > 0 + Regex.match?(~r/(?:^\s*(?:-\s+)+|:\s+)(?:[&!]\S*\s+)*$/, seen) -> true + st.depth > 0 -> Regex.match?(~r/[\[{,:]\s*(?:[&!]\S*\s+)*$/, seen) + true -> false + end + end + + # A block scalar's content is deeper than its key (`run: |`, `- run: |`) + # or, for a bare sequence entry (`- |`), deeper than the dash. + defp block_parent(seen) do + if Regex.match?(~r/^\s*-\s+(?:[&!]\S*\s+)*[|>]/, seen), + do: indent_of(seen), + else: byte_size(hd(Regex.run(~r/^\s*(?:-\s+)*/, seen))) + end + + # A block scalar a `uses:` key holds (`- uses: >-`), or one an anchor names + # (`BAD: &bad |`), whose content is a single line holding nothing but a + # reference (` github/codeql-action/init@`): YAML resolves the + # scalar to that reference, so that line is the value. Its index in + # `lines`, or nil — for any other block scalar, and for one whose content + # runs over more lines (`>-` folds them into one value, no reference). + # Inside a block scalar `#` is text, so a reference with a comment after + # it is no reference either. + defp block_value(lines, n, col, header) do + anchored? = + case Regex.run(~r/(?:^\s*(?:-\s+)*|:\s+)((?:[&!]\S*\s+)*)[|>][-+0-9]*$/, header) do + [_, props] -> String.contains?(props, "&") + _ -> false + end + + if anchored? or + Regex.match?(~r/^\s*(?:-\s+)*["']?uses["']?\s*:\s+(?:[&!]\S*\s+)*[|>][-+0-9]*$/, header) do + (n + 1)..(tuple_size(lines) - 1)//1 + |> Stream.map(&{&1, elem(lines, &1)}) + |> Stream.reject(fn {_, l} -> String.trim(l) == "" end) + |> Stream.take_while(fn {_, l} -> indent_of(l) > col end) + |> Enum.take(2) + |> case do + [{v, l}] -> if Regex.match?(@block_ref_re, l), do: v + _ -> nil + end + end + end + + defp indent_of(line), do: byte_size(line) - byte_size(String.trim_leading(line, " ")) + + # `prefix` is the masked line up to the site's action. + defp structural?(prefix) do + cond do + Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, prefix) -> + true + + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, prefix) -> + prefix |> flow_stack() |> structural_flow_stack?(prefix) + + true -> + false + end + end + + # A `uses:` read against the flow collections the lines above opened + # (`text` is their structure and the line up to the action): the key must + # stand at an entry position — after `{`, `,` or at the line's start — + # and the brackets still open must make it a step's or a job's own. + defp structural_in_flow?(text) do + Regex.match?(~r/(?:[{,]|\n)\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, text) and + text |> flow_stack() |> structural_flow_stack?(text) + end + + # The brackets still open in `text`, innermost first, each with what opens + # it: the key before it (`steps: [`), :none (nothing but indentation or a + # list dash), or :text (anything else — `echo {` is a script, not YAML). + defp flow_stack(text) do + text + |> String.graphemes() + |> Enum.reduce({[], ""}, fn + c, {stack, seg} when c in ["{", "["] -> {[{c, opener_key(seg)} | stack], ""} + c, {stack, _seg} when c in ["}", "]"] -> {Enum.drop(stack, 1), ""} + ",", {stack, _seg} -> {stack, ""} + c, {stack, seg} -> {stack, seg <> c} + end) + |> elem(0) + end + + # Node properties before a bracket name the collection, not its key: + # `steps: [ &step {` opens an anonymous step, `rust: &job {` a job. + defp opener_key(seg) do + seg = Regex.replace(~r/(?:[&!]\S*\s+)+$/, seg, "") + + cond do + Regex.match?(~r/^\s*(?:-\s*)?$/, seg) -> :none + m = Regex.run(~r/^\s*(?:-\s+)?["']?([\w.-]+)["']?\s*:\s*$/, seg) -> Enum.at(m, 1) + true -> :text + end + end + + # A `{` item of a sequence is a step when that sequence is `steps:` of a + # job: the whole stack the text shows (`steps: [ {` on a block job's + # line, which this reader cannot see past), or one that goes on through a + # job key into `jobs` (`jobs: { b: { steps: [ {`). The same `steps:` + # inside any other mapping (`fixture: { steps: [ {` in a matrix) is data. + # A composite action's steps are those of its root `runs:` (`runs: { + # steps: [ {`, KYAML or a flow line at column 0): `runs` must be the + # document's own key, its `{` the outermost bracket or one directly inside + # a root `{` the text starts with (`text`, from where the collection + # opened). + defp structural_flow_stack?(stack, text) do + Enum.all?(stack, fn {_, key} -> key != :text end) and + case stack do + [{"{", :none}] -> + true + + [{"{", :none}, {"[", key}] when key in ["steps", :none] -> + true + + [{"{", :none}, {"[", "steps"}, {"{", job}, {"{", "jobs"} | _] when is_binary(job) -> + true + + [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> + true + + [{"{", :none}, {"[", "steps"}, {"{", "runs"}] -> + Regex.match?(~r/\A["']?runs["']?\s*:/, text) + + [{"{", :none}, {"[", "steps"}, {"{", "runs"}, {"{", :none}] -> + String.starts_with?(text, "{") + + _ -> + false + end + end + + # `open?` blanks a quoted scalar the line leaves open, its quote too + # (`structure_lines/1`); `mask/1` keeps that quote. + defp do_mask(line, out, open? \\ false) + + defp do_mask("", out, _open?), do: out + + defp do_mask(<>, out, open?) when q in [?", ?'] do + {body, close, tail} = take_quoted(rest, q, "") + + kept = + if Regex.match?(~r/^\s*:/, tail) or Regex.match?(@uses_tail_re, out), + do: body, + else: String.duplicate("x", byte_size(body)) + + if open? and close == "", + do: out <> String.duplicate("x", 1 + byte_size(body)), + else: do_mask(tail, out <> <> <> kept <> close, open?) + end + + defp do_mask(<>, out, open?) do + if out == "" or String.ends_with?(out, [" ", "\t"]), + do: out, + else: do_mask(rest, out <> "#", open?) + end + + defp do_mask(<>, out, open?), + do: do_mask(rest, out <> <>, open?) + + defp do_mask(<>, out, open?), do: do_mask(rest, out <> <>, open?) + + # The body of a quoted scalar up to its closing quote; a backslash escapes + # the next character inside double quotes. An unterminated scalar runs to + # the end of the line. + defp take_quoted("", _q, acc), do: {acc, "", ""} + + defp take_quoted(<>, ?", acc), + do: take_quoted(rest, ?", acc <> <>) + + defp take_quoted(<>, q, acc) when c == q, do: {acc, <>, rest} + + defp take_quoted(<>, q, acc), + do: take_quoted(rest, q, acc <> <>) + + defp take_quoted(<>, q, acc), do: take_quoted(rest, q, acc <> <>) +end diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 0056c1fd..20cebd85 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -61,6 +61,8 @@ defmodule Hypatia.Rules.PrAutomerge do dependabot to re-raise it. """ + alias Hypatia.Rules.PinSites + @doc """ Classify a PR record against the policy. @@ -84,28 +86,59 @@ defmodule Hypatia.Rules.PrAutomerge do resolution = field(pr, :version_resolution, %{}) claims = body_claims(body) - deltas = pin_deltas(files, resolution, claims) - poison = poison_sites(files, policy) - lock_only = files != [] and Enum.all?(files, &lockfile?(filename_of(&1), policy)) - meta_only = files != [] and Enum.all?(files, &metadata_path?(filename_of(&1))) - pin_only = files != [] and Enum.all?(files, &pin_lines_only?/1) - manifest_file? = Enum.any?(files, &dependency_manifest?(filename_of(&1))) - licence_touch? = Enum.any?(files, &licence_path?(filename_of(&1))) + files + |> scan_of(pin_deltas(files, resolution, claims), poison_sites(files, policy), policy) + |> verdict(pr, policy) + end + defp scan_of(files, deltas, poison, policy) do %{ deltas: deltas, poison_sites: poison, major_delta: Enum.any?(deltas, &major_delta?/1), unresolved: Enum.count(deltas, &(&1.status == :unresolved)), conflicts: Enum.count(deltas, &(&1.status == :conflict)), - lock_only: lock_only, - meta_only: meta_only, - pin_only: pin_only, - manifest_file: manifest_file?, - licence_touch: licence_touch? + lock_only: files != [] and Enum.all?(files, &lockfile?(filename_of(&1), policy)), + meta_only: files != [] and Enum.all?(files, &metadata_path?(filename_of(&1))), + pin_only: files != [] and Enum.all?(files, &pin_lines_only?/1), + manifest_file: Enum.any?(files, &dependency_manifest?(filename_of(&1))), + licence_touch: Enum.any?(files, &licence_path?(filename_of(&1))) } - |> verdict(pr, policy) + end + + # The PR as it would stand once its poisoned pins are excised: a file whose + # every added line carries only denylisted sites drops out, and so does a + # delta onto a denylisted ref. Everything else is still there to classify. + defp remainder_scan(scan, files, policy) do + kept = Enum.reject(files, &poison_only_file?(&1, policy)) + + deltas = + Enum.reject( + scan.deltas, + &denylisted_pin?(%{base: action_base(&1.action), ref: &1.to_ref}, policy) + ) + + scan_of(kept, deltas, [], policy) + end + + defp poison_only_file?(file, policy) do + added = added_entries(file) + + pin_lines_only?(file) and added != [] and + Enum.all?(added, fn {line, _certain?, continued} -> + pins = parse_pins(line) ++ continued + pins != [] and Enum.all?(pins, &denylisted_pin?(&1, policy)) + end) + end + + defp denylisted_pin?(%{base: base, ref: ref}, policy) do + policy + |> Map.get("pin_denylist", []) + |> Enum.any?(fn entry -> + String.downcase(base) == String.downcase(Map.get(entry, "action", "")) and + denylisted_ref?(entry, ref) + end) end # ─── PA001 · denylisted pin ─────────────────────────────────────────── @@ -125,27 +158,33 @@ defmodule Hypatia.Rules.PrAutomerge do files |> Enum.flat_map(fn file -> file - |> added_lines() - |> Enum.flat_map(fn line -> - case parse_pin(line) do - %{action: action, ref: ref} -> - if action_base(action) == Map.get(entry, "action") and - denylisted_ref?(entry, ref) do - [ - %{ - file: filename_of(file), - line: String.trim(line), - denylist_id: Map.get(entry, "id"), - severity: Map.get(entry, "severity", "critical") - } - ] - else - [] - end - - _ -> - [] - end + |> added_entries() + |> Enum.flat_map(fn {line, certain?, continued} -> + # Every site on the line: a flow sequence can put several steps on + # one changed line, and a pin judged by its first site alone lets a + # denylisted second one through to `arm_auto`. A line that is the + # value of a `uses:` key on the line above it (the key unchanged + # context, the reference re-pinned) holds that site too. The action + # is compared case-folded — GitHub resolves `GitHub/CodeQL-Action` + # and `github/codeql-action` to one repository. + line + |> parse_pins() + |> Kernel.++(continued) + |> Enum.filter(fn %{base: base, ref: ref} -> + String.downcase(base) == String.downcase(Map.get(entry, "action", "")) and + denylisted_ref?(entry, ref) + end) + |> Enum.map(fn _pin -> + %{ + file: filename_of(file), + line: String.trim(line), + denylist_id: Map.get(entry, "id"), + severity: Map.get(entry, "severity", "critical"), + # false when the hunk shows the line among script text: it may + # be a fixture inside a `run: |` whose header is outside the hunk. + certain: certain? + } + end) end) end) end) @@ -166,7 +205,10 @@ defmodule Hypatia.Rules.PrAutomerge do end defp normalise_ref(ref) when is_binary(ref) do - ref |> String.trim() |> String.replace_prefix("v", "") + trimmed = ref |> String.trim() |> String.replace_prefix("v", "") + # A commit ID in upper-case hex is the same object the table carries in + # lower case; git resolves either spelling. + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, trimmed), do: String.downcase(trimmed), else: trimmed end defp normalise_ref(_), do: "" @@ -177,13 +219,37 @@ defmodule Hypatia.Rules.PrAutomerge do This is the proof obligation behind the meta-guard exemption `MGX-001`. A workflow edit that is *not* pin-only — a new step, a changed `permissions:` block, a moved `if:` — fails this, which is what keeps the reflexivity guard - from being talked around by a plausible title. + from being talked around by a plausible title. So does a diff of pin lines + that adds or swaps an action: the removed and added sites must name the + same actions, as many times each. """ @spec pin_lines_only?(map()) :: boolean() def pin_lines_only?(file) do - lines = content_lines(patch_of(file)) + lines = changed_lines(file) + + # A changed line inside a block scalar (`run: |`) is script text, never a + # pin line, so a script edit is not a pin-only change. + # A line the hunk shows among script text is not known to be a pin line. + # And every pin the file adds replaces a removed pin of the same action: + # a new step (`+ { uses: a/b@main },`) or a swapped action is a change to + # what the workflow runs, not a re-pin of what it already ran. + lines != [] and + Enum.all?(lines, fn {line, scalar?, certain?, continued} -> + certain? and not scalar? and (pin_line?(line) or continued_value_line?(line, continued)) + end) and + action_multiset(removed_pins(file)) == action_multiset(added_pins(file)) + end + + # A changed line that is nothing but the reference a `uses:` key above it + # holds (`- a/b@x` -> `+ a/b@y`): a re-pin as much as a + # one-line `uses:` is. + defp continued_value_line?(line, [_ | _]), + do: Regex.match?(~r/^\s*\S+\s*(?:#.*)?$/, String.replace(line, ~r/^[-+]/, "")) - lines != [] and Enum.all?(lines, fn line -> pin_line?(line) end) + defp continued_value_line?(_line, []), do: false + + defp action_multiset(pins) do + pins |> Enum.map(&String.downcase(&1.action)) |> Enum.sort() end # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── @@ -192,7 +258,8 @@ defmodule Hypatia.Rules.PrAutomerge do Version deltas for every action this PR re-pins. Each removed pin is paired with the first added pin for the same action - in the same file. Unpaired pins and unchanged refs are omitted. Returns a + in the same file that no earlier removal has claimed. Unpaired pins and + unchanged refs are omitted. Returns a flat list of maps with `:action`, `:file`, `:from`, `:to`, `:status`, `:source` and `:major?`. @@ -208,58 +275,86 @@ defmodule Hypatia.Rules.PrAutomerge do """ @spec pin_deltas([map()], map(), [map()]) :: [map()] def pin_deltas(files, resolution, claims) do + # Basenames two different actions of the PR share (`acme/setup`, + # `other/setup`): a claim naming only that basename belongs to neither. + shared = + files + |> Enum.flat_map(&removed_pins/1) + |> Enum.map(&String.downcase(&1.action)) + |> Enum.uniq() + |> Enum.frequencies_by(&(&1 |> String.split("/") |> List.last())) + |> Enum.filter(fn {_, n} -> n > 1 end) + |> MapSet.new(&elem(&1, 0)) + files |> Enum.flat_map(fn file -> - removed = file |> removed_lines() |> Enum.filter(&pin_line?/1) - added = file |> added_lines() |> Enum.filter(&pin_line?/1) - + # Every structural site of every changed line: a flow sequence carries + # several steps on one line (`steps: [ { uses: a@x }, { uses: b@y } ]`), + # and each of them is a delta of its own — including a major bump on a + # line that does not start with `uses:`. + removed = removed_pins(file) + added = added_pins(file) + + # Each added site pairs with ONE removed site of the same action, in + # order: a line that keeps one `actions/checkout` at v4 and moves a + # second from v4 to v7 carries the major bump, which pairing every + # removal with the first addition would hide. removed - |> Enum.flat_map(fn line -> - old = parse_pin(line) - - case Enum.find(added, &(parse_pin(&1).action == old.action)) do + |> Enum.flat_map_reduce(added, fn old, pool -> + case Enum.find_index(pool, &(String.downcase(&1.action) == String.downcase(old.action))) do nil -> - [] - - new_line -> - new = parse_pin(new_line) - - if new.ref == old.ref do - [] - else - from = resolve(resolution, old.action, old.ref) - to = resolve(resolution, new.action, new.ref) - claim = claim_for(claims, old.action) - - cond do - from != nil and to != nil and claim != nil and - (claim.from != from or claim.to != to) -> - delta(old, new, from, to, :conflict, "tags") - - from != nil and to != nil -> - delta(old, new, from, to, :ok, "tags") - - claim != nil -> - delta(old, new, claim.from, claim.to, :ok, "body-claim") - - true -> - delta(old, new, nil, nil, :unresolved, "none") - end - |> Map.put(:file, filename_of(file)) - # flat_map needs a list: a bare map would be flattened into - # its {key, value} pairs. - |> List.wrap() - end + {[], pool} + + i -> + {Enum.at(pool, i) |> pin_delta(old, file, resolution, claims, shared), + List.delete_at(pool, i)} end end) + |> elem(0) end) end - defp delta(old, _new, from, to, status, source) do + defp pin_delta(new, old, file, resolution, claims, shared) do + if new.ref == old.ref do + [] + else + from = resolve(resolution, old.action, old.ref) + to = resolve(resolution, new.action, new.ref) + claim = claim_for(claims, old.action, shared) + + cond do + from != nil and to != nil and claim != nil and + (claim.from != from or claim.to != to) -> + delta(old, new, from, to, :conflict, "tags") + + from != nil and to != nil -> + delta(old, new, from, to, :ok, "tags") + + # One endpoint resolved: the claim must agree with it, or the + # evidence there is contradicts it (a claimed `2.0.0 -> 2.1.0` from + # a ref that resolves to `1.0.0` may hide a major crossing). + claim != nil and ((from != nil and claim.from != from) or (to != nil and claim.to != to)) -> + delta(old, new, from || claim.from, to || claim.to, :conflict, "tags") + + claim != nil -> + delta(old, new, claim.from, claim.to, :ok, "body-claim") + + true -> + delta(old, new, nil, nil, :unresolved, "none") + end + |> Map.put(:file, filename_of(file)) + # flat_map_reduce needs a list: a bare map would be flattened into its + # {key, value} pairs. + |> List.wrap() + end + end + + defp delta(old, new, from, to, status, source) do %{ action: old.action, from: from, to: to, + to_ref: new.ref, status: status, source: source, file: nil, @@ -329,6 +424,12 @@ defmodule Hypatia.Rules.PrAutomerge do not dependency_bot?(author, policy) -> reject(base, :flag, "not_a_dependency_bot", "P3") + Enum.any?(scan.poison_sites, &(not &1.certain)) -> + # The hunk shows a denylisted pin among script text: it may be a + # fixture inside a `run: |` whose header is outside the hunk. Closing + # the PR or excising the line would both act on a guess. + reject(base, :flag, "denylisted_pin_context_unknown", "P1") + scan.poison_sites != [] and scan.major_delta -> reject(base, :close_poison_and_majors, "introduces_denylisted_pin_and_major_bumps", "P1") @@ -337,12 +438,23 @@ defmodule Hypatia.Rules.PrAutomerge do reject(base, :close_poison_only, "introduces_denylisted_pin", "P1") scan.poison_sites != [] -> - accept( - base, - :excise_poison_then_merge, - "introduces_denylisted_pin_alongside_wanted_updates", - "P1" - ) + # Finding the poison must not upgrade a PR that would be flagged + # without it: what remains after the excision is classified on its + # own, and only a remainder that would arm by itself is merged. + rest = verdict(remainder_scan(scan, field(pr, :files, []), policy), pr, policy) + + if rest.safety == "arm_auto" do + accept( + base, + :excise_poison_then_merge, + "introduces_denylisted_pin_alongside_wanted_updates", + "P1" + ) + else + base + |> Map.put(:remainder_blocked_by, rest.blocked_by) + |> reject(:flag, "introduces_denylisted_pin_alongside_unmergeable_changes", "P1") + end scan.major_delta -> reject(base, :flag, "major_version_delta", "P2") @@ -456,7 +568,10 @@ defmodule Hypatia.Rules.PrAutomerge do # - - uses: actions/checkout@v4.1.7 # so the marker is part of the match. (Without this, every real diff looks # like a non-pin change and every PR falls through to `flag`.) - @pin_re ~r/^[-+]?\s*-?\s*uses:\s*(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#]+)/ + # The key and the value may each be quoted (`"uses": "owner/repo@sha"`), the + # step may be a flow mapping (`- { uses: owner/repo@sha, with: ... }`), and + # the ref ends at a flow delimiter as well as at a quote, space or comment. + @pin_re ~r/^[-+]?\s*-?\s*(?:\{\s*)?["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ @doc "Parse a `uses:` line into `%{action, base, ref}`; `nil` when it is not one." def parse_pin(line) do @@ -477,20 +592,209 @@ defmodule Hypatia.Rules.PrAutomerge do end end - defp pin_line?(line), do: parse_pin(line) != nil + @doc """ + Every structural `uses:` site on a diff line, in order, as + `%{action, base, ref}`. + + A flow sequence may put several steps on one line + (`+ { uses: a/b@x }, { uses: c/d@y },`); `parse_pin/1` reads only the + first. Which sites count is `Hypatia.Rules.PinSites`'s rule, the same one + PI001 reads a file with. + """ + @spec parse_pins(String.t()) :: [map()] + def parse_pins(line) do + line + |> String.replace(~r/^[-+]/, "") + |> PinSites.sites() + |> Enum.map(fn %{action: action, ref: ref} -> + %{action: action, base: action_base(action), ref: ref} + end) + end - defp content_lines(patch) do - patch + defp pin_line?(line), do: line |> String.replace(~r/^[-+]/, "") |> PinSites.pin_line?() + + # Every changed line of a patch as `{line, scalar?, certain?, continued}`. + # + # `continued`: the pins the line holds as the value of a `uses:` key whose + # reference YAML continues below it (`- uses:`, then the reference on the + # line under it, past any blank or comment lines) — read, like + # `PinSites.continuations/1` reads a file, from the line's own side of the + # hunk, so a re-pinned value under an unchanged key is still a pin site. + # + # `scalar?`: the line sits inside a scalar an earlier line of its own side + # opened, as far as the hunk shows — a block scalar (`run: |`, + # `script: >-`) or a quoted one continued over lines (`- run: "echo` ... + # `done"`). Each side's context and changed lines are read in order by + # `PinSites.line_states/1`, the reader PI001 reads a file with. A pin + # written inside a script is text. So is a block `uses:` key the side's + # indentation shows under no step or job — an input of that name (`with:` + # then `uses: ...`, `PinSites.data_uses_lines/1`): data, not a pin line. + # + # `certain?`: the hunk does not show the line among script text. A line + # after an opening quote the hunk never shows closed is not certain: the + # scalar may run on past the hunk. A hunk + # that starts inside a long `run: |` has no header to see, so the line's + # YAML ancestry is unknown; the context around it says which. Every other + # line of its side at its indentation or shallower — its siblings and + # ancestors as far as the hunk shows — must be YAML (a key, a list item, a + # flow bracket, a comment); one that is not (`cat > f <<'EOF'`, `EOF`, + # `make test`) leaves it uncertain. A hunk with no such line gives no + # evidence and leaves it certain. + defp changed_lines(file) do + file + |> patch_of() |> String.split("\n") - |> Enum.filter(&Regex.match?(~r/^[-+]/, &1)) - |> Enum.reject(&Regex.match?(~r/^[-+]{3}/, &1)) + |> Enum.chunk_while( + [], + fn line, hunk -> + if String.starts_with?(line, "@@"), + do: {:cont, Enum.reverse(hunk), []}, + else: {:cont, [line | hunk]} + end, + fn hunk -> {:cont, Enum.reverse(hunk), []} end + ) + |> Enum.flat_map(&hunk_lines/1) + end + + defp hunk_lines(hunk) do + hunk = Enum.reject(hunk, &Regex.match?(~r/^[-+]{3}/, &1)) + old = side_view(hunk, "-") + new = side_view(hunk, "+") + old_cont = Map.merge(side_continuations(old), side_aliases(old), fn _, a, b -> a ++ b end) + new_cont = Map.merge(side_continuations(new), side_aliases(new), fn _, a, b -> a ++ b end) + + hunk + |> Enum.with_index() + |> Enum.flat_map(fn {line, i} -> + cond do + String.starts_with?(line, "+") -> + [line_status(line, i, new, Map.get(new_cont, i, []))] + + String.starts_with?(line, "-") -> + [line_status(line, i, old, Map.get(old_cont, i, []))] + + true -> + [] + end + end) + end + + # The continued `uses:` values of one side of a hunk, as + # `%{hunk_index => [pin]}` on the value's line. A `:text` line, a key that + # holds data among them, continues no pin. + defp side_continuations(view) do + texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) + index = view |> Enum.map(&elem(&1, 0)) |> List.to_tuple() + + for {_key, value, site} <- PinSites.continuations(texts), into: %{} do + {elem(index, value - 1), + [%{action: site.action, base: action_base(site.action), ref: site.ref}]} + end + end + + # The anchored values one side of a hunk shows an alias running, as + # `%{hunk_index => [pin]}` on the anchor's line: `- uses: *bad` in context + # with `BAD: &bad @` changed (`PinSites.alias_sites/2`), or a + # step given whole as an alias whose anchored mapping's `uses:` changed + # (`PinSites.node_alias_sites/1`). The re-pinned anchor is the site, once + # however many readers reach it; one that is already a site on its own + # line is counted there, not twice. + defp side_aliases(view) do + texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) + side = Enum.join(texts, "\n") + index = view |> Enum.map(&elem(&1, 0)) |> List.to_tuple() + masked = PinSites.mask_lines(side) + + (PinSites.alias_sites(masked, PinSites.flow_prefixes(side)) ++ PinSites.node_alias_sites(side)) + |> Enum.uniq_by(&{&1.anchor_line, &1.site.ref_at}) + |> Enum.reject(fn %{anchor_line: n, site: site} -> + texts |> Enum.at(n - 1) |> PinSites.sites() |> Enum.any?(&(&1.ref_at == site.ref_at)) + end) + |> Enum.group_by( + &elem(index, &1.anchor_line - 1), + &%{action: &1.site.action, base: action_base(&1.site.action), ref: &1.site.ref} + ) + end + + # One side of a hunk (context lines plus the changed lines of `mark`), in + # order, as `{hunk_index, text, state}` — `state` as + # `PinSites.line_states/1` reads the side: `:yaml`, `:text`, `:value` (a + # `uses:` block scalar's one reference line) or `:open`; a + # block `uses:` key the side shows under no step or job + # (`PinSites.data_uses_lines/1`) is `:text` too. + defp side_view(hunk, mark) do + entries = + hunk + |> Enum.with_index() + |> Enum.filter(fn {line, _} -> + String.starts_with?(line, mark) or String.starts_with?(line, " ") or line == "" + end) + |> Enum.map(fn {line, i} -> {i, String.slice(line, 1..-1//1)} end) + + side = Enum.map_join(entries, "\n", &elem(&1, 1)) + states = PinSites.line_states(side) + # Unless a step or job given whole as an alias runs the mapping it sits + # in (`- *step`, the mapping anchored in a matrix entry). + run = for %{key_line: n} <- PinSites.node_alias_sites(side), into: MapSet.new(), do: n + data = MapSet.difference(PinSites.data_uses_lines(side), run) + + entries + |> Enum.zip(states) + |> Enum.with_index(1) + |> Enum.map(fn {{{i, text}, {_, state}}, n} -> + {i, text, if(n in data, do: :text, else: state)} + end) + end + + defp line_status(line, i, view, continued) do + {before, [{_, text, state} | rest]} = Enum.split_while(view, fn {j, _, _} -> j != i end) + indent = indent_of(text) + + certain? = + state != :open and + (before ++ rest) + |> Enum.reject(fn {_, t, s} -> s in [:text, :value] or String.trim(t) == "" end) + |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) + |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) + + {line, state == :text, certain?, continued} + end + + defp indent_of(text), do: String.length(text) - String.length(String.trim_leading(text, " ")) + + # A line YAML structure could be: a key, a list item, a flow bracket, or + # nothing but a comment. Shell, prose and heredoc markers are not. + defp yaml_like?(text) do + t = text |> PinSites.mask() |> String.trim() + + t == "" or Regex.match?(~r/^(?:-\s+)?["']?[\w.\/-][\w.\/ -]*["']?\s*:(?:\s|$)/, t) or + Regex.match?(~r/^-(?:\s|$)/, t) or Regex.match?(~r/^[{}\[\],]/, t) end - defp added_lines(file), - do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "+")) + defp added_pins(file), + do: + Enum.flat_map(added_entries(file), fn {line, _, continued} -> + parse_pins(line) ++ continued + end) - defp removed_lines(file), - do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "-")) + defp removed_pins(file), + do: + for( + {line, false, _, continued} <- changed_lines(file), + String.starts_with?(line, "-"), + pin <- parse_pins(line) ++ continued, + do: pin + ) + + # Added lines outside block and continued quoted scalars, each with its + # certainty. + defp added_entries(file), + do: + for( + {line, false, certain?, continued} <- changed_lines(file), + String.starts_with?(line, "+"), + do: {line, certain?, continued} + ) defp patch_of(file), do: Map.get(file, :patch) || Map.get(file, "patch") || "" @@ -512,11 +816,21 @@ defmodule Hypatia.Rules.PrAutomerge do Map.get(resolution, ref) end - defp claim_for(claims, action) do - short = action |> String.split("/") |> List.last() - - Enum.find(claims, fn c -> c.action == action or c.action == short end) || - Enum.find(claims, fn c -> String.ends_with?(c.action, "/" <> short) end) + # The body claim for `action`: one naming the action itself, or its root + # repository (`github/codeql-action` for `github/codeql-action/init`), in + # either case. A claim naming only a basename (`[setup](...)`) is this + # action's only when no other action of the PR shares that basename + # (`shared`). A claim for another repository that ends the same way + # (`acme/setup` for `other/setup`) is never borrowed: the delta stays + # unresolved rather than verified by someone else's version numbers. + defp claim_for(claims, action, shared) do + full = String.downcase(action) + base = String.downcase(action_base(action)) + short = full |> String.split("/") |> List.last() + + Enum.find(claims, &(String.downcase(&1.action) == full)) || + Enum.find(claims, &(String.downcase(&1.action) == base)) || + if(short not in shared, do: Enum.find(claims, &(String.downcase(&1.action) == short))) end defp major_of(version) do diff --git a/lib/rules/sha_bump_propagation.ex b/lib/rules/sha_bump_propagation.ex index a71fd169..c667e044 100644 --- a/lib/rules/sha_bump_propagation.ex +++ b/lib/rules/sha_bump_propagation.ex @@ -70,7 +70,14 @@ defmodule Hypatia.Rules.ShaBumpPropagation do {"hyperpolymath/standards", ".github/workflows/codeql-reusable.yml"}, {"hyperpolymath/standards", ".github/workflows/hypatia-scan-reusable.yml"}, {"hyperpolymath/standards", ".github/workflows/mirror-reusable.yml"}, - {"hyperpolymath/standards", ".github/workflows/changelog-reusable.yml"} + {"hyperpolymath/standards", ".github/workflows/changelog-reusable.yml"}, + # hyperpolymath/cicd-suite (cicd-suite#54): the estate audit and the four + # language gates, consumed as thin callers at a full cicd-suite SHA. + {"hyperpolymath/cicd-suite", ".github/workflows/main-estate-audit.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/rust-ci.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/creusot-gate.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/spark-gate.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/kanren-gate.yml"} ] # Composite actions (referenced via `uses: /@`) also flow diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index fd895b71..0f96cb0b 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -93,6 +93,8 @@ defmodule Hypatia.Rules.WorkflowAudit do heading_regex_issues = check_unanchored_heading_regex(workflow_contents) d_burn_issues = check_d_burn_double_trigger(workflow_contents) secret_history = check_secret_history_coverage(workflow_contents) + toolchain_input_missing = check_sha_pinned_toolchain_without_input(workflow_contents) + broken_reusable_revisions = check_known_broken_reusable_revision(workflow_contents) %{ # `flawed_regexes` was computed at the top of this function and @@ -129,7 +131,9 @@ defmodule Hypatia.Rules.WorkflowAudit do heading_regex_issues ++ d_burn_issues ++ flawed_regexes ++ - secret_history, + secret_history ++ + toolchain_input_missing ++ + broken_reusable_revisions, missing_count: length(missing), secret_history_count: length(secret_history), unpinned_count: length(unpinned), @@ -153,6 +157,8 @@ defmodule Hypatia.Rules.WorkflowAudit do concurrency_missing_count: length(concurrency_missing), heading_regex_issues_count: length(heading_regex_issues), d_burn_issues_count: length(d_burn_issues), + toolchain_input_missing_count: length(toolchain_input_missing), + broken_reusable_revision_count: length(broken_reusable_revisions), workflow_count: length(workflow_files), standard_coverage: coverage_percentage(workflow_files) } @@ -1104,6 +1110,1909 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end + # ─── WF026: SHA-pinned dtolnay/rust-toolchain with no toolchain: input ── + + # The 40 hex characters must end the ref: `@-patched` is a tag that + # merely begins with one, and a tag carries its own channel. Git resolves + # a commit ID in either case, so upper-case hex is the same pin. + # The owner/repository is matched in either case — GitHub resolves + # `Dtolnay/Rust-Toolchain` to the same action — and the SHA as written. + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + + # The same key written inside a flow mapping on the line that opens it: + # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + + @doc """ + WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no + `toolchain:` input. + + The action reads the toolchain from its `@`-ref (`@stable`, `@1.85.0`). + A full-SHA ref carries no channel, so a SHA-pinned step with no + `with: toolchain:` dies in its first step with + `'toolchain' is a required input` — before any cargo command runs — and + the red reads as a code failure. plasma-parser-writer's + `rust-ci / Cargo check + clippy + fmt` was red this way from 2026-09-29 + to 2026-10-14 (plasma-parser-writer#105); standards' rust-ci-reusable.yml + carried the defect from d73c93e0 to 49ed4715 (standards#439 fixed it). + WF027 covers the CALLER side of that window; this rule is the defect + itself, wherever it is written — a reusable, a plain workflow, a fork. + + Sensitivity / specificity: + * Fires only on `uses: dtolnay/rust-toolchain@<40-hex>` whose step — + the whole list item, from its opening `- ` line (a bare `-` on its own + line opens a step too), so a `with:` written above `uses:` counts + (YAML mapping keys carry no order) — has no `with:` mapping naming + `toolchain:`, in block form (a direct child of `with:`) or flow form + (`with: { toolchain: stable }`). A tag-pinned step carries its channel + in the ref and never fires. Keys may be quoted (`"uses":`): the same + key to GitHub, the same key here. + * Only a `uses:` that is a KEY OF THE STEP MAPPING is the step's action. + The same text nested under `env:`, inside a `run: |` script that + writes a fixture, or under any other key is data: no finding, so the + fleet is never sent to rewrite script text. + * Only `with:` decides: `toolchain:` under `env:` or any other key is + not an action input and does not rescue the step. + * The step is bounded by indentation, so a later step's `toolchain:` + cannot rescue an earlier step. + * Severity is low and the finding is `advisory: true`: the rule reports + and routes to the fleet; it is never a blocking gate (AGENTIC + `must-not-promote-to-blocking`). The CLI never counts it toward exit 1 + and reports it below the threshold under `--advisory` + (`Hypatia.CLI.report_sets/2`). + * A KYAML workflow (standards YAML-POLICY Y-3, the output of + `yq -o kyaml`) is judged as its block equivalent, line for line + (`ResearchExtensions.flow_to_block/1`), so a quoted, braced step is + the same step. A flow-style step written inline in a block document + (`- { uses: ..., with: { toolchain: stable } }`, + `steps: [ { uses: ... } ]`, or a `- {` item spread over lines) is read + through its braces: its `with` entry decides, quotes and nested + braces respected, and a `uses:` inside a quoted string is text. A + workflow written in flow style on one line (`jobs: { build: { steps: + [ { uses: ... } ] } }`, with or without a root `{ }`) is read the same + way: the keys opening the brackets around the step must be `steps` + in a job in `jobs`. + * `with:` may be an anchor (`with: &inputs`, block or flow), an alias + (`with: *inputs`) or carry a merge key (`<<: *inputs`): the anchored + mapping is resolved from the document — as YAML resolves an alias, to + the latest `&inputs` before it — and its direct entries decide. An + alias with no anchor before it in the file cannot be read; the step is + reported so, not passed silently. + * A `uses:` whose value YAML continues below its key (`- uses:`, then — + past any blank or comment-only lines — the reference on a deeper line) + is the step's key all the same. + + Auto-fixable: gitbot-fleet `scripts/fix-rust-toolchain-input.sh` inserts + `toolchain: stable` (the channel the estate's reusables pass) into a + block-style step whose `with:` is absent, a block mapping or a flow + mapping. The fixer refuses every other shape rather than guess (exit 2, + file untouched), so a finding on a KYAML document, a flow-style step, a + `uses:` value continued below its key, or a `with:` that goes through an + anchor or alias carries `auto_fixable: false` and routes to review instead + of to the fixer. + """ + def check_sha_pinned_toolchain_without_input(workflow_contents) do + Enum.flat_map(workflow_contents, fn {filename, content} -> + # KYAML is read as the block layout it denotes; line numbers survive. + normalised = Hypatia.Rules.ResearchExtensions.flow_to_block(content) + kyaml? = normalised != content + lines = String.split(normalised, "\n") + continued = wf_continued_uses(lines) + # A step inside an open flow collection is a flow-style step even when + # its `{` stands alone on a line above (`steps: [`, `{`, `uses: ...`). + flow = wf_flow_lines(normalised) + # A step given whole as an alias (`- *step`, `steps: [*step]`) runs the + # mapping its anchor holds, wherever that mapping sits (a matrix entry): + # its `uses:` line is a live step's, judged as one, and routes to + # review — the fixer does not rewrite anchored steps. + aliased = + for %{key_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(normalised), + into: MapSet.new(), + do: n + + lines + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + # `- uses:` with the reference below it, or `- uses: *name` naming an + # anchored one, is read as the one-line key it stands for. + {line, resolved} = + case Map.fetch(continued, number) do + {:ok, {joined, how}} -> {joined, how} + :error -> {line, nil} + end + + live? = number in aliased + + case Regex.run(@toolchain_sha_pin, line) do + [_full, sha] -> + # The fleet fixer matches the identity in lower case only, and a + # value on its key's line only; any other spelling or layout is + # reported for review rather than handed to it. + review = + cond do + kyaml? -> :kyaml + resolved != nil -> resolved + live? -> :node_alias + wf_node_properties?(line) -> :properties + not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling + true -> nil + end + + if wf_in_flow?(flow, number), + do: wf026_flow_line_finding(filename, lines, flow, number, sha, review, 0, live?), + else: wf026_block_step_finding(filename, lines, line, number, sha, review, live?) + + _ -> + # Several flow-style steps may share one physical line + # (`steps: [ { uses: ... }, { uses: ... } ]`): each is judged. + @toolchain_sha_pin_inline + |> Regex.scan(line, return: :index) + |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> + sha = binary_part(line, sha_at, 40) + + review = + cond do + kyaml? -> :kyaml + live? -> :node_alias + true -> nil + end + + # On a line inside a flow collection opened above, the step's + # ancestry is in the brackets, not the line's indentation. + if wf_in_flow?(flow, number) do + wf026_flow_line_finding(filename, lines, flow, number, sha, review, at + 1, live?) + else + wf026_inline_step_finding( + filename, + lines, + line, + number, + at + 1, + sha, + review, + live? + ) + end + end) + end + end) + end) + end + + # The block-style path: `uses:` is a key on its own line, outside any flow + # collection (one inside is read by `wf026_flow_line_finding/7`). A step + # whose opener is `- {` is a flow-style step: the fixer refuses it. + defp wf026_block_step_finding(filename, lines, line, number, sha, review, live?) do + {step, key_indent, opener} = wf026_step(lines, number) + flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) + + cond do + # `uses:` nested under `env:`, inside a `run: |` script or under + # any other key is data, not the step's action. + wf026_key_indent(line) != key_indent -> + [] + + # A quoted scalar the step opened above (`- run: "echo start`) and has + # not closed by this line makes the `uses:` line part of the string. + # The scan starts at the step's opener or, where the string opens on + # the line that holds the step (`steps: [{"run":"echo start`, no list + # item of its own), at the line's YAML parent. + wf_in_open_quote?(lines, min(opener, wf_parent_line(lines, number)), number) -> + [] + + # A list item is a step only under `steps:`: one inside a block + # scalar (a fixture kept in `env:`, a `run: |` script) is text. A + # mapping a step alias runs (`live?`) is a step wherever it sits. + not live? and not wf026_under_steps?(lines, opener) -> + [] + + true -> + case wf026_with(step, key_indent, lines, opener) do + {:toolchain, _form} -> + [] + + verdict -> + form = if flow_item?, do: :flow_step, else: elem(verdict, 1) + [wf026_finding(filename, number, sha, verdict, form, review)] + end + end + end + + # The inline path: `uses:` sits inside a flow mapping on the line that + # opens it. The mapping is read through its braces (quote-aware, across + # lines) and its `with` entry decides. + defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, review, live?) do + with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), + true <- live? or wf026_inline_step_at?(lines, line, number, open_col), + {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do + # Where the step's `with` value stands, line and column: an alias + # there names the latest anchor before it, not one later on its line. + pos = + case wf_flow_entry_value_at(interior, "with") do + nil -> {number, :eol} + at -> wf_interior_pos(interior, at, number, open_col) + end + + case wf026_with_entry(wf_flow_entries(interior), lines, pos) do + {:toolchain, _} -> [] + verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] + end + else + _ -> [] + end + end + + # A `uses:` key inside a flow collection opened on an earlier line — + # KYAML-like layout, indented or not (`steps: [`, `{`, `uses: ...`) — at + # the line's start, or at byte `at` of it (`{ uses: ...`, `name: x, + # uses: ...`). Its ancestry is the brackets still open before it + # (`wf_flow_context/4`), not its indentation: it is a step's own key when + # the innermost one is a mapping that is an item of the `[` a `steps:` in + # a job opens, and the step is read through that mapping's braces. + defp wf026_flow_line_finding(filename, lines, flow, number, sha, review, at, live?) do + with {:ok, keys, kinds, {brace_line, brace_col}} <- wf_flow_context(lines, flow, number, at), + true <- live? or match?([nil, "steps", job, "jobs" | _] when is_binary(job), keys), + # The `{` is an item of the `[` that `steps:` opens, or of the + # block sequence under it (`- {` on a line of its own); a mapping + # a step alias runs is the step wherever it sits. + true <- + match?([:map, :seq | _], kinds) or kinds == [:map] or + (live? and match?([:map | _], kinds)), + {:ok, interior, _} <- wf_flow_interior(lines, brace_line, brace_col) do + pos = + case wf_flow_entry_value_at(interior, "with") do + nil -> {brace_line, :eol} + at -> wf_interior_pos(interior, at, brace_line, brace_col) + end + + case wf026_with_entry(wf_flow_entries(interior), lines, pos) do + {:toolchain, _} -> [] + verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] + end + else + _ -> [] + end + end + + # Is the flow mapping opening at `open_col` a step? Either it sits in a + # sequence position of a block document (below), or the line itself + # writes the path to it in flow style. + defp wf026_inline_step_at?(lines, line, number, open_col) do + (wf026_inline_item?(line, open_col) and + wf026_inline_under_steps?(lines, line, number, open_col)) or + wf026_flow_path_step?(lines, line, number, open_col) + end + + # A workflow written in flow style on one line — `jobs: { build: { steps: + # [ { uses: ... } ] } }`, the same under a root `{ ... }`, or a job line + # `build: { steps: [ ... ] }` under a block `jobs:`. The mapping must sit + # directly in a `[` that `steps:` opens, in a mapping a job key opens, in + # the one `jobs:` opens: the keys that open the brackets still unclosed on + # the line, nearest first, then the block ancestors of the line. Each key + # is read quote-aware, so a `name: "CI, build"` sibling separates nothing. + defp wf026_flow_path_step?(lines, line, number, open_col) do + case {wf_flow_opener_stack(line, open_col), wf_ancestor_keys(lines, number)} do + {{:ok, [{:seq, _} | _] = stack}, ancestors} when is_list(ancestors) -> + keys = wf_flow_opener_keys(line, stack) ++ ancestors + match?(["steps", job, "jobs" | _] when is_binary(job), keys) + + _ -> + false + end + end + + # The key that opens each bracket of an opener stack (innermost first), or + # nil for a bracket no key opens (a root `{`, a `{` item in a sequence). + defp wf_flow_opener_keys(line, stack) do + cols = + Enum.map(stack, fn + {:seq, col} -> col + col -> col + end) + + froms = cols |> Enum.drop(1) |> Enum.map(&(&1 + 1)) |> Kernel.++([0]) + + Enum.zip_with(cols, froms, fn col, from -> wf027_key_before(line, from, col) end) + end + + # The text before the mapping's `{` must be a sequence position: a list + # dash, the `steps:` key opening its sequence, a `[`, a comma, or nothing. + # Anything else (`run: "echo { uses: ... }"`) is a scalar carrying braces. + # An earlier sibling on the same line (`[ { uses: a }, { uses: b } ]`) is + # a closed mapping before this one; it is removed before the test. + # Node properties may stand before the mapping or the sequence it sits in + # (`- &step {`, `steps: &s [ &step {`): they name the node, not a key. + defp wf026_inline_item?(line, open_col) do + prefix = + line + |> binary_part(0, open_col) + |> wf_drop_closed_mappings() + |> String.replace(~r/(^|[\s\[{,:-])[&!]\S*(?=\s)/, "\\1") + + Regex.match?(~r/^\s*(?:-\s*)?(?:["']?steps["']?\s*:\s*)?[\[,\s]*$/, prefix) + end + + defp wf_drop_closed_mappings(text) do + case Regex.replace(~r/\{[^{}]*\}/, text, "") do + ^text -> text + shorter -> wf_drop_closed_mappings(shorter) + end + end + + defp wf026_inline_under_steps?(lines, line, number, open_col) do + prefix = binary_part(line, 0, open_col) + + if Regex.match?(~r/["']?steps["']?\s*:/, prefix) do + # The line is the `steps:` key itself; it belongs to a job. + match?([_job, "jobs" | _], wf_ancestor_keys(lines, number) || []) + else + wf026_under_steps?(lines, number) + end + end + + # The finding. Only a plain block-style step is handed to the fixer: + # the fleet script refuses flow-style steps, anchors and aliases (exit + # 2, file untouched), and a KYAML document is a flow-style document. + # `review` says why a finding must route to review whatever its form: + # :kyaml (a flow-style document) or :spelling (the action named in a case + # the fixer does not match); nil when only the form decides. + defp wf026_finding(filename, number, sha, verdict, form, review) do + repairable? = review == nil and form in [:none, :block, :flow] + + shape = + case {verdict, form} do + {{:unreadable, :alias, name}, _} -> + " Its `with:` is the alias `*#{name}` and no `&#{name}` anchor is defined before " <> + "it in the file, so the inputs cannot be read; the alias must be resolved by hand." + + {{:unreadable, :toolchain_alias, name}, _} -> + " Its `toolchain:` input is the alias `*#{name}`, and no `&#{name}` scalar this " <> + "reader can read is defined before it, so whether the action receives a channel " <> + "cannot be told; the alias must be checked by hand." + + {_, :flow_step} -> + " The step is written in flow style; the repair script rewrites block-style " <> + "steps only, so this one routes to review." + + {_, f} when f in [:anchor, :alias, :merge] -> + " Its `with:` goes through a YAML anchor or alias; the repair script does not " <> + "rewrite anchored mappings, so this one routes to review." + + _ when review == :kyaml -> + " The workflow is KYAML; the repair script rewrites block-style steps only, " <> + "so this one routes to review." + + _ when review == :properties -> + " The `uses:` value carries a YAML anchor or tag; the repair script reads a " <> + "bare reference only, so this one routes to review." + + _ when review == :continued -> + " The `uses:` value is continued below its key; the repair script reads a " <> + "reference on the key's own line only, so this one routes to review." + + _ when review == :node_alias -> + " The step is given as an alias (`- *name`) of a mapping anchored elsewhere; " <> + "the repair script does not rewrite anchored steps, so this one routes to review." + + _ when review == :alias -> + " The `uses:` value is an alias of an anchored reference; the repair script " <> + "does not rewrite anchored values, so this one routes to review." + + _ when review == :spelling -> + " The action is not spelled `dtolnay/rust-toolchain`; GitHub resolves the name in " <> + "either case, but the repair script matches it as written in lower case, so this " <> + "one routes to review." + + _ -> + "" + end + + %{ + rule: "WF026", + type: :sha_pinned_toolchain_without_input, + file: filename, + line: number, + current_sha: sha, + # Report and route, never block (AGENTIC `must-not-promote-to-blocking`): + # `:low` (rank 4) sits below the CLI's default `--severity medium` and + # the reusable's high/critical threshold, as `flawed_regex` does. The + # fleet routing reads `fix_script`/`auto_fixable`, not the severity. + # `advisory: true` keeps it out of the CLI's exit code at any + # threshold and lets `--advisory` report it below one. + severity: :low, + advisory: true, + auto_fixable: repairable?, + fix_script: if(repairable?, do: "fix-rust-toolchain-input.sh", else: nil), + reason: + "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> + "input. The action reads the channel from its `@`-ref; a commit SHA " <> + "carries none, so the step exits 1 with \"'toolchain' is a required " <> + "input\" before any cargo command runs and the red reads as a code " <> + "failure. Add `with: toolchain: stable` (or the channel you mean)." <> shape, + action: :add_toolchain_input + } + end + + # Indentation width of a line (spaces only; workflow YAML never uses tabs). + defp wf026_indent(line), do: String.length(line) - String.length(String.trim_leading(line, " ")) + + # A line belongs to the step opened at `step_indent` until the next list item + # at that indentation or shallower, or a shallower key. Blank lines and + # comment lines belong to whatever step they sit in. + defp wf026_step_line?(line, step_indent) do + trimmed = String.trim(line) + + cond do + trimmed == "" -> true + String.starts_with?(trimmed, "#") -> true + Regex.match?(~r/^\s*-(?:\s|$)/, line) and wf026_indent(line) <= step_indent -> false + wf026_indent(line) < step_indent -> false + true -> true + end + end + + defp wf026_plain?(line) do + trimmed = String.trim(line) + trimmed != "" and not String.starts_with?(trimmed, "#") + end + + # The whole list item the `uses:` line at `number` (1-based) belongs to: + # from its opening `- ` line — which may sit above `uses:`, since YAML + # mapping keys carry no order, and may be a bare `-` on its own line — to + # the next list item at that indentation or shallower, or a shallower key. + # Returns the step's lines and the column of its keys, so a caller can tell + # a key of the step mapping from the same text nested under another key. + defp wf026_step(lines, number) do + uses_line = Enum.at(lines, number - 1) + uses_indent = wf026_indent(uses_line) + + {opener, step_indent} = + if Regex.match?(~r/^\s*-(?:\s|$)/, uses_line) do + {number, uses_indent} + else + lines + |> Enum.take(number - 1) + |> Enum.with_index(1) + |> Enum.reverse() + |> Enum.find(fn {l, _} -> + Regex.match?(~r/^\s*-(?:\s|$)/, l) and wf026_indent(l) < uses_indent + end) + |> case do + {l, i} -> {i, wf026_indent(l)} + nil -> {number, uses_indent} + end + end + + rest = + lines + |> Enum.drop(opener) + |> Enum.take_while(&wf026_step_line?(&1, step_indent)) + + step = [Enum.at(lines, opener - 1) | rest] + {step, wf026_step_key_indent(step), opener} + end + + # A list item is a step only when its parent key is a job's `steps:`, or + # a composite action's under `runs:`. An item inside `run: |`, inside a + # fixture kept in `env:`, or of a `steps:` some other mapping holds (a + # matrix entry's `fixture: { steps: ... }`) is data however much it looks + # like a step, and must never send the fleet to rewrite it. A `steps:` + # with nothing above it (a fragment) is still read as a step list. + defp wf026_under_steps?(lines, opener) do + case wf_ancestor_keys(lines, opener) do + ["steps"] -> true + ["steps", "runs" | _] -> true + ["steps", job, "jobs" | _] when is_binary(job) -> true + _ -> false + end + end + + # The mapping keys enclosing the line at `number`, nearest first — list + # items are nodes, not keys, and are skipped — or nil when an ancestor + # opens a block scalar: the line is then text, not YAML structure. The + # parent of a list item may sit at the item's own indentation (YAML's + # `steps:\n- uses:` shape); every further ancestor is strictly shallower. + defp wf_ancestor_keys(lines, number) do + line = Enum.at(lines, number - 1) + indent = wf026_indent(line) + list_item? = Regex.match?(~r/^\s*-(?:\s|$)/, line) + limit = if list_item?, do: indent, else: indent - 1 + + lines + |> Enum.take(number - 1) + |> Enum.reverse() + |> Enum.filter(&wf026_plain?/1) + |> wf_walk_up(limit, list_item?, []) + end + + defp wf_walk_up([], _limit, _sibling_ok?, keys), do: Enum.reverse(keys) + + defp wf_walk_up([line | rest], limit, sibling_ok?, keys) do + indent = wf026_indent(line) + list_item? = Regex.match?(~r/^\s*-(?:\s|$)/, line) + + cond do + indent > limit -> + wf_walk_up(rest, limit, sibling_ok?, keys) + + # A sibling item of the one being placed. + list_item? and sibling_ok? and indent == limit -> + wf_walk_up(rest, limit, sibling_ok?, keys) + + # A flow closer (`}`, `],`) on its own line ends a collection some + # earlier line opened; it names nothing and bounds nothing. + Regex.match?(~r/^\s*[}\]]/, line) -> + wf_walk_up(rest, limit, sibling_ok?, keys) + + Regex.match?(~r/:\s*[|>][-+0-9]*\s*(?:#.*)?$/, line) -> + nil + + # A list item, or a `{`/`[` opening a flow item on its own line, is a + # node, not a key. + list_item? or Regex.match?(~r/^\s*[{\[]/, line) -> + wf_walk_up(rest, indent - 1, false, keys) + + true -> + wf_walk_up(rest, indent - 1, false, [wf_key_name(line) | keys]) + end + end + + # The flow depth and the structure of each line of a file + # (`PinSites.line_depths/1`, `PinSites.structure_lines/1`), read once. + defp wf_flow_lines(content) do + %{ + depths: content |> Hypatia.Rules.PinSites.line_depths() |> List.to_tuple(), + text: content |> Hypatia.Rules.PinSites.structure_lines() |> List.to_tuple() + } + end + + defp wf_in_flow?(flow, number), do: wf_flow_at(flow.depths, number, 0) > 0 + + defp wf_flow_at(tuple, number, default) do + if number >= 1 and number <= tuple_size(tuple), do: elem(tuple, number - 1), else: default + end + + # For line `number` inside a flow collection — its depth at the start + # (`PinSites.line_depths/1`) above 0 — read up to byte `at` of it, the + # position judged: `{:ok, keys, kinds, {line, col}}`, the keys opening the + # brackets still open there, innermost first (nil for one no key opens, a + # `{` item of a sequence), then the block ancestors of the line that + # opened the outermost; each bracket's kind (:map for `{`, :seq for `[`); + # and where the innermost one stands. Read from the structure of the lines + # since that one (`PinSites.structure_lines/1`: comments cut and quoted + # scalars blanked across lines, so a `# world"` continuing a string is + # text), so a collection spread over lines is one however its lines are + # indented — column 0 included. nil at depth 0, past the line's comment, + # or when that text cannot be read. + defp wf_flow_context(lines, flow, number, at \\ 0) do + start = + if number > 1, + do: Enum.find((number - 1)..1//-1, &(wf_flow_at(flow.depths, &1, 0) == 0)) + + here = wf_flow_at(flow.text, number, "") + + with true <- wf_in_flow?(flow, number), + s when is_integer(s) <- start, + true <- at <= byte_size(here), + # The key itself must be structure: a `uses:` the structure view + # blanks is inside a quoted scalar continued from above. + true <- + Regex.match?( + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:/, + binary_part(here, at, byte_size(here) - at) + ), + text = + Enum.map_join(s..(number - 1), "\n", &wf_flow_at(flow.text, &1, "")) <> + "\n" <> binary_part(here, 0, at), + {:ok, [inner | _] = stack} <- wf_flow_opener_stack(text, byte_size(text)) do + keys = wf_flow_opener_keys(text, stack) ++ (wf_ancestor_keys(lines, s) || []) + kinds = Enum.map(stack, &if(is_integer(&1), do: :map, else: :seq)) + at = if is_integer(inner), do: inner, else: elem(inner, 1) + {:ok, keys, kinds, wf_interior_pos(text, at, s, -1)} + else + _ -> nil + end + end + + defp wf_key_name(line) do + line + |> String.replace(~r/^\s*(?:-\s+)?/, "") + |> String.replace(~r/\s*:.*$/, "") + |> String.trim("\"") + |> String.trim("'") + end + + # The column of the step's keys: the opener's key (`- uses:` -> the column + # of `uses`), or for a bare `-` — or a `- {` opening a flow-style item — the + # indentation of the first plain line under it. + defp wf026_step_key_indent([opener | rest]) do + cond do + Regex.match?(~r/^\s*-\s+\S/, opener) and not Regex.match?(~r/^\s*-\s+\{\s*$/, opener) -> + wf026_key_indent(opener) + + Regex.match?(~r/^\s*-\s*(?:\{\s*)?$/, opener) -> + case Enum.find(rest, &wf026_plain?/1) do + nil -> wf026_indent(opener) + 2 + l -> wf026_indent(l) + end + + true -> + wf026_indent(opener) + end + end + + # What the step's `with:` says about `toolchain:`, as a verdict: + # `{:toolchain, form}`, `{:missing, form}` or `{:unreadable, :alias, name}`, + # where form is :none (no `with:`), :block, :flow, :anchor (`with: &x`), + # :alias (`with: *x`) or :merge (a `<<: *x` child). Only a `with:` at the + # step's key column is the step's own; `toolchain:` under `env:` or any + # other key is not an action input. + # `step` starts at line `opener`, so its line i is line `opener + i`: where + # an alias in `with:` stands, which decides the anchor it names. + defp wf026_with(step, key_indent, lines, opener) do + step + |> Enum.with_index() + |> Enum.find_value({:missing, :none}, fn {line, i} -> + if wf026_key_indent(line) == key_indent do + case Regex.run(~r/^\s*(?:-\s+)?["']?with["']?\s*:(.*)$/, line) do + [_, value] -> + value = value |> wf_strip_comment() |> String.trim() + wf026_with_value(value, line, Enum.drop(step, i + 1), key_indent, lines, opener + i) + + nil -> + nil + end + end + end) + end + + defp wf026_with_value(value, line, rest, key_indent, lines, number) do + cond do + # `with: *inputs` — the mapping the latest `&inputs` before it anchors. + match?([_ | _], Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value) + [{col, _}] = Regex.run(~r/\*/, line, return: :index) + wf026_alias_verdict(name, lines, :alias, {number, col}) + + # `with: {` / `with: &inputs {` — a flow mapping, read to its brace. + String.contains?(value, "{") -> + form = if wf026_anchored?(value), do: :anchor, else: :flow + [{at, _}] = Regex.run(~r/\{/, line, return: :index) + + case wf_flow_interior([line | rest], 1, at) do + {:ok, interior, _} -> + wf026_entries_verdict(wf_flow_entries(interior), lines, form, {number, :eol}) + + :error -> + {:missing, form} + end + + # `with:` / `with: &inputs` — a block mapping: only a DIRECT child is + # an input; the contents of another input's block scalar + # (`components: |` carrying `toolchain: stable`) sit deeper and are text. + true -> + form = if wf026_anchored?(value), do: :anchor, else: :block + pos = {:block, number, wf_block_child_lines(rest, key_indent, number)} + wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form, pos) + end + end + + # Does a `with:` value carry an anchor? Node properties come in either + # order: `&inputs`, `&inputs !!map`, `!!map &inputs`. + defp wf026_anchored?(value), do: Regex.match?(~r/^(?:!\S*\s+)*&/, value) + + # The `with` entry of a flow-style step's own entries; `pos` is where its + # value stands, which decides the anchor an alias there names. + defp wf026_with_entry(entries, lines, pos) do + case List.keyfind(entries, "with", 0) do + nil -> + {:missing, :none} + + {_, value} -> + value = String.trim(value) + + cond do + match?([_ | _], Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value) + wf026_alias_verdict(name, lines, :alias, pos) + + String.contains?(value, "{") -> + form = if wf026_anchored?(value), do: :anchor, else: :flow + [{at, _}] = Regex.run(~r/\{/, value, return: :index) + + case wf_flow_interior([value], 1, at) do + {:ok, interior, _} -> + wf026_entries_verdict(wf_flow_entries(interior), lines, form, pos) + + :error -> + {:missing, form} + end + + true -> + {:missing, :flow} + end + end + end + + # Does a list of `{key, value}` entries name a toolchain? A `<<: *x` merge + # key brings the anchored mapping's entries in, as does each alias of a + # `<<: [*base, *rust]` sequence, but an explicit key wins over a merged one + # (YAML merge semantics): `toolchain:` written empty beside `<<: *inputs` + # is the empty value, whatever the anchor names. A merge this reader does + # not resolve (`<<: { ... }`, a block sequence) leaves the inputs unknown: + # the finding routes to review (`:merge`), never to the fixer. + # The key present with no value (`toolchain:`, `null`, `~`) is not an input + # the action can read; the fleet fixer fills such a key in place. `pos` + # (`{line, column}`, column `:eol` for the whole line) is where the + # mapping stands: an alias in it names the latest anchor before it. + defp wf026_entries_verdict(entries, lines, form, pos, seen \\ []) do + case wf026_toolchain_in(entries, lines, seen, pos) do + {:set, :own} -> {:toolchain, form} + {:set, :merge} -> {:toolchain, :merge} + :null -> {:missing, form} + :absent -> {:missing, form} + :unknown -> {:missing, :merge} + {:unreadable, _, _} = unreadable -> unreadable + end + end + + # Where a mapping's `toolchain` comes from, under YAML merge semantics: its + # own key first; else each merged mapping in order, the first that defines + # the key deciding — an explicit `toolchain: null` in `*empty` of + # `<<: [*empty, *rust]` is the input, `*rust`'s value is not. Returns + # {:set, :own | :merge}, :null, :absent, :unknown (a merge this reader + # does not resolve) or {:unreadable, :alias, name}. + defp wf026_toolchain_in(entries, lines, seen, pos) do + case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do + {_, v} when is_binary(v) -> + case Regex.run(~r/^\s*\*([^\s,\[\]{}]+)\s*$/, v) do + [_, name] -> wf026_scalar_alias(name, lines, wf_entry_pos(pos, "toolchain")) + nil -> if wf026_null_input?(v), do: :null, else: {:set, :own} + end + + _ -> + entries + |> Enum.filter(fn {k, _} -> k == "<<" end) + |> Enum.find_value(:absent, fn {_, v} -> + case wf026_merge_aliases(v) do + {:ok, names} -> + Enum.find_value(names, fn name -> + case wf026_alias_lookup(name, lines, seen, wf_entry_pos(pos, "<<")) do + :absent -> nil + {:set, _} -> {:set, :merge} + other -> other + end + end) + + :error -> + :unknown + end + end) + end + end + + defp wf026_alias_lookup(name, lines, seen, pos) do + if name in seen do + {:unreadable, :alias, name} + else + case wf_anchor_entries(lines, name, pos) do + {:ok, entries, def_pos} -> wf026_toolchain_in(entries, lines, [name | seen], def_pos) + :error -> {:unreadable, :alias, name} + end + end + end + + # The aliases a merge key's value names: `*x`, or a flow sequence of + # aliases `[*a, *b]`, in order; :error for anything else. + defp wf026_merge_aliases(value) do + v = String.trim(value) + + cond do + m = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, v) -> + {:ok, [Enum.at(m, 1)]} + + m = Regex.run(~r/^\[\s*(\*[^\s,\[\]{}]+(?:\s*,\s*\*[^\s,\[\]{}]+)*)\s*,?\s*\]$/, v) -> + names = + m + |> Enum.at(1) + |> String.split(",") + |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("*"))) + + {:ok, names} + + true -> + :error + end + end + + # Where an entry's value stands, for an alias in it: a block mapping's + # position (`{:block, line, %{key => line}}`) is each child's own line — + # `source: &channel nightly` then `toolchain: *channel` names the anchor + # one line above it — any other position is the mapping's. + defp wf_entry_pos({:block, number, child_lines}, key), + do: {Map.get(child_lines, key, number), :eol} + + defp wf_entry_pos(pos, _key), do: pos + + # `toolchain: *name` is the scalar the latest `&name` before it anchors: + # null or empty there is no input, exactly as if written in place + # (`x: &empty ""`). An anchor this reader cannot find, or one that names a + # mapping, a sequence or a value continued on the lines below, leaves the + # input unreadable, and the finding routes to review. + defp wf026_scalar_alias(name, lines, pos) do + case wf_anchor_scalar(lines, name, pos) do + {:ok, value} -> if wf026_null_input?(value), do: :null, else: {:set, :own} + :error -> {:unreadable, :toolchain_alias, name} + end + end + + # Is a `toolchain` value, as written, no input the action can read? Its + # node properties come off first: an anchor names the node, not its value + # (`&channel ""` is the empty string, `&channel null` null), and a tag + # says how the scalar resolves — `!!null` always to null, `!!str` to the + # string it spells, so `!!str ""` is still empty but `!!str null` is the + # string null, an input. + defp wf026_null_input?(value) do + [_, props, scalar] = Regex.run(~r/^\s*((?:[&!]\S*(?:\s+|$))*)(.*)$/s, value) + tags = Regex.scan(~r/!\S*/, props) |> List.flatten() + + cond do + "!!null" in tags -> true + "!!str" in tags -> String.trim(scalar) in ["", ~s(""), "''"] + true -> wf026_null_value?(scalar) + end + end + + # YAML's null: nothing, `~`, or `null` in the three spellings the core + # schema admits, unquoted. `"null"` is the string null, an input the + # action receives; `""` and `''` are the empty string, which it does not. + defp wf026_null_value?(value) do + case String.trim(value) do + "" -> true + "~" -> true + ~s("") -> true + "''" -> true + bare -> bare in ["null", "Null", "NULL"] + end + end + + # An alias that reaches itself (`&x` whose mapping merges `*x`) is as + # unreadable as one with no anchor. + defp wf026_alias_verdict(name, lines, form, pos, seen \\ []) do + if name in seen do + {:unreadable, :alias, name} + else + case wf_anchor_entries(lines, name, pos) do + {:ok, entries, def_pos} -> + wf026_entries_verdict(entries, lines, form, def_pos, [name | seen]) + + :error -> + {:unreadable, :alias, name} + end + end + end + + # The entries of the mapping anchored `&name` that an alias at `pos` + # (`{line, column}`, column `:eol` for anywhere on the line) names. YAML + # resolves an alias to the latest `&name` before it, and a document may + # define one again (`defaults: &inputs {...}`, then `other: &inputs {}`): + # the last definition before `pos` decides, and one only after it defines + # nothing for it. Definitions are read outside comments, quoted scalars + # and block-scalar content (a `run: |` fixture writing `fake: &inputs` is + # text; so is a `note: "see &name"`), and the mapping as a flow mapping + # when a `{` follows the anchor, else as the block mapping under its line. + # Returns `{:ok, entries, pos}` with the definition's own position (where + # an alias inside the anchored mapping stands), or :error. + defp wf_anchor_entries(lines, name, pos) do + case wf_anchor_def(lines, name, pos) do + nil -> + :error + + {line, masked, number, at, len} -> + after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) + + case Regex.run(~r/\{/, after_anchor, return: :index) do + [{brace, _}] -> + case wf_flow_interior(lines, number, at + len + brace) do + {:ok, interior, _} -> {:ok, wf_flow_entries(interior), {number, :eol}} + :error -> :error + end + + nil -> + # A bare sequence item carrying the anchor (`- &inputs`, its keys + # on the lines below) holds the mapping under the dash, not under + # the column its first key would take. + parent = + if Regex.match?(~r/^\s*-\s+(?:[&!]\S*\s*)+$/, wf_strip_comment(line)), + do: wf026_indent(line), + else: wf026_key_indent(line) + + rest = Enum.drop(lines, number) + children = wf_block_children(rest, parent) + {:ok, children, {:block, number, wf_block_child_lines(rest, parent, number)}} + end + end + end + + # The scalar the anchor `&name` an alias at `pos` names holds, as written + # (quoted or plain, "" for none): `{:ok, value}`, or :error when there is + # no such anchor, or it names a mapping or sequence, or its value is + # continued on the lines below. + defp wf_anchor_scalar(lines, name, pos) do + with {line, masked, number, at, len} <- wf_anchor_def(lines, name, pos), + rest = binary_part(line, at + len, byte_size(line) - at - len), + false <- Regex.match?(~r/^\s*(?:![^\s]*\s+)*[\[{*&|>]/, rest) do + cond do + m = + Regex.run( + ~r/^\s+((?:![^\s]*\s+)*)("(?:[^"\\]|\\.)*"|'(?:[^']|'')*'|[^\s,}\]#][^,}\]#]*?)\s*(?:[,}\]]|\s#|$)/, + rest + ) -> + # The scalar with its tag (`&x !!str null` is the string null). + {:ok, Enum.at(m, 1) <> Enum.at(m, 2)} + + # Nothing after the anchor but a tag or a comment: null, unless a + # deeper line continues the scalar. + Regex.match?( + ~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, + binary_part(masked, at + len, byte_size(masked) - at - len) + ) -> + next = lines |> Enum.drop(number) |> Enum.find(&wf026_plain?/1) + if next && wf026_indent(next) > wf026_indent(line), do: :error, else: {:ok, ""} + + true -> + :error + end + else + _ -> :error + end + end + + # The latest definition of `&name` before `pos`, as + # `{line, masked line, number, offset, length}`, or nil. + # An anchor is `&name` where YAML lets a node start: at the line's start or + # after its list dashes, after `key: `, or — inside a flow collection (one + # the line opens, or the line sits in) — after `[`, `{` or `,`, each past + # any tag. `NOTE: foo&channel nightly` is one plain scalar, no anchor. + defp wf_anchor_def(lines, name, {pos_line, pos_col}) do + re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ + + content = Enum.join(lines, "\n") + states = Hypatia.Rules.PinSites.line_states(content) + depths = Hypatia.Rules.PinSites.line_depths(content) + + lines + |> Enum.zip(Enum.zip(states, depths)) + |> Enum.with_index(1) + |> Enum.take(pos_line) + |> Enum.flat_map(fn {{l, {{_, state}, depth}}, number} -> + if state not in [:text, :value] and wf026_plain?(l) do + masked = wf_mask_quoted(l) + + re + |> Regex.scan(masked, return: :index) + |> Enum.filter(fn [{at, _}] -> number < pos_line or pos_col == :eol or at < pos_col end) + |> Enum.filter(fn [{at, _}] -> wf_node_start?(binary_part(masked, 0, at), depth) end) + |> Enum.map(fn [{at, len}] -> {l, masked, number, at, len} end) + else + [] + end + end) + |> List.last() + end + + # Inside a flow collection a node also starts directly after the colon of + # a quoted key, JSON-style (`{"BAD":&bad "a/b@x"}`). + defp wf_node_start?(before, depth) do + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or + ((depth > 0 or String.contains?(before, ["[", "{"])) and + Regex.match?(~r/(?:[\[{,]|["']:)\s*(?:![^\s]*\s+)*$/, before)) + end + + # The line of each direct child of the block mapping whose key sits at + # `key_indent`, `rest` being the lines after the key's line `number`, as + # `%{key => line}` — the children `wf_block_children/2` reads, the first + # of a key spelled twice. + defp wf_block_child_lines(rest, key_indent, number) do + block = + rest + |> Enum.with_index(number + 1) + |> Enum.take_while(fn {l, _} -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent + end) + |> Enum.filter(fn {l, _} -> wf026_plain?(l) end) + + case block do + [] -> + %{} + + [{first, _} | _] -> + child_indent = wf026_indent(first) + + block + |> Enum.filter(fn {l, _} -> wf026_indent(l) == child_indent end) + |> Enum.reduce(%{}, fn {l, n}, acc -> + case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:/, l) do + [_, k] -> Map.put_new(acc, k, n) + nil -> acc + end + end) + end + end + + # The direct children of a block mapping whose key sits at `key_indent`, + # as `{key, value}` entries: the plain lines deeper than the key, at the + # first child's indentation. Deeper lines are nested mappings or scalar + # text, not entries; under a child with no value on its own line they are + # that value (`toolchain:` then `nightly` indented is `toolchain: nightly`). + defp wf_block_children(rest, key_indent) do + children = + rest + |> Enum.take_while(fn l -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent + end) + |> Enum.filter(&wf026_plain?/1) + + case children do + [] -> + [] + + [first | _] -> + child_indent = wf026_indent(first) + + children + |> Enum.with_index() + |> Enum.filter(fn {l, _} -> wf026_indent(l) == child_indent end) + |> Enum.flat_map(fn {l, i} -> + case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:(.*)$/, l) do + [_, k, v] -> + v = v |> wf_strip_comment() |> String.trim() + + # A value on the lines below — nothing on the key's line, or a + # block scalar's indicator (`toolchain: >-`) — is what those + # lines hold: an empty block scalar is the empty string. + # So is one with nothing on the line but node properties + # (`toolchain: &channel`), kept in front of what is below. + props = Regex.run(~r/^(?:[&!]\S*(?:\s+|$))*/, v) |> hd() + continued? = v == "" or Regex.match?(~r/^(?:[&!]\S*\s+)*[|>][-+0-9]*$/, v) + + cond do + continued? -> [{k, wf_continued_value(children, i, child_indent)}] + props == v -> [{k, props <> " " <> wf_continued_value(children, i, child_indent)}] + true -> [{k, v}] + end + + nil -> + [] + end + end) + end + end + + # The value of the child at `i` continued on the lines below it, deeper + # than its key and up to the next child; "" when there are none. + defp wf_continued_value(children, i, child_indent) do + children + |> Enum.drop(i + 1) + |> Enum.take_while(&(wf026_indent(&1) > child_indent)) + |> Enum.map_join(" ", &(&1 |> wf_strip_comment() |> String.trim())) + end + + # ─── flow-mapping reading (quote- and depth-aware) ────────────────── + + # Byte column of the innermost `{` still open at byte `at` of `line`, read + # from the start with quotes and comments respected; :error when `at` is + # not inside a flow mapping opened on this line (it is inside a quoted + # scalar, or no brace precedes it). + defp wf_flow_opener_before(line, at) do + line + |> binary_part(0, at) + |> :binary.bin_to_list() + |> wf_flow_prefix(0, nil, [], ?\s) + end + + # State: nil (plain text), ?" or ?' (inside that quote), :comment. The + # stack holds the byte column of each open `{` (`:seq` for a `[`); the + # innermost must be a mapping for `at` to be a key position. + # Every `{`/`[` still open before byte `at` of `line`, innermost first: + # the column of each `{`, `{:seq, column}` for a `[`. Quoted scalars and + # comments are skipped, as in wf_flow_opener_before/2; :error inside one. + defp wf_flow_opener_stack(line, at) do + line + |> binary_part(0, at) + |> :binary.bin_to_list() + |> wf_flow_stack(0, nil, [], ?\s) + end + + defp wf_flow_stack([], _pos, nil, stack, _prev), do: {:ok, stack} + defp wf_flow_stack([], _pos, _state, _stack, _prev), do: :error + + defp wf_flow_stack([c | rest], pos, nil, stack, prev) do + cond do + c == ?# and prev in [?\s, ?\t] -> wf_flow_stack(rest, pos + 1, :comment, stack, c) + c in [?", ?'] -> wf_flow_stack(rest, pos + 1, c, stack, c) + c == ?{ -> wf_flow_stack(rest, pos + 1, nil, [pos | stack], c) + c == ?[ -> wf_flow_stack(rest, pos + 1, nil, [{:seq, pos} | stack], c) + c in [?}, ?]] -> wf_flow_stack(rest, pos + 1, nil, Enum.drop(stack, 1), c) + true -> wf_flow_stack(rest, pos + 1, nil, stack, c) + end + end + + defp wf_flow_stack([?\\, c | rest], pos, ?", stack, _prev), + do: wf_flow_stack(rest, pos + 2, ?", stack, c) + + defp wf_flow_stack([q | rest], pos, q, stack, _prev), + do: wf_flow_stack(rest, pos + 1, nil, stack, q) + + defp wf_flow_stack([c | rest], pos, state, stack, _prev), + do: wf_flow_stack(rest, pos + 1, state, stack, c) + + defp wf_flow_prefix([], _pos, nil, [col | _], _prev) when is_integer(col), do: {:ok, col} + defp wf_flow_prefix([], _pos, _state, _stack, _prev), do: :error + + defp wf_flow_prefix([c | rest], pos, nil, stack, prev) do + cond do + c == ?# and prev in [?\s, ?\t] -> wf_flow_prefix(rest, pos + 1, :comment, stack, c) + c in [?", ?'] -> wf_flow_prefix(rest, pos + 1, c, stack, c) + c == ?{ -> wf_flow_prefix(rest, pos + 1, nil, [pos | stack], c) + c == ?[ -> wf_flow_prefix(rest, pos + 1, nil, [:seq | stack], c) + c in [?}, ?]] -> wf_flow_prefix(rest, pos + 1, nil, Enum.drop(stack, 1), c) + true -> wf_flow_prefix(rest, pos + 1, nil, stack, c) + end + end + + defp wf_flow_prefix([?\\, c | rest], pos, ?", stack, _prev), + do: wf_flow_prefix(rest, pos + 2, ?", stack, c) + + defp wf_flow_prefix([q | rest], pos, q, stack, _prev), + do: wf_flow_prefix(rest, pos + 1, nil, stack, q) + + defp wf_flow_prefix([c | rest], pos, state, stack, _prev), + do: wf_flow_prefix(rest, pos + 1, state, stack, c) + + # The interior of the flow collection whose opener sits at byte `open_col` + # of line `number` (1-based): the text between that brace and the one + # closing it, which may be lines later. Quoted scalars and comments are + # skipped for depth, comments are dropped from the text. :error when the + # document ends first. + defp wf_flow_interior(lines, number, open_col) do + first = Enum.at(lines, number - 1) + head = binary_part(first, open_col + 1, byte_size(first) - open_col - 1) + + text = + [head | Enum.drop(lines, number)] + |> Enum.join("\n") + |> :binary.bin_to_list() + + wf_flow_take(text, 1, nil, [], 0) + end + + defp wf_flow_take([], _depth, _state, _acc, _nl), do: :error + + defp wf_flow_take([c | rest], depth, nil, acc, nl) do + cond do + c == ?# and (acc == [] or hd(acc) in [?\s, ?\t, ?\n]) -> + wf_flow_take(rest, depth, :comment, acc, nl) + + c in [?", ?'] -> + wf_flow_take(rest, depth, c, [c | acc], nl) + + c in [?{, ?[] -> + wf_flow_take(rest, depth + 1, nil, [c | acc], nl) + + c in [?}, ?]] and depth == 1 -> + {:ok, acc |> Enum.reverse() |> List.to_string(), nl} + + c in [?}, ?]] -> + wf_flow_take(rest, depth - 1, nil, [c | acc], nl) + + c == ?\n -> + wf_flow_take(rest, depth, nil, [c | acc], nl + 1) + + true -> + wf_flow_take(rest, depth, nil, [c | acc], nl) + end + end + + defp wf_flow_take([?\\, c | rest], depth, ?", acc, nl), + do: wf_flow_take(rest, depth, ?", [c, ?\\ | acc], nl) + + defp wf_flow_take([?\n | rest], depth, :comment, acc, nl), + do: wf_flow_take(rest, depth, nil, [?\n | acc], nl + 1) + + defp wf_flow_take([_ | rest], depth, :comment, acc, nl), + do: wf_flow_take(rest, depth, :comment, acc, nl) + + defp wf_flow_take([q | rest], depth, q, acc, nl), + do: wf_flow_take(rest, depth, nil, [q | acc], nl) + + defp wf_flow_take([c | rest], depth, q, acc, nl), + do: wf_flow_take(rest, depth, q, [c | acc], nl) + + # The top-level `{key, value}` entries of a flow mapping's interior: + # commas inside nested `{}`/`[]` or quoted scalars do not split, and the + # key ends at the first top-level colon. + defp wf_flow_entries(interior) do + interior + |> wf_flow_split(?,) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> Enum.map(fn entry -> + case wf_flow_split(entry, ?:) do + [key] -> {wf_unquote(key), ""} + [key | value] -> {wf_unquote(key), value |> Enum.join(":") |> String.trim()} + end + end) + end + + # Split at `sep` where it sits at depth 0 and outside quotes. + defp wf_flow_split(text, sep) do + text + |> :binary.bin_to_list() + |> wf_flow_split(sep, 0, nil, [], []) + end + + defp wf_flow_split([], _sep, _depth, _q, cur, out), + do: Enum.reverse([cur |> Enum.reverse() |> List.to_string() | out]) + + defp wf_flow_split([c | rest], sep, depth, nil, cur, out) do + cond do + c == sep and depth == 0 -> + wf_flow_split(rest, sep, depth, nil, [], [cur |> Enum.reverse() |> List.to_string() | out]) + + c in [?", ?'] -> + wf_flow_split(rest, sep, depth, c, [c | cur], out) + + c in [?{, ?[] -> + wf_flow_split(rest, sep, depth + 1, nil, [c | cur], out) + + c in [?}, ?]] -> + wf_flow_split(rest, sep, max(depth - 1, 0), nil, [c | cur], out) + + true -> + wf_flow_split(rest, sep, depth, nil, [c | cur], out) + end + end + + defp wf_flow_split([?\\, c | rest], sep, depth, ?", cur, out), + do: wf_flow_split(rest, sep, depth, ?", [c, ?\\ | cur], out) + + defp wf_flow_split([q | rest], sep, depth, q, cur, out), + do: wf_flow_split(rest, sep, depth, nil, [q | cur], out) + + defp wf_flow_split([c | rest], sep, depth, q, cur, out), + do: wf_flow_split(rest, sep, depth, q, [c | cur], out) + + # Drop a trailing ` # comment` that sits outside quotes. + # The byte offset in a flow mapping's interior at which the value of its + # top-level `key` entry starts, or nil. Entries are split at commas outside + # quotes and nested brackets. + defp wf_flow_entry_value_at(interior, key) do + re = ~r/^\s*["']?#{Regex.escape(key)}["']?\s*:\s*/ + + interior + |> wf_flow_entry_starts() + |> Enum.find_value(fn at -> + case Regex.run(re, binary_part(interior, at, byte_size(interior) - at), return: :index) do + [{0, len}] -> at + len + _ -> nil + end + end) + end + + # Where each top-level entry of a flow interior starts: byte 0, and past + # every comma outside quotes and nested brackets. Inside double quotes a + # `\` escapes the byte after it (`"say \"hi\", then"` is one scalar); a + # doubled `''` inside single quotes closes and reopens with nothing + # between, so it hides no comma either. + defp wf_flow_entry_starts(interior), + do: interior |> :binary.bin_to_list() |> wf_entry_starts(0, 0, nil, [0]) |> Enum.reverse() + + defp wf_entry_starts([], _i, _d, _q, acc), do: acc + + defp wf_entry_starts([?\\, _ | rest], i, d, ?", acc), + do: wf_entry_starts(rest, i + 2, d, ?", acc) + + defp wf_entry_starts([q | rest], i, d, q, acc), do: wf_entry_starts(rest, i + 1, d, nil, acc) + + defp wf_entry_starts([_ | rest], i, d, q, acc) when q != nil, + do: wf_entry_starts(rest, i + 1, d, q, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?", ?'], + do: wf_entry_starts(rest, i + 1, d, c, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?{, ?[], + do: wf_entry_starts(rest, i + 1, d + 1, nil, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?}, ?]], + do: wf_entry_starts(rest, i + 1, d - 1, nil, acc) + + defp wf_entry_starts([?, | rest], i, 0, nil, acc), + do: wf_entry_starts(rest, i + 1, 0, nil, [i + 1 | acc]) + + defp wf_entry_starts([_ | rest], i, d, nil, acc), do: wf_entry_starts(rest, i + 1, d, nil, acc) + + # Offset `at` of a flow interior that `wf_flow_interior/3` read from the + # `{` at `open_col` of line `number`, as `{line, column}` in the file. + defp wf_interior_pos(interior, at, number, open_col) do + case :binary.matches(binary_part(interior, 0, at), "\n") do + [] -> + {number, open_col + 1 + at} + + nls -> + {nl, _} = List.last(nls) + {number + length(nls), at - nl - 1} + end + end + + # Does the line's own `uses:` value carry YAML node properties — an anchor + # or tag before the reference (`uses: &rust a/b@x`, `uses: !!str a/b@x`)? + defp wf_node_properties?(line), + do: Regex.match?(~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*[&!]/, line) + + # The `uses:` keys whose reference is not written on their own line, as + # `%{key_line => {joined, how}}`: the key's line with the reference in + # place, which the rules read as they read a one-line key. `how` is + # :continued for a reference YAML continues below the key (`- uses:`, then + # — past any blank or comment-only lines — the reference on a deeper line; + # or a block scalar, `- uses: >-`, whose one content line is the reference) + # and :alias for `uses: *name`, the reference the latest `&name` before it + # anchors (`PinSites.alias_sites/1`). Found on the masked lines + # (`PinSites.mask_lines/1`), so a `uses:` in a `run: |` script or a quoted + # string stands for nothing. + defp wf_continued_uses(lines) do + content = Enum.join(lines, "\n") + masked = Hypatia.Rules.PinSites.mask_lines(content) + prefixes = Hypatia.Rules.PinSites.flow_prefixes(content) + + continued = + for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do + # A block scalar's indicator (`>-`, `|`) is no part of the value. + head = + lines + |> Enum.at(key - 1) + |> Hypatia.Rules.PinSites.head() + |> String.trim_trailing() + |> String.replace(~r/\s+[|>][-+0-9]*$/, "") + + {key, {head <> " " <> site.action <> "@" <> site.ref, :continued}} + end + + for %{line: n, at: at, len: len, site: site} <- + Hypatia.Rules.PinSites.alias_sites(masked, prefixes), + not Map.has_key?(continued, n), + into: continued do + line = Enum.at(lines, n - 1) + tail = binary_part(line, at + len, byte_size(line) - at - len) + {n, {binary_part(line, 0, at) <> site.action <> "@" <> site.ref <> tail, :alias}} + end + end + + defp wf_strip_comment(text) do + text + |> :binary.bin_to_list() + |> wf_strip_comment(nil, []) + end + + defp wf_strip_comment([], _q, acc), do: acc |> Enum.reverse() |> List.to_string() + + defp wf_strip_comment([?# | _], nil, acc) when acc == [] or hd(acc) in [?\s, ?\t], + do: acc |> Enum.reverse() |> List.to_string() + + defp wf_strip_comment([c | rest], nil, acc) when c in [?", ?'], + do: wf_strip_comment(rest, c, [c | acc]) + + defp wf_strip_comment([?\\, c | rest], ?", acc), do: wf_strip_comment(rest, ?", [c, ?\\ | acc]) + defp wf_strip_comment([q | rest], q, acc), do: wf_strip_comment(rest, nil, [q | acc]) + defp wf_strip_comment([c | rest], q, acc), do: wf_strip_comment(rest, q, [c | acc]) + + # Is line `number` (1-based) inside a quoted scalar that a line from `from` + # up to it opened and did not close? A double- or single-quoted value may + # run over several physical lines (`- run: "echo start` … `echo end"`), and + # a `uses:` written on one of them is string text, not a key. The content + # of a block scalar (`run: |`) is skipped: a quote there is script text and + # opens nothing. Only the lines of the one step or job are read, so a quote + # the scan misreads elsewhere in the file cannot hide a real pin. + defp wf_in_open_quote?(lines, from, number) when from < number do + lines + |> Enum.slice((from - 1)..(number - 2)//1) + |> Enum.reduce({nil, nil, 0}, &wf_quote_line/2) + |> elem(0) + |> Kernel.!=(nil) + end + + defp wf_in_open_quote?(_lines, _from, _number), do: false + + # One line of that scan: `{open_quote | nil, block_scalar_key_indent | nil, + # flow_depth}`. The depth of the flow collections still open is carried from + # line to line, as the shared pin reader carries it: a `{"run":"echo` whose + # `{` (or `steps: [`) opened on an earlier line is still inside one. + defp wf_quote_line(line, {nil, block, depth}) when is_integer(block) do + if String.trim(line) == "" or wf026_indent(line) > block, + do: {nil, block, depth}, + else: wf_quote_line(line, {nil, nil, depth}) + end + + defp wf_quote_line(line, {q, _, depth}) do + case wf_quote_scan(line, q, "", depth) do + {nil, outside} -> + depth = max(depth + wf_bracket_delta(outside), 0) + + if Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, outside), + do: {nil, wf026_key_indent(line), depth}, + else: {nil, nil, depth} + + {open, outside} -> + {open, nil, max(depth + wf_bracket_delta(outside), 0)} + end + end + + # Walk one line from quote state `q` (nil outside a quoted scalar), keeping + # the text outside quotes; `depth` is the flow depth the line starts at. A + # quote opens a scalar only where a scalar can start (wf_scalar_start?/2); + # `echo it's` is a plain scalar. A comment ends the line. + defp wf_quote_scan(<<>>, q, out, _depth), do: {q, out} + + defp wf_quote_scan(<>, ?", out, depth), + do: wf_quote_scan(rest, ?", out, depth) + + defp wf_quote_scan(<>, ?', out, depth), + do: wf_quote_scan(rest, ?', out, depth) + + defp wf_quote_scan(<>, q, out, depth) when c == q, + do: wf_quote_scan(rest, nil, out <> "x", depth) + + defp wf_quote_scan(<<_, rest::binary>>, q, out, depth) when q != nil, + do: wf_quote_scan(rest, q, out, depth) + + defp wf_quote_scan(<>, nil, out, _depth) + when out == "" or binary_part(out, byte_size(out) - 1, 1) in [" ", "\t"], + do: {nil, out} + + defp wf_quote_scan(<>, nil, out, depth) when c in [?", ?'] do + if wf_scalar_start?(out, depth), + do: wf_quote_scan(rest, c, out <> "x", depth), + else: wf_quote_scan(rest, nil, out <> <>, depth) + end + + defp wf_quote_scan(<>, nil, out, depth), + do: wf_quote_scan(rest, nil, out <> <>, depth) + + # Can a scalar start after `out` (the line so far, quote bodies dropped)? + # After `: `, a list dash, `[`, `{` or `,`, or at the line's start; inside + # a flow collection, opened on this line or one before it (`depth`), also + # directly after `:` (JSON-style `{"run":"echo`); in each case past any + # anchor or tag (`run: &cmd "echo`), as Hypatia.Rules.PinSites reads a file. + defp wf_scalar_start?(out, depth) do + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)(?:[&!]\S*\s+)*$/, out) or + (depth + wf_bracket_delta(out) > 0 and Regex.match?(~r/:\s*(?:[&!]\S*\s+)*$/, out)) + end + + # Flow brackets opened minus closed in text with its quote bodies dropped. + defp wf_bracket_delta(out), + do: length(Regex.scan(~r/[\[{]/, out)) - length(Regex.scan(~r/[\]}]/, out)) + + # The nearest line above `number` that is shallower than it: the key whose + # value holds it (a job's key line for a job-level `uses:`). + defp wf_parent_line(lines, number) do + indent = wf026_indent(Enum.at(lines, number - 1)) + + (number - 1)..1//-1 + |> Enum.find(number, fn n -> + l = Enum.at(lines, n - 1) + wf026_plain?(l) and wf026_indent(l) < indent + end) + end + + # The line with its comment dropped and every quoted scalar's body blanked + # to `x` byte for byte, so `&name` or `{` written inside a string is not + # structure, and an offset into the result is an offset into the line. + defp wf_mask_quoted(text), do: wf_mask_quoted(text, nil, "") + + defp wf_mask_quoted(<<>>, _q, out), do: out + + defp wf_mask_quoted(<>, nil, out) + when out == "" or binary_part(out, byte_size(out) - 1, 1) in [" ", "\t"], + do: out + + defp wf_mask_quoted(<>, nil, out) when c in [?", ?'], + do: wf_mask_quoted(rest, c, out <> <>) + + defp wf_mask_quoted(<>, nil, out), do: wf_mask_quoted(rest, nil, out <> <>) + + defp wf_mask_quoted(<>, ?", out), + do: wf_mask_quoted(rest, ?", out <> "xx") + + defp wf_mask_quoted(<>, q, out), do: wf_mask_quoted(rest, nil, out <> <>) + defp wf_mask_quoted(<<_, rest::binary>>, q, out), do: wf_mask_quoted(rest, q, out <> "x") + + defp wf_unquote(text) do + text + |> String.trim() + |> String.replace(~r/^"(.*)"$/s, "\\1") + |> String.replace(~r/^'(.*)'$/s, "\\1") + |> String.trim() + end + + # Column of a key, with a list dash counted as indentation (`- with:` is + # the column of `with`). + defp wf026_key_indent(line) do + line + |> String.replace(~r/^(\s*)-\s/, "\\1 ") + |> wf026_indent() + end + + # ─── WF027: caller pinned at a reusable revision the estate has measured dead ── + + # One entry per (reusable, window). The SHAs are the commits of + # hyperpolymath/standards main whose rust-ci-reusable.yml pins + # dtolnay/rust-toolchain by SHA and passes no `toolchain:` input — a walk of + # every first-parent commit from d73c93e0 (2026-06-01) to 49ed4715 + # (2026-06-24), each one's file content checked, not the dates. The + # replacement is the current standards main whose own actions.lock GitHub + # accepts (see pr-automerge-policy.json PIN-003 for the lock-side window, + # which kills EVERY standards reusable and so lives in the pin denylist + # rather than here). gitbot-fleet scripts/lib/stale-reusable-pins.tsv carries + # the same rows (Window A) for the repair; regenerate both together. + @broken_reusable_revisions [ + %{ + reusable: "hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml", + window: "standards main 2026-06-01..2026-06-24 (d73c93e0..49ed4715)", + fixed_by: "standards#439 (9fe0d8ffa33a6fa0c6ba64117bef9981dacd3020)", + replacement: "75bd5994470661a099df562d6d5ad2fe744dbd66", + reason: + "rust-ci-reusable.yml at this revision pins dtolnay/rust-toolchain by SHA and " <> + "passes no toolchain: input, so every caller fails in its first step with " <> + "\"'toolchain' is a required input\" before any cargo command runs.", + shas: [ + "bf32531befa4bb7f692a5f7ecac1eab71af20506", + "583a9346ebf28e0fd8114f22aef140eaa1be4ac0", + "d7c22711e830e1f383846472f6e9b99debdb201e", + "747b2f0f690505b24c90991a030a01e564ff59c6", + "509142da4ec04d74dccfc9e52813aeabc2c6fa02", + "e2ef79eecdd623ff631fcbf3fe0268ba4f4ce166", + "0597948e81c0be7570f805d7ec49f64fa0eeefd6", + "da1ee5ebaec4de70e7530a9fcb4b2c693434bcc1", + "0f49246bc5055f7159c92d649c4d3fb97b75cf66", + "f15ced3f44146de9931bcb70157e1825d07b83ed", + "5e29ef7923aff017419bdce9354e9df4c3823cb5", + "ad8c45387878f2b00217330b0e1260edc6f878b5", + "3ce22c244bd4b31ed99397a585cd458735a79819", + "8c785d41adbbee534b158747a5c6796402e450f0", + "c1070339c80c4f1af71bb35cacc58c19f611f3a2", + "49ed471515bdf587aeb9931af2d1314cd6aa5de2", + "7282b1ac828d89eaecd3403b59591975746ec3ff", + "d135b05bfc647d0c0fbfedc7e80f37ea50f49236", + "4762ba66441de05bccb36e074ba3d2202219126a", + "59429dc666d347d030114cd6ef6f4ff7b7f6a6b4", + "e8fe153b568af5e3534a261eee0258fa84fa6c5e", + "280cb7e3f2db8b0e5f26c42877ea5fa80243b06f", + "c6c095b38f7e729d3d355629b80e771dfe1a351b", + "09e7023d24682621bea4e11965a1ef5e87d86c3b", + "42455b19727fa0ccf73a9b914f0e9781d4c34577", + "4ddc926b4b86451c2711a16bf927ed16ad2db45e", + "0809b8e3630d38229b6c3629956e0dff4b937c33", + "d72fe5a14e841ac6d78514b53624b6173038ee20", + "5a93d9d57cc04de4002d6d0ecd336fc7a8698910", + "3e57141d38672438a7a9110bd10ea0f76e6099a7", + "78b29005efe954822c86c553b40523b9fdae78d4", + "f290d4a696b8af78f9141daeced486b7b8a85d11", + "a8c69ecfcd87d5418dd4559019ac860fff20e834", + "a04c1d9f32e3c6cb5e91b7ba220080475449a4d2", + "c43aafaa2d707f3f195cad9742d3ccc8d068e3db", + "a955527810b82dbf7a9bc541502c5235f119499d", + "2ff85f8f84212ae2cbedb1ea232c6f5a470acef3", + "8e6ba7d4a0890d25425855a77404d4690d0ab4b5", + "426368b72949d8235c8cf47f8bdc5e0e5cd5b54f", + "025bba3f35f5040bf76d4954f02c744d61f68e8f", + "8466cee8adbb814a2507a9a33e36ba415c5d7eba", + "f5f0506a6ec88e574753eee701a268e0d4b3a7f2", + "ae5b6731e0c1ce192edc14737a226d6b201341aa", + "742424ec9cffeb414174276d9321325859e3aa64", + "19995ace5f1179e9e2d4783fe1d36df0f343492d", + "524523c1a6057942aadaef5999912fbf13a355c5", + "e4c7d968fa9aad6fcecab23bd60d1025679d1ad9", + "e96096c6614469e29e02edcc923fdacc0e611b34", + "ca29b1950ac493e4a29db3b61b6991fbdc8a57b2", + "e59affd85741ac786867a667bc1526ade31b1dd4", + "7694d58cd4c82d6d706ef9653bcf545e28126794", + "33a82ee5e3635bd137b2bc8ef2b1d71367195546", + "52db79bee40380820e01c454efe6e85ae6115b2c", + "6022e498a06552d71e51662283b9453855185b94", + "c97b2bdd12aeaff4624efcaec874ea7665dadae6", + "9726a3814c895bf531f22c2b82cc15a3b1efa1f3", + "bec61613c821fdb1035f18b88e6e5d19d23328f7", + "4236493691a06a6393b5ec4480f38e5d804f5b69", + "6cd3772824e59c8c9affeab66061e25383544242", + "e763d82b93f46e7201d95e6d817ac61d739a8cc1", + "aa665acc04fff66632ede8368dbc9c2b5a630d3c", + "3a5c23cb5888eca79df090fde6d52a8dc4820285", + "0b96f61a3f043bd5409b913e526b29f3fd38a217", + "ae6619ecddd0eff0f957c73280efc36bb9302d75", + "3b3549e293f87be99d7dbb939d380c8128842c8e", + "2b88f34c97a9478b7a96b07425bb37ee5822bb8e", + "9bdaff9479d53328ed907bdd5ba885398800fb00", + "142dd7a0b9cd6101de8c4279c55b7051c537f76b", + "ac6b3252e7170d9459779c03bf31ae04c3629a61", + "f2b9d5eb5fb68a60b30468b8289a0f8cefbdd558", + "e401e47106ce0b0e32e3d0e2b7d910410e5b165e", + "e3ad4ca266e4dc93b7cebce5e3b3b48c6d523e45", + "74748546395bf3dcedd5b660f58feb36be9d53d6", + "fc7abf589b93f9a33b8af74dbfe98ba22707156f", + "7814dc67a489afc39f0289af5d3b6c3e23645156", + "c98915a225c98f2b90980cf045a00eb603e6af0f", + "18dd92c19d5b9e50740826a4be8301afbab91bae", + "82a8ac8e2fd446b710139a759bf22f175dc5e299", + "55da6b331922f0ff7389c783d363ae12d8c61068", + "1376eb6251fa9beeaea75241739b1d00dd6b4028", + "28946811fb7fe2d05e9ac11c9ebee7295e966d40", + "d7c3b8f75d70ab640e6f37a4c8f9e6e8f2b6df56", + "d73c93e0e7d38f673b68e616d51a62f5860081fd" + ] + } + ] + + @doc "The (reusable, window) table WF027 fires on." + def broken_reusable_revisions, do: @broken_reusable_revisions + + @doc """ + WF027: a caller pins an estate reusable workflow at a revision the estate + has MEASURED dead for that reusable. + + Unlike WF016 (a SHA no branch reaches any more), these commits are + reachable and resolve: the run starts, the reusable's first step fails, + and the failure names something the caller never wrote. The table is + data, keyed by reusable path, so a newly measured window is a row, not a + rule. Full 40-hex pins only — that is the only form sha_pinning_required + lets a caller write. + + A job's `uses:` whose value YAML continues below its key (past any blank + or comment-only lines, or as a block scalar) is the call all the same. So + is the `uses:` of a mapping a job is given whole as an alias (`rust: + *job`), wherever that mapping is anchored (`PinSites.node_alias_sites/1`). + + Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites + the pin to the replacement the table names, never to "newest". It refuses + a value continued below its key (exit 2, file untouched), so that finding + carries `auto_fixable: false` and routes to review, as does a call made + through a job alias whose mapping is no job's own: the fixer rewrites the + keys of `jobs:` only. + + Severity is low and the finding is `advisory: true`: like WF026 the rule + reports and routes to the fleet; it is never a blocking gate (AGENTIC + `must-not-promote-to-blocking`). + """ + def check_known_broken_reusable_revision(workflow_contents) do + Enum.flat_map(workflow_contents, fn {filename, content} -> + lines = String.split(content, "\n") + continued = wf_continued_uses(lines) + flow = wf_flow_lines(content) + # A job given whole as an alias (`rust: *job`) runs the mapping its + # anchor holds: that mapping's `uses:` line is a call wherever it sits. + aliased = + for %{key_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(content), + into: MapSet.new(), + do: n + + Enum.flat_map(@broken_reusable_revisions, fn rev -> + # Anchored to a `uses:` key at the start of a line (quoted or not) that + # is a key of a job under `jobs:` — the only place a reusable is + # called — or to the same key inside the flow mapping a job opens on + # its own line (`rust: { uses: ... }`). The reusable's path inside a + # `run: |` script, under `env:`, in a string value or in a comment is + # not a call, so the fleet is never sent to rewrite script data. + # The 40 hex characters end the ref (`@-patched` is a tag), in + # either case: git resolves the commit ID case-insensitively. + reusable = wf027_reusable_re(rev.reusable) + + ref_re = + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + + inline_re = + ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + + lines + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + # A job's `uses:` with the reference below it, or naming an anchored + # one (`uses: *call`), is the one-line key it stands for. + {line, resolved} = + case Map.fetch(continued, number) do + {:ok, {joined, how}} -> {joined, how} + :error -> {line, nil} + end + + live? = number in aliased + + # Each call with the layout the fleet fixer would meet: it rewrites + # a bare reference on a block key's own line, nothing else. A call + # only a job alias makes (its mapping anchored outside `jobs:`) is + # no key the fixer places. + shas = + case Regex.run(ref_re, line) do + [_full, sha] -> + review = + cond do + resolved != nil -> resolved + wf_node_properties?(line) -> :properties + true -> nil + end + + case wf027_job_key(lines, flow, number) do + :block -> [{sha, review}] + :flow -> [{sha, review || :flow}] + nil when live? -> [{sha, review || :node_alias}] + nil -> [] + end + + nil -> + inline_re + |> Regex.scan(line, return: :index) + |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> + cond do + wf027_inline_job?(lines, flow, line, number, at + 1) -> [:flow] + live? -> [:node_alias] + true -> [] + end + |> Enum.map(&{binary_part(line, sha_at, 40), &1}) + end) + end + + Enum.flat_map(shas, fn {sha, review} -> + if String.downcase(sha) in rev.shas do + [wf027_finding(filename, number, sha, rev, review)] + else + [] + end + end) + end) + end) + end) + end + + # Is the `uses:` that starts line `number` a key of a job under `jobs:`? + # :block when its indentation says so, outside any quoted scalar the job + # opened above it — the reading the fleet fixer makes. A line inside a flow + # collection opened on an earlier line is judged by the brackets still open + # before it (`wf_flow_context/4`), however it is indented: the innermost + # must be the mapping a job key opens in `jobs`. Such a call is :block too + # when its indentation agrees (KYAML-style `rust: {` / ` uses: ...`, which + # the fixer rewrites), and :flow when only the brackets say so (`jobs: {`, + # `rust: {`, `uses: ...` all in column 0, which the fixer cannot place). + # nil when it is no job's key. + defp wf027_job_key(lines, flow, number) do + block? = + match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and + not wf_in_open_quote?(lines, wf_parent_line(lines, number), number) + + cond do + not wf_in_flow?(flow, number) -> if block?, do: :block + not wf027_flow_job?(wf_flow_context(lines, flow, number)) -> nil + block? -> :block + true -> :flow + end + end + + # Is the innermost bracket of a `wf_flow_context/4` the mapping a job key + # opens in `jobs`? + defp wf027_flow_job?({:ok, [job, "jobs" | _], [:map | _], _}) when is_binary(job), do: true + defp wf027_flow_job?(_), do: false + + # The reusable's identity as a regex source. GitHub resolves the owner and + # the repository name in either case (`Hyperpolymath/Standards` is + # `hyperpolymath/standards`); the workflow path inside the repository is a + # file path and is matched as written. The fleet fixer reads the pin + # case-insensitively and rewrites it as the caller spells it. + defp wf027_reusable_re(reusable) do + case String.split(reusable, "/", parts: 3) do + [owner, repo, path] -> "(?i:#{Regex.escape(owner <> "/" <> repo)})/#{Regex.escape(path)}" + _ -> Regex.escape(reusable) + end + end + + # Is the `uses` at `uses_at` the entry of a flow mapping a job key opens? + # The job's own line under `jobs:` (`rust: { uses: ... }`), the whole + # `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`), or + # the same inside a root `{ ... }`: the keys that open the brackets still + # unclosed before `uses` (nearest first), then the line's block ancestors, + # must read a job key, then `jobs`. Each key is the last entry before its + # brace, split quote-aware, so a sibling's `name: "CI, build"` separates + # nothing. + defp wf027_inline_job?(lines, flow, line, number, uses_at) do + # On a line inside a flow collection opened above, the brackets before + # `uses` are read from where the collection began. + if wf_in_flow?(flow, number), + do: wf027_flow_job?(wf_flow_context(lines, flow, number, uses_at)), + else: wf027_line_job?(lines, line, number, uses_at) + end + + defp wf027_line_job?(lines, line, number, uses_at) do + case {wf_flow_opener_stack(line, uses_at), wf_ancestor_keys(lines, number)} do + {{:ok, [open | _] = stack}, ancestors} when is_integer(open) and is_list(ancestors) -> + keys = wf_flow_opener_keys(line, stack) ++ ancestors + match?([job, "jobs" | _] when is_binary(job), keys) + + _ -> + false + end + end + + # The key whose value is the flow mapping opening at byte `open`: the last + # top-level entry of `line[from, open)` must read `key:` and nothing else. + defp wf027_key_before(line, from, open) do + # Node properties before the bracket name the collection, not its key: + # `rust: &job {` is the job `rust`, `[ &step {` an anonymous step. + entry = + line + |> binary_part(from, open - from) + |> wf_flow_split(?,) + |> List.last("") + |> String.trim() + |> String.replace(~r/(?:^|\s+)(?:[&!]\S*)(?:\s+[&!]\S*)*$/, "") + |> String.trim() + + case Regex.run(~r/^["']?([\w.-]+)["']?\s*:$/, entry) do + [_, key] -> key + _ -> nil + end + end + + # The fleet fixer refuses (exit 2, file untouched) a value continued below + # its key, one carrying an anchor or tag, and a flow-style job mapping: a + # finding on any of them (`review`) routes to review. + defp wf027_finding(filename, number, sha, rev, review) do + %{ + rule: "WF027", + type: :known_broken_reusable_revision, + file: filename, + line: number, + reusable: rev.reusable, + current_sha: sha, + replacement_sha: rev.replacement, + # Advisory, as WF026: reported and routed, never a blocking finding. + severity: :low, + advisory: true, + auto_fixable: review == nil, + fix_script: if(review == nil, do: "fix-stale-reusable-pin.sh", else: nil), + reason: + "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> + rev.reason <> + " Fixed by #{rev.fixed_by}; move the pin to " <> + "`@#{rev.replacement}`." <> wf027_review_note(review), + action: :rewrite_stale_reusable_pin + } + end + + defp wf027_review_note(nil), do: "" + + defp wf027_review_note(:continued), + do: + " The `uses:` value is continued below its key; the repair script rewrites " <> + "a reference on the key's own line only, so this one routes to review." + + defp wf027_review_note(:alias), + do: + " The `uses:` value is an alias of an anchored reference; the repair script " <> + "does not rewrite anchored values, so this one routes to review." + + defp wf027_review_note(:properties), + do: + " The `uses:` value carries a YAML anchor or tag; the repair script rewrites " <> + "a bare reference only, so this one routes to review." + + defp wf027_review_note(:flow), + do: + " The job is written as a flow mapping; the repair script rewrites block-style " <> + "jobs only, so this one routes to review." + + defp wf027_review_note(:node_alias), + do: + " The call is made by a job given as an alias of a mapping anchored outside " <> + "`jobs:`; the repair script rewrites the keys of `jobs:` only, so this one " <> + "routes to review." + # ─── WF017: Secret-consuming action without secret-presence gate ────── # Actions whose primary input is a secret and that fail outright when diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 727f9178..e3a1b804 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -28,7 +28,10 @@ # * `--via clone` is the authoritative mode (reads the files themselves); # `--via search` is the fast reconnaissance mode and can lag the index. # -# EXIT CODES: 0 clean, 1 error, 2 findings present, 3 nothing to do +# EXIT CODES: 0 clean, 1 error, 2 confirmed findings or an INCOMPLETE run +# (failed/partial searches, listings or clones, unread candidates, +# or limits imposed by --max-repos or --sample), +# 3 nothing to do # # usage: estate-pin-integrity.sh [--org ORG]... [--policy FILE] [--out DIR] # [--via search|clone] [--rewrite] @@ -44,6 +47,7 @@ ORGS=() VIA="search" REWRITE=0 MAX_REPOS=0 +SAMPLE=0 while [ $# -gt 0 ]; do case "$1" in @@ -53,6 +57,7 @@ while [ $# -gt 0 ]; do --via) VIA="$2"; shift 2 ;; --rewrite) REWRITE=1; shift ;; --max-repos) MAX_REPOS="$2"; shift 2 ;; + --sample) SAMPLE="$2"; shift 2 ;; -h|--help) sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 1 ;; esac @@ -82,6 +87,21 @@ list_repos() { --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | .name' 2>/dev/null } +# Code search names an owner by its kind: `org:` for an organisation and +# `user:` for a personal account (the default `hyperpolymath` is one); `org:` +# on a personal account matches nothing at all, which read as a clean org. +# The kind is asked of GitHub; an owner whose kind cannot be read is a failed +# search, never an empty one. +owner_qualifier() { + local kind + kind=$(gh api "users/$1" --jq '.type' 2>/dev/null) || return 1 + case "$kind" in + Organization) printf 'org' ;; + User) printf 'user' ;; + *) return 1 ;; + esac +} + # ─── Policy extraction ─────────────────────────────────────────────────── mapfile -t DENY_SHAS < <(jq -r '.pin_denylist[] | .blocked_shas[]?' "$POLICY") mapfile -t DENY_VERSIONS < <(jq -r '.pin_denylist[] | .blocked_versions[]?' "$POLICY") @@ -98,12 +118,265 @@ REF_ALTERNATION=$( } | sed 's/\./\\./g' | paste -sd'|' - ) +# What may follow a ref token: a closing quote (`uses: "…@sha"`), a flow +# delimiter (`{ uses: …@sha }`, `…@sha,`), whitespace, a comment, or the end +# of the line. A pin written in any of these forms is the same pin. +REF_END='([]"'"'"'[:space:]#,}]|$)' +# The `uses` key, quoted or bare, with the quote a quoted value opens with: +# `uses: x@…`, `"uses": "x@…"`, `{ uses: x@… }` all name an action. +USES_KEY='["'"'"']?uses["'"'"']?[[:space:]]*:[[:space:]]*["'"'"']?' +REF_END_CLASS='[]"'"'"'[:space:]#,}]' +# One `uses:` site: the action and a denylisted token, bounded. Matched in +# either case (-i): a commit ID in upper-case hex is the same object. +SITE_RE="${USES_KEY}[^@\"'[:space:],}]+@(${REF_ALTERNATION})(${REF_END})" +SHA_ALTERNATION=$(printf '%s\n' "${DENY_SHAS[@]}" | paste -sd'|' -) + +lc() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } + +# Does the policy deny at ? A blocked version is blocked +# for ITS action only (`some/action@v4.38.1` is no finding when only +# codeql-action's 4.38.1 is denied); a SHA is one object and meets its entry +# in either spelling. The action is compared case-folded too: GitHub resolves +# `GitHub/CodeQL-Action` and `github/codeql-action` to one repository, so a +# mixed-case caller meets the entry its lower-case spelling would. +# Only a 40-hex commit ID is case-folded: git reads an object ID in either +# case, but a ref NAME is case-sensitive, so `@V4.38.1` is a different tag +# from the blocked `v4.38.1` and may point somewhere else. +denylisted_pair() { + jq -e --arg a "$(lc "$1")" --arg r "$2" ' + def norm: if test("^[0-9a-fA-F]{40}$") then ascii_downcase else . end; + ($r | norm) as $r + | [.pin_denylist[] | select((.action | ascii_downcase) == $a) + | (.blocked_shas // []), (.blocked_versions // []), ((.blocked_versions // []) | map("v" + .))] + | add // [] | map(norm) | index($r) != null' "$POLICY" >/dev/null 2>&1 +} + +# The action and ref of one site as `grep -o` printed it: the key and the +# trailing boundary stripped, into site_action and site_ref. +site_parts() { + local t + t=$(printf '%s' "$1" | sed -E "s/^${USES_KEY}//; s/${REF_END_CLASS}\$//") + site_action="${t%@*}"; site_ref="${t##*@}" +} + +# Everything that is not YAML structure, blanked IN PLACE, one output line +# per input line: a quoted scalar that is neither a key nor the value of a +# `uses` key has its text replaced by `x` (its quotes kept), a comment by +# spaces, and the content lines of a block scalar (`run: |`, `script: >-`) +# entirely by `x` after their indentation. A quoted scalar that runs over +# several lines (`- run: "echo start` ... `echo end"`) is carried from record +# to record like a block scalar: each continuation is `x` up to and including +# its closing quote. A quote opens such a scalar only where YAML lets one +# start (as Hypatia.Rules.PinSites.mask_lines/1 reads it), so `echo it's` +# opens nothing. Lengths are kept, so a site found on a masked line sits at +# the same column of the real one. A `uses:` written inside +# `- run: "echo uses: a/b@"`, on a continuation line of a quoted scalar, +# in a `run: |` script or in a comment is therefore no site this sweep +# rewrites; a denied SHA or version there is still seen by line_has_denied_token on +# the raw line, and flagged for the owner. A quote never closed before the +# end of the file keeps its lines masked: whatever they hold is flagged, +# never repaired. +# shellcheck disable=SC2016 # an awk program, not a shell expansion +MASK_AWK=' +function key_col(s, t) { t = s; sub(/[^[:space:]-].*$/, "", t); return length(t) } +function xs(n, t) { t = ""; while (n-- > 0) t = t "x"; return t } +function mask(s, out, n, i, c, j, body) { + out = ""; n = length(s); i = 1 + while (i <= n) { + c = substr(s, i, 1) + if (c == "#" && (i == 1 || substr(s, i - 1, 1) ~ /[[:space:]]/)) { while (i <= n) { out = out " "; i++ }; break } + if (c == "\"" || c == sq) { + j = i + 1 + if (c == "\"") { while (j <= n && substr(s, j, 1) != "\"") { if (substr(s, j, 1) == "\\") j++; j++ } } + else { while (j <= n && substr(s, j, 1) != sq) j++ } + if (j > n) j = n + body = substr(s, i + 1, j - i - 1) + if (substr(s, j + 1) ~ /^[[:space:]]*:/ || tolower(out) ~ USES_TAIL) out = out substr(s, i, j - i + 1) + else out = out c xs(length(body)) (j > i ? substr(s, j, 1) : "") + i = j + 1; continue + } + out = out c; i++ + } + return out +} +# The quote closing a q scalar in s from position j, or 0: `\` escapes inside +# double quotes, and `'"''"'` is a literal quote inside single ones. +function close_pos(s, j, q, n, c) { + n = length(s) + while (j <= n) { + c = substr(s, j, 1) + if (q == "\"" && c == "\\") { j += 2; continue } + if (c == q) { if (q == sq && substr(s, j + 1, 1) == sq) { j += 2; continue }; return j } + j++ + } + return 0 +} +# Can a scalar start after seen? After `key: ` or a list dash (past any +# anchor or tag); inside a flow collection after `[`, `{` or `,`, and +# directly after a `:` (JSON-style `{"run":"echo ...`); and at the start of +# a line when the line above left a value pending (`key:`); each past any +# anchor or tag (`"run": &cmd "echo`). In a plain scalar (`echo it'"'"'s`) a +# quote opens nothing. +function scalar_start(seen) { + if (seen ~ /^[[:space:]]*([&!][^[:space:]]*[[:space:]]+)*$/) return PENDING || DEPTH > 0 + if (seen ~ /(^[[:space:]]*(-[[:space:]]+)+|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) return 1 + if (DEPTH > 0) return seen ~ /[:{,[][[:space:]]*([&!][^[:space:]]*[[:space:]]+)*$/ + return 0 +} +# Walk s from position i outside any quote (seen is what precedes it, quote +# bodies as x): track the flow depth and whether the line leaves a value +# pending, and return the quote a scalar opened on this line and left open, +# or "". +function walk(s, i, seen, n, c, j, t) { + n = length(s) + while (i <= n) { + c = substr(s, i, 1) + if (c == "#" && (seen == "" || seen ~ /[[:space:]]$/)) break + if ((c == "\"" || c == sq) && scalar_start(seen)) { + j = close_pos(s, i + 1, c) + if (j == 0) return c + seen = seen xs(j - i + 1); i = j + 1; continue + } + if (c == "[" || c == "{") { + if (DEPTH > 0 || seen ~ /(^[[:space:]]*(-[[:space:]]+)*|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) DEPTH++ + } else if ((c == "]" || c == "}") && DEPTH > 0) DEPTH-- + seen = seen c; i++ + } + t = seen; sub(/[[:space:]]+$/, "", t) + # A key or dash followed only by node properties (`- run: &cmd`, `key: + # !!str`) leaves its value pending as a bare one does. + if (t !~ /^[[:space:]]*$/) PENDING = (t ~ /(:|^[[:space:]]*-)([[:space:]]+[&!][^[:space:]]*)*$/ || (DEPTH > 0 && t ~ /[{,[]$/)) + return "" +} +BEGIN { USES_TAIL = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*$"; in_scalar = 0; inq = ""; DEPTH = 0; PENDING = 0 } +{ + if (in_scalar) { + if ($0 ~ /^[[:space:]]*$/ || key_col($0) > scalar_col) { t = $0; sub(/[^[:space:]].*$/, "", t); print t xs(length($0) - length(t)); next } + in_scalar = 0 + } + # The continuation of a quoted scalar an earlier line opened is text up to + # and including its closing quote; the rest of the line is masked as a line. + if (inq != "") { + j = close_pos($0, 1, inq) + if (j == 0) { print xs(length($0)); next } + m = xs(j) mask(substr($0, j + 1)) + inq = walk($0, j + 1, xs(j)) + } else { + m = mask($0) + inq = walk($0, 1, "") + } + print m + if (inq == "" && m ~ /:[[:space:]]*[|>][-+0-9]*[[:space:]]*$/) { in_scalar = 1; scalar_col = key_col($0) } +}' + +# The masked form of one file, line for line (block scalars tracked). +mask_file() { awk -v sq="'" "$MASK_AWK" "$1"; } + +# The masked form of one line on its own: used after a site on that line has +# been rewritten, which a line inside a block scalar never is. +mask_line() { printf '%s\n' "$1" | awk -v sq="'" "$MASK_AWK"; } + +# The `uses:` sites of a masked line, as grep -o prints them. +masked_sites() { printf '%s\n' "$1" | grep -oiE "$SITE_RE" || true; } + +# Replace ONE site on a line — the action and ref exactly as written — with +# the known-good SHA for that action; the other sites on the line keep theirs. +# The site is located on the masked line, so the same `action@ref` text +# inside a quoted scalar or a comment earlier on the line is never the one +# replaced. +replace_site() { # + PIN_LINE="$1" PIN_MASKED="$2" PIN_ACTION="$3" PIN_REF="$4" PIN_GOOD="$5" \ + awk -v sq="'" 'BEGIN { + line = ENVIRON["PIN_LINE"]; masked = ENVIRON["PIN_MASKED"] + a = ENVIRON["PIN_ACTION"]; r = ENVIRON["PIN_REF"]; good = ENVIRON["PIN_GOOD"] + tail = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*[\"" sq "]?$" + ends = "]\"" sq " \t#,}" + # The action is found case-folded; the ref only when it is a 40-hex + # commit ID (a tag name is case-sensitive: `@V4.38.1` is not `@v4.38.1`). + lm = tolower(masked); la = tolower(a "@"); from = 1 + hex = (length(r) == 40 && r ~ /^[0-9a-fA-F]+$/) + while ((p = index(substr(lm, from), la)) > 0) { + p += from - 1 + got = substr(masked, p + length(la), length(r)) + same = hex ? (tolower(got) == tolower(r)) : (got == r) + nxt = substr(lm, p + length(la) + length(r), 1) + if (same && (nxt == "" || index(ends, nxt) > 0) && substr(lm, 1, p - 1) ~ tail) { + print substr(line, 1, p + length(a)) good substr(line, p + length(la) + length(r)) + exit + } + from = p + 1 + } + print line + }' +} + +# A denied pin token in any position (a lock record key, a fixture, a block +# scalar), not a `uses:` site this sweep rewrites: a denylisted SHA in either +# spelling, or a blocked version written after ITS action +# (`github/codeql-action/init@v4.38.1` in a `run: |` script) — phase 1 found +# the candidate by that token, so phase 2 must see it too, never call the +# file clean. A SHA is a token wherever it stands, bounded by anything but +# a letter or digit: after `@` in a reference, but also as a lock record's +# `ref:` or `commit: 'sha1-'` under a key that names a tag. A blocked +# version is blocked for its own action only, as at a site +# (`denylisted_pair`): `some/action@v4.38.1` is no finding. +line_has_denied_token() { + local tok t + if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then + return 0 + fi + [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 + while IFS= read -r tok; do + [ -n "$tok" ] || continue + t=$(printf '%s' "$tok" | sed -E "s/${REF_END_CLASS}\$//") + denylisted_pair "$(printf '%s' "${t%@*}" | awk -F/ '{print $1"/"$2}')" "${t##*@}" && return 0 + done < <(printf '%s\n' "$1" | grep -oiE "[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]*@(${REF_ALTERNATION})(${REF_END})" || true) + return 1 +} + +# Lines carrying a denied pin: a `uses:` site whose action/ref pair the +# policy denies, or a denylisted SHA token anywhere else. +count_denied_lines() { + local n=0 line mline site site_action site_ref masked + masked=$(mktemp) + mask_file "$1" > "$masked" + while IFS= read -r line <&3 && IFS= read -r mline <&4; do + if line_has_denied_token "$line"; then n=$((n + 1)); continue; fi + while IFS= read -r site; do + [ -n "$site" ] || continue + site_parts "$site" + if denylisted_pair "$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}')" "$site_ref"; then + n=$((n + 1)); break + fi + done < <(masked_sites "$mline") + done 3< "$1" 4< "$masked" + rm -f "$masked" + printf '%s' "$n" +} + +# The column of a line's comment, found outside quoted scalars: a `#` at +# column 1 or after whitespace. `name: "C# build"` carries no comment; 0. +comment_column() { + printf '%s\n' "$1" | awk -v sq="'" '{ + q = "" + for (i = 1; i <= length($0); i++) { + c = substr($0, i, 1) + if (q == "") { + if (c == "\"" || c == sq) q = c + else if (c == "#" && (i == 1 || substr($0, i - 1, 1) ~ /[[:space:]]/)) { print i; exit } + } else if (q == "\"" && c == "\\") i++ + else if (c == q) q = "" + } + print 0 + }' +} + known_good_sha_for() { - jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_sha' "$POLICY" | head -n1 + jq -r --arg a "$(lc "$1")" '.pin_denylist[] | select((.action | ascii_downcase) == $a) | .known_good_sha' "$POLICY" | head -n1 } known_good_version_for() { - jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_version' "$POLICY" | head -n1 + jq -r --arg a "$(lc "$1")" '.pin_denylist[] | select((.action | ascii_downcase) == $a) | .known_good_version' "$POLICY" | head -n1 } # Relabel a pin's inline comment ONLY when the comment leads with a version @@ -140,14 +413,58 @@ log "policy $(jq -r '.version' "$POLICY") · blocked SHAs ${#DENY_SHAS[@]} · bl HITS="$OUT_DIR/hits.tsv" : > "$HITS" +# Search mode is reconnaissance: one code-search query per token, two seconds +# apart. An entry that blocks a window of commits (PIN-003 names 545) would +# cost one query per SHA — ~18 minutes of sleeps for one action — so such an +# entry is searched by its action path instead (one query), and phase 2 +# confirms each hit against the full token alternation. Small denylists keep +# the exact per-SHA search. +mapfile -t SEARCH_TOKENS < <(jq -r ' + .pin_denylist[] + | (if ((.blocked_shas // []) | length) > 8 then [.action] else (.blocked_shas // []) end)[], + (.blocked_versions // [])[]' "$POLICY" | sort -u) + +# GitHub's code search returns at most 1,000 results per query (ten pages +# of 100), whatever the true count. A broad query that overflows the cap +# leaves candidates phase 2 never sees, so the sweep says so and refuses a +# clean verdict: a search that could not see everything is INCOMPLETE, and +# only `--via clone` can clear the org. A query that FAILS (expired auth, +# rate limit, outage: `gh api` exits non-zero) is the same case, not an +# empty page — zero results from a request that never answered is not a +# clean org. +incomplete=0 +failed=0 +skipped_repos=0 if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do - for token in "${DENY_SHAS[@]}" "${DENY_VERSIONS[@]}"; do + if ! qual=$(owner_qualifier "$org"); then + log " ! ${org}: could not tell whether it is a user or an organisation, so it cannot be searched — phase 1 is INCOMPLETE for this org, use --via clone" + failed=$((failed + 1)); continue + fi + for token in "${SEARCH_TOKENS[@]}"; do page=1 while :; do - n=$(gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ - --jq '.items[] | "'"${org}"'\t\(.repository.name)\t\(.path)"' 2>/dev/null \ + if ! gh api "search/code?q=${token}+${qual}:${org}&per_page=100&page=${page}" \ + > "$OUT_DIR/.search.json" 2> "$OUT_DIR/.search.err"; then + log " ! ${org}: search for '${token}' (page ${page}) failed: $(head -c 200 "$OUT_DIR/.search.err" | tr '\n' ' ') — phase 1 is INCOMPLETE for this org, use --via clone" + failed=$((failed + 1)) + break + fi + n=$(jq -r '.items[]? | "'"${org}"'\t\(.repository.name)\t\(.path)"' "$OUT_DIR/.search.json" 2>/dev/null \ | tee -a "$HITS" | wc -l) + # GitHub answers a timed-out search with HTTP 200, the matches found + # so far and `incomplete_results: true` — a partial page, on any page. + if [ "$(jq -r '.incomplete_results // false' "$OUT_DIR/.search.json" 2>/dev/null)" = "true" ]; then + log " ! ${org}: '${token}' (page ${page}) is marked incomplete_results: GitHub timed the search out and answered with a partial page — phase 1 is INCOMPLETE for this org, use --via clone" + incomplete=$((incomplete + 1)) + fi + if [ "$page" -eq 1 ]; then + total=$(jq -r '.total_count // 0' "$OUT_DIR/.search.json" 2>/dev/null || echo 0) + if [ "${total:-0}" -gt 1000 ] 2>/dev/null; then + log " ! ${org}: '${token}' has ${total} results; GitHub shows only the first 1000 — phase 1 is INCOMPLETE for this org, use --via clone" + incomplete=$((incomplete + 1)) + fi + fi [ "$n" -lt 100 ] && break page=$((page + 1)) [ "$page" -gt 10 ] && break @@ -161,16 +478,40 @@ else WORK="$OUT_DIR/clones" mkdir -p "$WORK" for org in "${ORGS[@]}"; do - mapfile -t reponames < <(list_repos "$org") - [ "$MAX_REPOS" -gt 0 ] && reponames=("${reponames[@]:0:$MAX_REPOS}") + # A listing or a clone that fails is a repository the authoritative mode + # never inspected: INCOMPLETE, never a clean verdict — as a failed search is. + if ! names=$(list_repos "$org"); then + log " ! ${org}: listing repositories failed — phase 1 is INCOMPLETE for this org" + failed=$((failed + 1)); continue + fi + reponames=() + [ -n "$names" ] && mapfile -t reponames <<<"$names" + if [ "$MAX_REPOS" -gt 0 ] && [ "${#reponames[@]}" -gt "$MAX_REPOS" ]; then + skipped_repos=$((skipped_repos + ${#reponames[@]} - MAX_REPOS)) + reponames=("${reponames[@]:0:$MAX_REPOS}") + fi log " cloning ${#reponames[@]} repo(s) in ${org}" + # Clones live under the organisation: two organisations may each own a + # repository of the same name, and the second must not reuse the first's. + mkdir -p "$WORK/$org" for name in "${reponames[@]}"; do - d="$WORK/$name" - [ -d "$d/.git" ] || git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null || { - log " ! clone failed: ${org}/${name}"; continue; } + d="$WORK/$org/$name" + # A clone kept from an earlier run with the same --out is brought to the + # default branch's current head before it is read: an old checkout + # misses a pin added since, and a candidate never listed is never read. + if [ -d "$d/.git" ]; then + if ! { git -C "$d" fetch --depth 1 --quiet origin HEAD 2>/dev/null \ + && git -C "$d" reset --hard --quiet FETCH_HEAD 2>/dev/null \ + && git -C "$d" clean -fdxq 2>/dev/null; }; then + log " ! refreshing the cached clone of ${org}/${name} failed — phase 1 is INCOMPLETE for this org" + failed=$((failed + 1)); continue + fi + elif ! git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null; then + log " ! clone failed: ${org}/${name} — phase 1 is INCOMPLETE for this org"; failed=$((failed + 1)); continue + fi while IFS= read -r f; do printf '%s\t%s\t%s\n' "$org" "$name" "${f#"$d"/}" >> "$HITS" - done < <(grep -rlE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) + done < <(grep -rliE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) done done sort -u "$HITS" -o "$HITS" @@ -178,15 +519,49 @@ fi total_hits=$(wc -l < "$HITS") log "phase 1: ${total_hits} candidate file(s)" +# Repositories with a candidate file the sweep did not read (left out by +# --sample, or unreadable in phase 2): no repair is planned for them. +: > "$OUT_DIR/.unread-repos" +sampled=0 +if [ "$SAMPLE" -gt 0 ] && [ "$total_hits" -gt "$SAMPLE" ]; then + # A bounded run reads the first N candidates only: INCOMPLETE by construction. + sampled=$((total_hits - SAMPLE)) + tail -n +"$((SAMPLE + 1))" "$HITS" | cut -f1,2 | tr '\t' '/' >> "$OUT_DIR/.unread-repos" + head -n "$SAMPLE" "$HITS" > "$HITS.sample" && mv "$HITS.sample" "$HITS" + log " --sample ${SAMPLE}: ${sampled} candidate file(s) left unread; the sweep cannot report clean" +fi + +# What phase 1 could not see. A sweep that saw everything and found nothing +# is clean; one that could not see everything never is. +unseen() { + local parts=() + [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search page(s) overflowed GitHub's 1,000-result cap or came back marked incomplete") + [ "$failed" -gt 0 ] && parts+=("${failed} search(es), listing(s) or clone(s) failed") + [ "$skipped_repos" -gt 0 ] && parts+=("${skipped_repos} repo(s) skipped by --max-repos") + [ "${unread:-0}" -gt 0 ] && parts+=("${unread} candidate file(s) could not be read") + [ "${sampled:-0}" -gt 0 ] && parts+=("${sampled} candidate file(s) left unread by --sample") + local IFS=';' + printf '%s' "${parts[*]}" +} if [ "$total_hits" -eq 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$sampled" -gt 0 ]; then + echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but $(unseen); a clean verdict needs --via clone" + exit 2 + fi echo "clean: no denylisted pin tokens found in ${ORGS[*]}" exit 0 fi # ─── Phase 2: confirm, classify, and build the repaired tree ───────────── +# A phase-1 candidate is a file the search (or grep) associated with a +# token; for a broad action-path query that includes every healthy caller. +# Only a file in which a denylisted token is CONFIRMED is a finding, and +# only confirmed findings decide the exit code. repairable=0 flagged=0 +confirmed=0 +unread=0 while IFS=$'\t' read -r org repo path; do [ -n "${path:-}" ] || continue @@ -194,14 +569,17 @@ while IFS=$'\t' read -r org repo path; do content=$(gh api "repos/${org}/${repo}/contents/${path}?ref=HEAD" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true) if [ -z "$content" ]; then log " ! cannot read ${org}/${repo}/${path} (binary, missing, or no permission)" + unread=$((unread + 1)) + printf '%s\n' "${org}/${repo}" >> "$OUT_DIR/.unread-repos" continue fi printf '%s\n' "$content" > "$OUT_DIR/.current" - sites_before=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.current" || true) - denied_before=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.current" || true) + sites_before=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.current" || true) + denied_before=$(count_denied_lines "$OUT_DIR/.current") [ "$denied_before" -eq 0 ] && continue + confirmed=$((confirmed + 1)) # Scope guard: a repair is only meaningful where a pin is *live*. A # documentation code block quoting the poisoned SHA is a finding, not a @@ -213,39 +591,56 @@ while IFS=$'\t' read -r org repo path; do *) in_scope=0 ;; esac - # Rewrite one line at a time so each pin gets the known-good SHA for ITS - # action, and so an action with no known-good pin blocks the whole file. + # Rewrite one pin SITE at a time: each `uses:` site gets the known-good + # SHA for ITS action (two actions on one flow line get two replacements); + # an action with no known-good pin blocks the whole file; a denylisted + # token outside a `uses:` key (a lock record, a fixture) is a finding the + # owner reviews, never a rewrite. Only the comment after the line's one + # replaced pin is relabelled, found outside quoted scalars. fixable=1 actions="" + outside_uses=0 [ "$in_scope" -eq 0 ] && fixable=0 : > "$OUT_DIR/.rewritten" - while IFS= read -r line || [ -n "$line" ]; do - if printf '%s\n' "$line" | grep -qE "@(${REF_ALTERNATION})([[:space:]#]|$)"; then - action=$(printf '%s\n' "$line" | sed -E 's/^[[:space:]]*-?[[:space:]]*uses:[[:space:]]*([^@]+)@.*/\1/') - base=$(printf '%s\n' "$action" | awk -F/ '{print $1"/"$2}') - good=$(known_good_sha_for "$base") + mask_file "$OUT_DIR/.current" > "$OUT_DIR/.masked" + while IFS= read -r line <&3 && IFS= read -r mline <&4; do + out="$line"; mout="$mline"; replaced=0; site_seen=0; good_version="" + while IFS= read -r site; do + [ -n "$site" ] || continue + site_parts "$site" + base=$(lc "$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}')") + denylisted_pair "$base" "$site_ref" || continue + site_seen=1 actions="${actions}${base} " - if [ -z "$good" ] || [ "$good" = "null" ]; then - fixable=0 - printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" - else - good_version=$(known_good_version_for "$base") - swapped=$(printf '%s\n' "$line" | sed -E "s%@(${REF_ALTERNATION})([[:space:]#]|$)%@${good}\2%g") - head="${swapped%%#*}" - if [ "$head" != "$swapped" ]; then - tail_comment="#${swapped#*#}" - printf '%s%s\n' "$head" "$(relabel_comment "$tail_comment" "$good_version")" >> "$OUT_DIR/.rewritten" - else - printf '%s\n' "$swapped" >> "$OUT_DIR/.rewritten" - fi + good=$(known_good_sha_for "$base") + if [ -z "$good" ] || [ "$good" = "null" ]; then fixable=0; continue; fi + out=$(replace_site "$out" "$mout" "$site_action" "$site_ref" "$good") + mout=$(mask_line "$out") + replaced=$((replaced + 1)) + good_version=$(known_good_version_for "$base") + done < <(masked_sites "$mline") + # A denylisted SHA left on the line that is not at a `uses:` site — the + # line had none, or the SHA sits in a quoted scalar or a comment the + # mask blanked — is a finding for the owner, never part of a repair. + if line_has_denied_token "$out" && { [ "$site_seen" -eq 0 ] || ! line_has_denied_token "$mout"; }; then + outside_uses=$((outside_uses + 1)); fixable=0 + fi + # The comment is relabelled only when the line has ONE `uses:` site: on a + # flow line with several, the trailing comment cannot be tied to the + # replaced one (`{ uses: }, { uses: actions/checkout@ } # v7` + # annotates checkout), so it is kept as written. + nsites=$(printf '%s\n' "$mline" | grep -oiE "(^|[-{,[:space:]])${USES_KEY}[^@\"'[:space:],}]+@" | wc -l || true) + if [ "$replaced" -eq 1 ] && [ "$nsites" -eq 1 ] && [ -n "$good_version" ] && [ "$good_version" != "null" ]; then + at=$(comment_column "$out") + if [ "$at" -gt 0 ]; then + out="${out:0:$((at - 1))}$(relabel_comment "${out:$((at - 1))}" "$good_version")" fi - else - printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" fi - done < "$OUT_DIR/.current" + printf '%s\n' "$out" >> "$OUT_DIR/.rewritten" + done 3< "$OUT_DIR/.current" 4< "$OUT_DIR/.masked" - sites_after=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.rewritten" || true) - denied_after=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.rewritten" || true) + sites_after=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.rewritten" || true) + denied_after=$(count_denied_lines "$OUT_DIR/.rewritten") status="flag" reason="owner_review" @@ -255,6 +650,8 @@ while IFS=$'\t' read -r org repo path; do reason="substitution" elif [ "$in_scope" -eq 0 ]; then reason="out_of_scope_file" + elif [ "$outside_uses" -gt 0 ]; then + reason="denylisted_token_outside_uses_key" elif [ "$fixable" -eq 0 ]; then reason="no_known_good_pin_for_action" elif [ "$sites_after" -ne "$sites_before" ]; then @@ -263,7 +660,7 @@ while IFS=$'\t' read -r org repo path; do jq -cn \ --arg repo "${org}/${repo}" --arg path "$path" --arg status "$status" \ - --arg reason "$reason" --arg actions "$(printf '%s' "$actions" | xargs || true)" \ + --arg reason "$reason" --arg actions "$(printf '%s' "$actions" | tr -s '[:space:]' ' ' | sed -E 's/^ //; s/ $//')" \ --argjson denied "$denied_before" --argjson sites "$sites_before" \ '{repo:$repo, path:$path, rule:"PI001", status:$status, repair:$reason, actions:($actions|split(" ")|map(select(.!=""))), @@ -281,12 +678,53 @@ while IFS=$'\t' read -r org repo path; do fi done < "$HITS" -# One repair PR per repository. +if [ "$confirmed" -eq 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then + echo "INCOMPLETE: none of the ${total_hits} candidate file(s) carries a denylisted pin, but $(unseen); a clean verdict needs --via clone" + exit 2 + fi + echo "clean: ${total_hits} candidate file(s) read, none carries a denylisted pin in ${ORGS[*]}" + exit 0 +fi + +# One repair PR per repository, and only for a repository the sweep has +# proved whole: every confirmed file repairable and every candidate file +# read. Repairing one workflow while a flagged lock entry or fixture in the +# same repository keeps the denylisted pin (or an unread file may) is a +# partial repair, and arming it would merge a repository that is still +# poisoned. Such a repository is held for owner review with its findings. +# A search that failed or came back partial may have hidden any sibling in +# any repository, so then no repository is proved whole and none is armed; +# a clone, a listing or --max-repos that missed a repository missed all of +# it, which leaves the repositories that were cloned whole. +held=0 +search_partial=false +if [ "$VIA" = "search" ] && { [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ]; }; then + search_partial=true +fi if [ "$repairable" -gt 0 ]; then - jq -src ' - map(select(.status == "repairable")) - | group_by(.repo) + unread_repos=$(jq -Rsc 'split("\n") | map(select(. != "")) | unique' "$OUT_DIR/.unread-repos") + jq -sr --argjson unread "$unread_repos" --argjson partial "$search_partial" ' + group_by(.repo)[] + | select(any(.status == "repairable")) + | .[0].repo as $r + | if $partial then "\($r)\tphase 1 search failed or was partial, so a sibling file may be unseen" + elif any($unread[]; . == $r) then "\($r)\tcandidate file not read (unreadable, or left out by --sample)" + elif all(.status == "repairable") | not then + "\($r)\tflagged sibling file(s): \([.[] | select(.status != "repairable") | .path] | join(", "))" + else empty end + ' "$FINDINGS" > "$OUT_DIR/.held" + held=$(grep -c . "$OUT_DIR/.held" || true) + while IFS=$'\t' read -r r why; do + log " held ${r}: repairable file(s) not planned — ${why}" + done < "$OUT_DIR/.held" + + jq -src --argjson unread "$unread_repos" --argjson partial "$search_partial" ' + group_by(.repo) | .[] + | select($partial | not) + | select(all(.status == "repairable")) + | select(.[0].repo as $r | any($unread[]; . == $r) | not) | {repo: .[0].repo, files: length, paths: [.[].path], @@ -302,13 +740,19 @@ fi log "" log "─── pin-integrity sweep ────────────────────────────────────────────" log " candidate files : ${total_hits}" +log " confirmed denylisted : ${confirmed}" log " mechanically repairable : ${repairable}" log " needs owner review : ${flagged}" -log " repos with a repair ready : $(grep -c . "$PLAN" 2>/dev/null || echo 0)" +planned=$(grep -c . "$PLAN" 2>/dev/null || true) +log " repos with a repair ready : ${planned:-0}" +log " repos held (partial repair): ${held}" log " findings : ${FINDINGS}" log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" log " NOTE: this sweep never writes to a remote. Branch + PR creation is" log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" +if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then + log " INCOMPLETE: $(unseen); what they hid was not seen — rerun with --via clone" +fi exit 2 diff --git a/test/cli_test.exs b/test/cli_test.exs index ef1fad84..c95fca3d 100644 --- a/test/cli_test.exs +++ b/test/cli_test.exs @@ -40,6 +40,80 @@ defmodule Hypatia.CLITest do assert help =~ "HYPATIA_EXIT_ZERO" assert help =~ "EXIT CODES" end + + test "documents --advisory and HYPATIA_ADVISORY" do + help = capture_io(fn -> Hypatia.CLI.main(["help"]) end) + + assert help =~ "--advisory" + assert help =~ "HYPATIA_ADVISORY" + end + end + + # WF026/WF027 name a repair the fleet can route and must never fail a gate + # (AGENTIC must-not-promote-to-blocking). At `:low` the default threshold + # hid them from every consumer; lowering `--severity` to see them made them, + # and every other low finding, fail the scan. `report_sets/2` separates + # what is shown from what decides exit 1. + describe "report_sets/2 -- advisory findings are shown on request, never blocking" do + @advisory %{rule: "WF027", severity: "low", advisory: true, fix_script: "x.sh"} + @plain_low %{rule: "flawed_regex", severity: "low"} + @medium %{rule: "unpinned_action", severity: "medium"} + @base %{severity: "medium", exit_zero: false, advisory: false} + + test "default threshold: an advisory low finding is neither shown nor blocking" do + assert {[@medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], @base) + end + + test "--advisory shows it below the threshold, in order, without blocking" do + config = %{@base | advisory: true} + + assert {[@advisory, @medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], config) + end + + test "an advisory finding alone never decides exit 1" do + assert {[@advisory], []} = + Hypatia.CLI.report_sets([@advisory], %{@base | advisory: true}) + end + + test "at --severity low it is shown, a plain low finding blocks, it does not" do + config = %{@base | severity: "low"} + + assert {[@advisory, @plain_low, @medium], [@plain_low, @medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], config) + end + + test "a config without the :advisory key behaves as before" do + assert {[@medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @medium], %{severity: "medium"}) + end + + test "advisory survives the workflow_audit normalizer" do + dir = Path.join(System.tmp_dir!(), "hyp-advisory-#{:erlang.unique_integer([:positive])}") + File.mkdir_p!(Path.join(dir, ".github/workflows")) + on_exit(fn -> File.rm_rf!(dir) end) + + File.write!(Path.join(dir, ".github/workflows/rust-ci.yml"), """ + name: rust-ci + on: push + permissions: {} + jobs: + rust-ci: + uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@d135b05bfc647d0c0fbfedc7e80f37ea50f49236 + """) + + wf027 = + dir + |> Hypatia.CLI.collect_findings([:workflow_audit]) + |> Enum.filter(&(&1.type == "known_broken_reusable_revision")) + + assert [%{advisory: true, severity: "low", fix_script: "fix-stale-reusable-pin.sh"}] = + wf027 + + assert {[], []} = Hypatia.CLI.report_sets(wf027, %{@base | advisory: false}) + assert {^wf027, []} = Hypatia.CLI.report_sets(wf027, %{@base | advisory: true}) + end end # Regression coverage for the placeholder-message defect. diff --git a/test/rules/actions_lock_test.exs b/test/rules/actions_lock_test.exs index f3fb3d97..53622391 100644 --- a/test/rules/actions_lock_test.exs +++ b/test/rules/actions_lock_test.exs @@ -83,6 +83,60 @@ defmodule Hypatia.Rules.ActionsLockTest do assert {:error, {:line, _, :malformed_dependency_commit}} = ActionsLock.parse(non_hex) end + test "accepts lockfile v0.0.3, which gh-actions-lock v0.2.0 writes, with the same grammar" do + v003 = String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.3'") + + assert {:ok, lock} = ActionsLock.parse(v003) + assert lock.version == "v0.0.3" + assert ActionsLock.pinned?(lock, "ci.yml", "actions/checkout@v7.0.1") + refute ActionsLock.pinned?(lock, "other.yml", "actions/checkout@v7.0.1") + + # The grammar is shared, not loosened: a malformed v0.0.3 lock still fails closed. + malformed = String.replace(v003, ~r/sha1-[0-9a-f]{40}/, "sha1-short") + assert {:error, {:line, _, :malformed_dependency_commit}} = ActionsLock.parse(malformed) + end + + test "v0.0.3 hostname: github.com is the default, a GHE.com tenant host is refused" do + v003 = String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.3'") + + with_host = fn lock, host -> + String.replace( + lock, + " repo_id: 197814629", + " repo_id: 197814629\n hostname: '#{host}'" + ) + end + + assert {:ok, lock} = ActionsLock.parse(with_host.(v003, "github.com")) + assert ActionsLock.pinned?(lock, "ci.yml", "actions/checkout@v7.0.1") + + assert {:error, {:line, 13, {:unsupported_dependency_hostname, "acme.ghe.com"}}} = + ActionsLock.parse(with_host.(v003, "acme.ghe.com")) + + # The field is new in v0.0.3; a v0.0.2 lock carrying it is refused. + assert {:error, {:line, 13, {:hostname_not_in_version, "v0.0.2"}}} = + ActionsLock.parse(with_host.(@valid_lock, "github.com")) + + duplicate = with_host.(with_host.(v003, "github.com"), "github.com") + + assert {:error, {:line, _, {:duplicate_dependency_field, _, :hostname}}} = + ActionsLock.parse(duplicate) + end + + test "rejects every version outside the supported set, and a second version line" do + for version <- ["v0.0.1", "v0.0.4", "v0.1.0", "0.0.3"] do + assert {:error, {:line, 2, {:unsupported_version, ^version}}} = + @valid_lock + |> String.replace("version: 'v0.0.2'", "version: '#{version}'") + |> ActionsLock.parse() + end + + twice = + String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.2'\nversion: 'v0.0.3'") + + assert {:error, {:line, 3, :duplicate_version}} = ActionsLock.parse(twice) + end + test "rejects unsupported versions and missing dependency entries" do assert {:error, {:line, 2, {:unsupported_version, "v9"}}} = @valid_lock diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 59c26da7..af7c5ef4 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -80,6 +80,668 @@ defmodule Hypatia.Rules.PinIntegrityTest do end end + # Read one line at a time, a `uses:` written inside a `run: |` script or a + # quoted scalar continued from an earlier line looked like a key: PI001 + # reported script text and substitute_denylisted_pins/3 rewrote it as an + # automatic repair. The file is now read through PinSites.mask_lines/1. + describe "pin_sites/1 across lines — block and quoted scalars are text" do + @real_step " - uses: github/codeql-action/init@#{@poison} # v4.38.0\n" + + test "a uses: line inside a run: | script is no site, the step after it is" do + content = + "jobs:\n b:\n steps:\n - run: |\n" <> + " cat > fixture.yml <<'EOF'\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " - uses: github/codeql-action/analyze@#{@poison} # v4.38.1\n" <> + " EOF\n\n" <> + " shell: bash\n" <> @real_step + + assert [%{line: 11}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + assert new_content =~ "analyze@#{@poison} # v4.38.1\n" + assert new_content =~ " - uses: github/codeql-action/init@#{@good} # v4.38.0\n" + end + + test "a script that is all text leaves the file untouched" do + content = + "steps:\n - name: write\n run: >-\n uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [] + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + end + + test "a bare block-scalar sequence entry is text until the dash's level" do + content = + " - args:\n - |\n uses: github/codeql-action/init@#{@poison}\n" <> + @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "the continuation of a multi-line double-quoted run: is text" do + content = + " - run: \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo \\\"end\\\"\"\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + end + + test "a single-quoted one, with an escaped quote inside, is text too" do + content = + " - run: 'echo it''s\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " done'\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a value pending from the line above may open the quote" do + content = + " - run:\n \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\"\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "an apostrophe in a plain scalar opens nothing" do + content = " - run: echo it's here\n" <> @real_step <> " - run: echo don't\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a quote inside a run: | script opens nothing after the script" do + content = " - run: |\n echo \"unterminated\n" <> @real_step + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a quote never closed by the end of the file hides nothing" do + content = " - run: \"echo start\n" <> @real_step + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "the file mask keeps every line's bytes where they were" do + content = + " - run: |\n uses: a/b@#{@poison}\n - run: \"x\n y\" # c\n" <> + @real_step + + masked = Hypatia.Rules.PinSites.mask_lines(content) + lines = String.split(content, "\n") + + assert Enum.map(masked, &byte_size/1) == Enum.map(lines, &byte_size/1) + assert Enum.at(masked, 1) == String.duplicate("x", byte_size(Enum.at(lines, 1))) + assert Enum.at(masked, 3) == "xxxx # c" + assert Enum.at(masked, 4) == String.trim_trailing(@real_step, "\n") + end + + test "a JSON-style flow value after a bare colon opens its quote too" do + content = + " - {\"run\":\"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo end\"}\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + end + + test "an anchor or tag before the quote does not hide where the scalar starts" do + for opener <- [ + " - {\"run\": &cmd \"echo start\n", + " - {\"run\":&cmd \"echo start\n", + " - run: !!str \"echo start\n" + ] do + content = + opener <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo end\"#{if String.contains?(opener, "{"), do: "}", else: ""}\n" <> + @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + end + + test "a uses: whose reference continues on the next line is a site there" do + content = + " - uses:\n github/codeql-action/init@#{@poison} # v4.38.0\n" <> + " - uses:\n \"github/codeql-action/analyze@#{@poison}\"\n" <> @real_step + + assert [%{line: 2}, %{line: 4}, %{line: 5}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_, _, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + assert new_content =~ " github/codeql-action/init@#{@good} # v4.38.0\n" + assert new_content =~ " \"github/codeql-action/analyze@#{@good}\"\n" + + # Controls: another key's continued value, and a `uses:` whose next + # line is no deeper, hold no site. + assert PI.pi001_denylisted_pin( + " - run:\n github/codeql-action/init@#{@poison}\n" <> + " - uses:\n - name: github/codeql-action/init@#{@poison}\n", + @policy + ) == [] + end + + test "blank and comment-only lines between a uses: key and its reference are skipped" do + # YAML lets them stand between a key and its scalar: the reference is + # still the `uses:` value, so it is still a site — reported and repaired. + content = + " - uses:\n\n # pinned for the 4.38 line\n" <> + " github/codeql-action/init@#{@poison}\n" <> @real_step + + assert [%{line: 4}, %{line: 5}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " github/codeql-action/init@#{@good}\n" + + # Control: a comment and then a line no deeper than the key end it. + assert PI.pi001_denylisted_pin( + " - uses:\n # note\n - name: github/codeql-action/init@#{@poison}\n", + @policy + ) == [] + end + + test "a uses: value carrying an anchor or tag is the pin all the same" do + # YAML node properties name the node; the scalar after them is the + # action reference, so it is a site — reported and repaired. + content = + " - uses: &init github/codeql-action/init@#{@poison}\n" <> + " - uses: !!str \"github/codeql-action/analyze@#{@poison}\"\n" <> + " - { name: x, uses: !!str github/codeql-action/upload-sarif@#{@poison} }\n" <> + " - uses: !!str\n github/codeql-action/autobuild@#{@poison}\n" + + assert [%{line: 1}, %{line: 2}, %{line: 3}, %{line: 5}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_, _, _, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + assert new_content =~ " - uses: &init github/codeql-action/init@#{@good}\n" + assert new_content =~ " - uses: !!str \"github/codeql-action/analyze@#{@good}\"\n" + + # Control: `&` or `!` inside a quoted name is text, not a property. + assert PI.pi001_denylisted_pin( + " - name: \"uses: !!str github/codeql-action/init@#{@poison}\"\n", + @policy + ) == [] + end + + test "a uses: entry of a flow step whose reference continues on the next line is a site" do + content = + " - { name: x, uses:\n github/codeql-action/init@#{@poison} }\n" <> + " steps: [ { uses:\n github/codeql-action/analyze@#{@poison}, name: y } ]\n" + + assert [%{line: 2}, %{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " github/codeql-action/init@#{@good} }\n" + assert new_content =~ " github/codeql-action/analyze@#{@good}, name: y } ]\n" + + # Control: an input's mapping is not a step — `with: { uses:` is data. + assert PI.pi001_denylisted_pin( + " with: { uses:\n github/codeql-action/init@#{@poison} }\n", + @policy + ) == [] + end + + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do + states = + Hypatia.Rules.PinSites.line_states( + "- run: |\n uses: a/b@x\n- uses: c/d@y\n- run: \"open\n uses: e/f@z" + ) + + assert Enum.map(states, &elem(&1, 1)) == [:yaml, :text, :yaml, :yaml, :open] + assert List.last(states) == {" uses: e/f@z", :open} + end + + test "a uses: alias runs the reference its anchor holds; the site is the anchor's value" do + content = + "env:\n BAD: &bad github/codeql-action/init@#{@poison} # v4.38.0\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n - { name: y, uses: *bad }\n" + + # One site however many aliases name it: the anchor's line is where the + # reference stands, and where a repair rewrites it. + assert [%{line: 2, detail: %{ref: @poison, action: "github/codeql-action/init"}}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " BAD: &bad github/codeql-action/init@#{@good} # v4.38.0\n" + assert new_content =~ " - uses: *bad\n" + + # Controls: an alias names the latest anchor before it, so a poisoned + # `&bad` defined after it is not what runs; an anchor whose value is no + # action reference names no site; nor does an alias that is not a + # `uses:` value. + later = + "env:\n OK: &bad github/codeql-action/init@#{@good}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" <> + "x:\n BAD: &bad github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(later, @policy) == [] + + not_action = "x: &bad stable\njobs:\n a:\n steps:\n - uses: *bad\n" + assert PI.pi001_denylisted_pin(not_action, @policy) == [] + + not_uses = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n ALSO: *bad\n" <> + " - name: \"uses: *bad\"\n" + + assert PI.pi001_denylisted_pin(not_uses, @policy) == [] + end + + test "node properties before a step's own mapping leave it a step" do + content = + " steps: [ &step { uses: github/codeql-action/init@#{@poison} } ]\n" <> + " - &other { name: x, uses: github/codeql-action/analyze@#{@poison} }\n" + + assert [%{line: 1}, %{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + + # Control: an anchored input mapping is still data. + assert PI.pi001_denylisted_pin( + " with: &w { uses: github/codeql-action/init@#{@poison} }\n", + @policy + ) == [] + end + + test "an anchor whose scalar continues on the next line holds that reference" do + content = + "env:\n BAD: &bad\n github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3, detail: %{ref: @poison}}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " BAD: &bad\n github/codeql-action/init@#{@good}\n" + + # Control: a line that is not deeper is no continuation — `&bad` is + # null, and the alias names no reference. + null = + "env:\n BAD: &bad\n NEXT: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert PI.pi001_denylisted_pin(null, @policy) == [] + end + + test "inside a flow collection opened above, a step's uses: after a sibling is a site" do + # The quoted sibling closes on the line the `uses:` stands on; the + # brackets the lines above opened still make it the step's own key. + quoted = + "jobs:\n a:\n steps: [{ name: \"hello\n world\", " <> + "uses: github/codeql-action/init@#{@poison} }]\n" + + sibling = + "jobs:\n a:\n steps: [\n {\n name: x, " <> + "uses: github/codeql-action/init@#{@poison}\n }\n ]\n" + + for {content, line} <- [{quoted, 4}, {sibling, 5}] do + assert [%{line: ^line}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ "github/codeql-action/init@#{@good}" + end + + # Controls: the same layout inside a step input's mapping is data, and + # an own-line `uses:` there is too. + for content <- [ + "jobs:\n a:\n steps: [{ with: { note: \"x\n y\", " <> + "uses: github/codeql-action/init@#{@poison} } }]\n", + "jobs:\n a:\n steps:\n - with: {\n " <> + "uses: github/codeql-action/init@#{@poison}\n }\n" + ] do + assert PI.pi001_denylisted_pin(content, @policy) == [] + end + end + + test "a value pending after node properties opens the quoted scalar below it" do + # `run: &cmd` leaves its value pending: the quote on the next line + # opens the string, and the `uses:` text inside it is no site. + content = + " - run: &cmd\n \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n echo end\"\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [] + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + + # Control: a value written on the property's own line pends nothing. + live = " - run: &cmd echo\n - uses: github/codeql-action/init@#{@poison}\n" + assert [%{line: 2}] = PI.pi001_denylisted_pin(live, @policy) + end + + test "an aliased uses: inside a flow collection opened above is judged by its brackets" do + # `uses: *bad` in a step's `env: {` spread over lines is an env var. + anchor = "x: &bad github/codeql-action/init@#{@poison}\n" + + env = + anchor <> + "jobs:\n a:\n steps: [\n { run: echo, env: {\n uses: *bad\n } }\n ]\n" + + assert PI.pi001_denylisted_pin(env, @policy) == [] + assert {:ok, ^env, []} = PI.substitute_denylisted_pins(env, @policy) + + # Control: the step's own `uses:` after a sibling is the step's action. + step = + anchor <> "jobs:\n a:\n steps: [\n { run: echo,\n uses: *bad }\n ]\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(step, @policy) + end + + test "a steps: sequence is a job's only when the brackets say so" do + # A matrix value holding `steps: [ { uses: ... } ]` is data. + data = + " include: [ { fixture: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] } } ]\n" + + assert PI.pi001_denylisted_pin(data, @policy) == [] + + # Controls: through a job key into `jobs`, or the whole stack a block + # job's own line shows. + for line <- [ + "jobs: { b: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] } }\n", + " steps: [ { uses: github/codeql-action/init@#{@poison} } ]\n" + ] do + assert [%{line: 1}] = PI.pi001_denylisted_pin(line, @policy) + end + end + + test "an anchor is a node property only where a node starts" do + # In block context `a, &bad b` is one plain scalar: no anchor. + text = + "env:\n NOTE: a, &bad github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert PI.pi001_denylisted_pin(text, @policy) == [] + + # Control: after `[` in a flow sequence a node starts. + flow = + "env:\n LIST: [ &bad github/codeql-action/init@#{@poison} ]\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(flow, @policy) + end + + test "a block uses: key under no step or job is data, not a site" do + # An input named `uses` is the step's data, as `with: { uses: ... }` + # is: PI001 does not report it and the repair leaves it as is. + input = + "jobs:\n b:\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(input, @policy) == [] + assert {:ok, ^input, []} = PI.substitute_denylisted_pins(input, @policy) + + # Controls: a step's own key, a job's own key, a composite action's + # step, and a line whose ancestry the file does not show (judged on + # its own, as before). + step = + "jobs:\n b:\n steps:\n - name: init\n uses: github/codeql-action/init@#{@poison}\n" + + job = "jobs:\n b:\n uses: github/codeql-action/init@#{@poison}\n" + + composite = + "runs:\n using: composite\n steps:\n - uses: github/codeql-action/init@#{@poison}\n" + + fragment = " - uses: github/codeql-action/init@#{@poison}\n" + + for {text, line} <- [{step, 5}, {job, 3}, {composite, 4}, {fragment, 1}] do + assert [%{line: ^line}] = PI.pi001_denylisted_pin(text, @policy), text + end + end + + test "an input named uses holds no site below its key or through its alias" do + input = + "jobs:\n b:\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " uses: github/codeql-action/init@#{@poison}\n" + + continued = String.replace(input, "uses: github/", "uses:\n github/") + + aliased = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n" <> + String.replace(input, "uses: github/codeql-action/init@#{@poison}", "uses: *bad") + + assert PI.pi001_denylisted_pin(continued, @policy) == [] + assert PI.pi001_denylisted_pin(aliased, @policy) == [] + end + + test "a list item is a step only in a job's steps:" do + # A sequence another key holds, and a `steps:` inside a step's input. + listed = + "jobs:\n b:\n steps:\n - uses: some/runner@v1\n with:\n" <> + " list:\n - uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(listed, @policy) == [] + assert PI.pi001_denylisted_pin(String.replace(listed, "list:", "steps:"), @policy) == [] + end + + test "an anchor directly after a quoted key's colon in a flow mapping is the anchor" do + # JSON-style `{"BAD":&bad ...}`: the value starts right after the colon. + content = + "env: {\"BAD\":&bad \"github/codeql-action/init@#{@poison}\"}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "an anchored scalar continued inside a flow mapping may start at column 0" do + content = + "env: {\n BAD: &bad\ngithub/codeql-action/init@#{@poison}\n}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "a step given whole as an alias of a flow mapping runs that mapping's uses:" do + # The anchored mapping sits in a matrix entry, which is data: the + # alias in `steps:` makes it the step. + matrix = + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - step: &step { uses: github/codeql-action/init@#{@poison} }\n" <> + " steps:\n - *step\n" + + assert [%{line: 6}] = PI.pi001_denylisted_pin(matrix, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(matrix, @policy) + refute fixed =~ @poison + + flow_seq = + "x: &step { uses: github/codeql-action/init@#{@poison} }\n" <> + "jobs:\n a:\n steps: [*step]\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(flow_seq, @policy) + + # Control: with no alias in a step position, the matrix entry is data. + unused = String.replace(matrix, " - *step\n", " - run: echo\n") + assert PI.pi001_denylisted_pin(unused, @policy) == [] + end + + test "a step or job given whole as an alias of a block mapping runs its uses:" do + step = + "x:\n - &step\n uses: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - *step\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(step, @policy) + + # The job's mapping anchored in a matrix entry, data until aliased. + job = + "jobs:\n b:\n strategy:\n matrix:\n include:\n - job: &job\n" <> + " uses: github/codeql-action/init@#{@poison}\n a: *job\n" + + assert [%{line: 7}] = PI.pi001_denylisted_pin(job, @policy) + assert PI.pi001_denylisted_pin(String.replace(job, " a: *job\n", ""), @policy) == [] + end + + test "an anchor name runs to whitespace or a flow indicator" do + content = + "env:\n BAD: &bad/x github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad/x\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a step alias runs a flow mapping anchored over several lines" do + content = + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - step: &step {\n name: init,\n" <> + " uses: github/codeql-action/init@#{@poison}\n }\n" <> + " steps:\n - *step\n" + + assert [%{line: 8}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "a version is matched past one optional v only" do + policy = put_in(@policy, ["pin_denylist", Access.at(0), "blocked_versions"], ["4.38.1"]) + + assert [_] = PI.pi001_denylisted_pin("- uses: github/codeql-action/init@v4.38.1\n", policy) + assert PI.pi001_denylisted_pin("- uses: github/codeql-action/init@vv4.38.1\n", policy) == [] + end + + test "a substitution that leaves a denylisted pin is refused" do + # A `known_good_sha` that is itself blocked repairs nothing. + policy = put_in(@policy, ["pin_denylist", Access.at(0), "known_good_sha"], @poison) + content = "- uses: github/codeql-action/init@#{@poison}\n" + + assert {:error, reason} = PI.substitute_denylisted_pins(content, policy) + assert reason =~ "leaves a denylisted pin" + end + + test "a uses: key under a root key that holds no job is data in a whole workflow" do + for root <- ["env", "x-template"] do + content = + "#{root}:\n uses: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - run: echo\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [], root + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + end + + # Control: a fragment that shows no root (no `jobs:`) is judged on its + # own line, as before: a job's `uses:` for all it shows. + assert [%{line: 2}] = + PI.pi001_denylisted_pin( + "build:\n uses: github/codeql-action/init@#{@poison}\n", + @policy + ) + end + + test "a step alias whose mapping names its action by alias runs the anchored reference" do + content = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - &step\n uses: *bad\n" <> + " steps:\n - *step\n" + + # The site is the anchor's value, where the repair rewrites it. + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + + # Control: with no step alias the matrix entry, alias and all, is data. + data = String.replace(content, " - *step\n", " - run: echo\n") + assert PI.pi001_denylisted_pin(data, @policy) == [] + end + + test "a uses: block scalar whose one line is a reference is a site on that line" do + content = + "jobs:\n a:\n steps:\n" <> + " - uses: >-\n github/codeql-action/init@#{@poison}\n" <> + " - uses: |\n github/codeql-action/init@#{@poison}\n" + + assert [%{line: 5}, %{line: 7}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + assert fixed =~ " - uses: >-\n github/codeql-action/init@#{@good}\n" + + # An anchored block scalar names the same reference to a `uses: *bad`. + anchored = + "env:\n BAD: &bad >-\n github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(anchored, @policy) + + # Controls: a block scalar that folds more lines into its value holds + # no reference; under `run:` or an input it is text or data. + for other <- [ + " - uses: >-\n github/codeql-action/init@#{@poison}\n extra\n", + " - run: |\n github/codeql-action/init@#{@poison}\n", + " - uses: a/b@v1\n with:\n uses: >-\n github/codeql-action/init@#{@poison}\n" + ] do + assert PI.pi001_denylisted_pin("jobs:\n a:\n steps:\n" <> other, @policy) == [], other + end + end + + test "a composite action written as KYAML: its runs: steps are sites" do + # `yq -p yaml -o kyaml` of a composite action (YAML-POLICY Y-3). + content = """ + { + name: "setup", + runs: { + using: "composite", + steps: [ + { + uses: "github/codeql-action/init@#{@poison}", # v4.38.1 + }, + ], + }, + } + """ + + assert [%{line: 7}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert fixed =~ ~s(uses: "github/codeql-action/init@#{@good}", # v4.38.0) + + # Control: a `runs:` mapping that is an input's value, not the action's + # own root key, holds data. + data = + "runs:\n using: composite\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " runs: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] }\n" + + assert PI.pi001_denylisted_pin(data, @policy) == [] + end + + test "a double-quoted value is the reference YAML decodes it to" do + tail = String.slice(@poison, 1..-1//1) + escaped = "\\x31" <> tail + steps = "jobs:\n a:\n steps:\n" + + # An escape in the ref, an escaped `/` in the action, the `\u` form + # and a value continued below its key: each is the denylisted + # commit GitHub runs, and the repair rewrites exactly the ref's bytes. + for {value, repaired} <- [ + {~s( - uses: "github/codeql-action/init@#{escaped}"\n), + ~s( - uses: "github/codeql-action/init@#{@good}"\n)}, + {~s( - uses: "github\\/codeql-action/init@#{@poison}"\n), + ~s( - uses: "github\\/codeql-action/init@#{@good}"\n)}, + {~s( - { uses: "github/codeql-action/init@\\u0031#{tail}" }\n), + ~s( - { uses: "github/codeql-action/init@#{@good}" }\n)}, + {~s( - uses:\n "github/codeql-action/init@#{escaped}"\n), + ~s( - uses:\n "github/codeql-action/init@#{@good}"\n)} + ] do + content = steps <> value + assert [_] = PI.pi001_denylisted_pin(content, @policy), value + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert fixed == steps <> repaired + end + + # Control: an escape YAML does not define decodes to nothing. + bad = steps <> ~s( - uses: "github/codeql-action/init@\\q#{@poison}"\n) + assert PI.pi001_denylisted_pin(bad, @policy) == [] + end + + test "a KYAML flow step on its own line is still a site" do + content = + "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> + " { uses: \"github/codeql-action/init@#{@poison}\" },\n]\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + end + end + describe "action_base/1" do test "keys on the release unit, not the sub-action" do assert PI.action_base("github/codeql-action/init") == "github/codeql-action" @@ -96,6 +758,10 @@ defmodule Hypatia.Rules.PinIntegrityTest do PI.denylisted?(@policy, "github/codeql-action/analyze", "v4.38.1") assert %{"id" => "PIN-001"} = PI.denylisted?(@policy, "github/codeql-action/init", "4.38.1") + + # Upper-case hex is the same commit; the table is written in lower case. + assert %{"id" => "PIN-001"} = + PI.denylisted?(@policy, "github/codeql-action/init", String.upcase(@poison)) end test "does not match a good ref, another action, or another repo's entry" do @@ -276,6 +942,67 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi001_denylisted_pin(new_content, @policy) == [] end + test "a quoted \"uses\" key is the same key: caught by PI001 and repaired in place" do + content = """ + jobs: + a: + steps: + - "uses": "github/codeql-action/init@#{@poison}" # v4.38.1 + """ + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ ~s("uses": "github/codeql-action/init@#{@good}" # v4.38.0) + assert PI.pi001_denylisted_pin(new_content, @policy) == [] + end + + test "a compact flow step is a site: caught by PI001 and spliced at its own offset" do + # `- { name: ..., uses: ... }` carries the dash, the brace and a sibling + # key; the old whole-line pattern allowed only one of the first two and + # nothing after the ref, so this pin was invisible to the file scan. + content = """ + jobs: + a: + steps: + - { name: "scan #{@poison}", uses: github/codeql-action/init@#{@poison} } # v4.38.1 + - uses: actions/checkout@v7.0.1 + """ + + assert [%{line: 4, detail: %{denylist_id: "PIN-001"}}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [excision]} = PI.substitute_denylisted_pins(content, @policy) + assert excision.from_ref == @poison + + # Only the site moves: the same 40 hex characters inside `name:` are + # text and stay as written, and the one-site line's label is relabelled. + assert new_content =~ + ~s(- { name: "scan #{@poison}", uses: github/codeql-action/init@#{@good} } # v4.38.0) + + assert PI.pi001_denylisted_pin(new_content, @policy) == [] + end + + test "a multi-site flow line is repaired site by site and its shared label is kept" do + content = + " steps: [ { uses: github/codeql-action/init@#{@poison} }, " <> + "{ uses: actions/checkout@v7.0.1 } ] # v7.0.1\n" + + assert [_] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ "init@#{@good} }, { uses: actions/checkout@v7.0.1 } ] # v7.0.1" + end + + test "a mixed-case action spelling meets its lower-case denylist entry" do + # GitHub resolves `GitHub/CodeQL-Action` to `github/codeql-action`. + content = " - uses: GitHub/CodeQL-Action/init@#{@poison} # v4.38.1\n" + + assert [%{rule_id: "PI001"}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content == " - uses: GitHub/CodeQL-Action/init@#{@good} # v4.38.0\n" + end + test "a file with no denylisted site is returned untouched" do clean = "- uses: actions/checkout@v7.0.1\n" @@ -364,6 +1091,20 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi003_locked_moving_ref(lock, @policy) == [] end + + test "two actions locked at one moving ref are two findings, each pair once" do + lock = """ + '.github/workflows/ci.yml': + - 'dtolnay/rust-toolchain@stable' + - 'some/other-action@stable' + dependencies: + 'dtolnay/rust-toolchain@stable': + """ + + assert [a, b] = PI.pi003_locked_moving_ref(lock, @policy) + assert {a.detail.action, a.line} == {"some/other-action", 3} + assert {b.detail.action, b.line} == {"dtolnay/rust-toolchain", 5} + end end describe "pi004_lock_divergence/3" do @@ -396,6 +1137,65 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi004_lock_divergence(workflow, lock) == [] end + + test "an action sharing its locked ref with another action is still compared" do + # Keyed by ref alone, `github/codeql-action@v4` replaced the checkout + # entry and the divergence below went unreported. + workflow = "- uses: actions/checkout@v3\n" + lock = " - 'actions/checkout@v4'\n - 'github/codeql-action@v4'\n" + + assert [finding] = PI.pi004_lock_divergence(workflow, lock) + assert finding.detail.lock_refs == ["v4"] + end + + test "a commit ID in upper case agrees with the same commit locked in lower case" do + workflow = "- uses: github/codeql-action/init@#{String.upcase(@good)}\n" + lock = " - 'github/codeql-action@#{@good}'\n" + + assert PI.pi004_lock_divergence(workflow, lock) == [] + + # Control: a symbolic ref is a name — `V4` is not the tag `v4`. + assert [_] = + PI.pi004_lock_divergence( + "- uses: actions/checkout@V4\n", + " - 'actions/checkout@v4'\n" + ) + end + + test "with the workflow's path, only that workflow's own lock entry vouches for its pin" do + lock = """ + version: 'v0.0.2' + workflows: + '.github/workflows/a.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/b.yml': + - 'actions/checkout@v6.0.0' + dependencies: + 'actions/checkout@v6.0.0': + ref: 'v6.0.0' + commit: 'sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + owner_id: 44036562 + repo_id: 197814629 + 'actions/checkout@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 + """ + + workflow = "- uses: actions/checkout@v7.0.1\n" + + # `b.yml` locks v6.0.0: v7.0.1, locked for `a.yml`, does not vouch for it. + assert [finding] = + PI.pi004_lock_divergence(workflow, lock, path: ".github/workflows/b.yml") + + assert finding.detail.lock_refs == ["v6.0.0"] + + # Controls: `a.yml` agrees with its own entry; with no path the lock's + # refs are pooled, as before. + assert PI.pi004_lock_divergence(workflow, lock, path: ".github/workflows/a.yml") == [] + assert PI.pi004_lock_divergence(workflow, lock) == [] + end end describe "pi005_stale_pin/2 — advisory, never an action" do @@ -421,4 +1221,40 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi005_stale_pin(content, %{}) == [] end end + + describe "PIN-003 in the committed policy (standards revisions whose lock GitHub rejects)" do + @policy_path Path.expand( + "../../.machine_readable/merge-orchestration/pr-automerge-policy.json", + __DIR__ + ) + @rejected "8f2ee50841e216cd8c192eeb68953118190f105c" + @accepted "75bd5994470661a099df562d6d5ad2fe744dbd66" + @governance "hyperpolymath/standards/.github/workflows/governance-reusable.yml" + + defp committed_policy, do: Jason.decode!(File.read!(@policy_path)) + + test "the entry exists, is keyed on SHA only, and names the known-good revision" do + policy = committed_policy() + entry = Enum.find(policy["pin_denylist"], &(&1["id"] == "PIN-003")) + assert entry["action"] == "hyperpolymath/standards" + assert entry["severity"] == "critical" + assert entry["blocked_versions"] == [] + assert entry["blocked_refs"] == [] + assert entry["known_good_sha"] == @accepted + assert entry["replacement"] == "pin_to_known_good" + assert length(entry["blocked_shas"]) > 500 + assert Enum.all?(entry["blocked_shas"], &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) + refute @accepted in entry["blocked_shas"] + end + + test "a caller pinned inside the window is denylisted through the reusable path; the replacement is not" do + policy = committed_policy() + assert %{"id" => "PIN-003"} = PI.denylisted?(policy, @governance, @rejected) + assert PI.denylisted?(policy, @governance, @accepted) == nil + # The June toolchain window is WF027's, not this entry's: a June revision + # has no lock and must not be denylisted here. + assert PI.denylisted?(policy, @governance, "d135b05bfc647d0c0fbfedc7e80f37ea50f49236") == + nil + end + end end diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index acd3ad97..26c65199 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -112,6 +112,44 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert [delta] = decision.deltas assert delta.source == "body-claim" end + + test "a claim for one action is never borrowed by another of the same basename" do + patch = """ + @@ -1,2 +1,2 @@ + - - uses: acme/setup@v1.0.0 + - - uses: other/setup@v3.0.0 + + - uses: acme/setup@v1.1.0 + + - uses: other/setup@v4.0.0 + """ + + body = "Bumps [acme/setup](https://github.com/acme/setup) from 1.0.0 to 1.1.0.\n" + f = file(".github/workflows/ci.yml", patch) + deltas = PA.pin_deltas([f], %{}, PA.body_claims(body)) + + assert %{status: :ok, source: "body-claim"} = + Enum.find(deltas, &(&1.action == "acme/setup")) + + assert %{status: :unresolved} = Enum.find(deltas, &(&1.action == "other/setup")) + + # A claim naming only the basename belongs to neither action. + bare = PA.body_claims("Bumps [setup](https://example.com) from 1.0.0 to 1.1.0.\n") + assert Enum.all?(PA.pin_deltas([f], %{}, bare), &(&1.status == :unresolved)) + end + + test "a claim is checked against whichever endpoint resolved" do + resolution = %{{"actions/checkout", "v4.1.7"} => "1.0.0"} + + body = + "Bumps [actions/checkout](https://github.com/actions/checkout) from 2.0.0 to 2.1.0.\n" + + f = file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v4.2.0")) + + assert [%{status: :conflict}] = PA.pin_deltas([f], resolution, PA.body_claims(body)) + + # Control: a claim that agrees with the resolved endpoint is taken. + agreeing = String.replace(body, "from 2.0.0 to 2.1.0", "from 1.0.0 to 1.1.0") + assert [%{status: :ok}] = PA.pin_deltas([f], resolution, PA.body_claims(agreeing)) + end end describe "classify/2 — PA002, majors are never armed" do @@ -162,7 +200,53 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert decision.pool == "P1" end - test "poison alongside a wanted update is excised, then merged" do + test "an upper-case spelling of a denylisted SHA is the same poison" do + decision = + Hypatia.Rules.PrAutomerge.classify( + pr([ + file(".github/workflows/codeql.yml", codeql_patch(@good, String.upcase(@poison))) + ]), + @policy + ) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + + test "an unquoted flow-style pin ends at its delimiter: PA001 sees the poison" do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: github/codeql-action/init@#{@good}, with: { languages: rust } } + + - { uses: github/codeql-action/init@#{@poison}, with: { languages: rust } } + """ + + decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + + assert [_] = decision.poison_sites + refute decision.pin_only + # The `with:` beside the pin makes the line more than a pin line, so + # what an excision would leave is not known to be mergeable: flagged. + assert decision.disposition == :flag + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + end + + test "a quoted uses: key and a quoted value are still a pin PA001 sees" do + patch = """ + @@ -1,1 +1,1 @@ + - - "uses": "github/codeql-action/init@#{@good}" + + - "uses": "github/codeql-action/init@#{@poison}" + """ + + decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + + test "poison alongside a change that would be flagged on its own is flagged, not armed" do + # A `permissions:` edit is no pin bump (MGX-001): without the poison this + # PR is not_unambiguously_classifiable, and finding the poison must not + # upgrade it to an armed excise-then-merge. patch = """ @@ -1,4 +1,4 @@ - - uses: github/codeql-action/init@#{@good} # v4.38.0 @@ -174,9 +258,51 @@ defmodule Hypatia.Rules.PrAutomergeTest do decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + assert decision.disposition == :flag + assert decision.safety == "flag" + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + assert decision.remainder_blocked_by == "not_unambiguously_classifiable" + refute decision.pin_only + end + + test "poison alongside a wanted lockfile update is excised, then merged" do + # The workflow's only change is the poisoned pin: excised, it drops out, + # and what remains is a lockfile-only bump that arms on its own. + decision = + PA.classify( + pr([ + file(".github/workflows/codeql.yml", codeql_patch(@good, @poison)), + file("mix.lock", "@@ -1,1 +1,1 @@\n- a 1.0.0\n+ a 1.0.1\n") + ]), + @policy + ) + assert decision.disposition == :excise_poison_then_merge + assert decision.safety == "arm_auto" assert decision.blocked_by == "introduces_denylisted_pin_alongside_wanted_updates" - refute decision.pin_only + end + + test "an unresolved bump sharing a line with a blocked pin, beside a source edit, is flagged" do + # Codex's shape: excision leaves the unresolved ordinary bump and the + # source file, neither of which arms on its own. + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/cache@v4.0.0 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/cache@v4.1.0 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + decision = + PA.classify( + pr([ + file(".github/workflows/ci.yml", patch), + file("lib/runtime.ex", "@@ -1,1 +1,1 @@\n-x = 1\n+x = 2\n") + ]), + @policy + ) + + assert decision.safety == "flag" + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + assert decision.remainder_blocked_by == "pin_delta_unresolvable" end test "poison plus a major bump closes both reasons" do @@ -269,6 +395,45 @@ defmodule Hypatia.Rules.PrAutomergeTest do end describe "poison_sites/2 and pin_lines_only?/1" do + test "a flow pin with neighbouring input, condition or environment edits is not pin-only" do + for {before, after_edit} <- [ + {"with: { languages: rust }", "with: { languages: javascript }"}, + {"if: false", "if: true"}, + {"env: { MODE: test }", "env: { MODE: deploy }"} + ] do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: actions/checkout@v4.1.7, #{before} } + + - { uses: actions/checkout@v4.2.0, #{after_edit} } + """ + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", patch)], + resolution: %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v4.2.0"} => "4.2.0" + } + ), + @policy + ) + + refute decision.pin_only + assert decision.disposition == :flag + end + end + + test "quoted flow pins and their comments are pin-only, punctuation alone is not" do + patch = """ + @@ -1,1 +1,1 @@ + - - [{ "uses": "actions/checkout@v4.1.7" }, { 'uses': 'actions/cache@v4' }] # old + + - [{ "uses": "actions/checkout@v4.2.0" }, { 'uses': 'actions/cache@v5' }] # new + """ + + assert PA.pin_lines_only?(file(".github/workflows/ci.yml", patch)) + refute PA.pin_lines_only?(file(".github/workflows/ci.yml", "@@ -1 +1 @@\n- []\n+ {}\n")) + end + test "the poisoned SHA is found on the added side only" do patch = codeql_patch(@good, @poison) sites = PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) @@ -297,6 +462,411 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) == [] end + + test "a re-pinned value under an unchanged `uses:` key is the site it continues" do + # The key is context; only the reference on the line below it changes. + patch = """ + @@ -1,3 +1,3 @@ + - uses: + # pinned + - github/codeql-action/init@#{@good} + + github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + assert PA.pin_lines_only?(f) + + # Control: the same value line under a key that is not `uses:` is data. + data = String.replace(patch, " - uses:", " - run:") + assert PA.poison_sites([file(".github/workflows/codeql.yml", data)], @policy) == [] + end + + test "every site of a flow-sequence line is inspected, not only the first" do + # The first step is an ordinary resolvable bump; the second is the + # poison. Judging the line by its first pin armed automerge. + patch = """ + @@ -1,1 +1,1 @@ + - { uses: actions/checkout@v4.1.6 }, { uses: github/codeql-action/init@#{@good} }, + + { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@poison} }, + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [site] = PA.poison_sites([f], @policy) + assert site.denylist_id == "PIN-001" + + decision = + PA.classify( + pr([f], + resolution: %{ + {"actions/checkout", "v4.1.6"} => "4.1.6", + {"actions/checkout", "v4.1.7"} => "4.1.7" + } + ), + @policy + ) + + assert decision.pin_only + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + + test "every flow-sequence site is a delta of its own" do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: actions/checkout@v4.1.6 }, { uses: actions/cache@v4.0.0 } + + - { uses: actions/checkout@v4.1.7 }, { uses: actions/cache@v5.0.0 } + """ + + deltas = PA.pin_deltas([file(".github/workflows/ci.yml", patch)], %{}, []) + assert Enum.map(deltas, & &1.action) == ["actions/checkout", "actions/cache"] + end + + test "a mixed-case action spelling meets its lower-case denylist entry" do + patch = + codeql_patch(@good, @poison) + |> String.replace("github/codeql-action", "GitHub/CodeQL-Action") + + assert [site] = PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) + assert site.denylist_id == "PIN-001" + end + + test "a one-line flow sequence with a major bump and a poison pin closes both reasons" do + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/checkout@v7.0.1 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v7.0.1"} => "7.0.1" + } + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", patch)], resolution: resolution), + @policy + ) + + assert decision.major_delta + assert decision.disposition == :close_poison_and_majors + end + + test "a pin written inside a run: | script is text, not a site" do + in_hunk = """ + @@ -1,4 +1,4 @@ + - run: | + cat > fixture.yml <<'EOF' + - - uses: github/codeql-action/init@#{@good} + + - uses: github/codeql-action/init@#{@poison} + EOF + """ + + f = file(".github/workflows/ci.yml", in_hunk) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + + free_text = """ + @@ -9,1 +9,1 @@ + - echo done + + echo { uses: github/codeql-action/init@#{@poison} } + """ + + assert PA.poison_sites([file(".github/workflows/ci.yml", free_text)], @policy) == [] + end + + test "a diff of pin lines that adds or swaps an action is not pin-only, and is not armed" do + added = """ + @@ -3,2 +3,3 @@ + steps: [ + + { uses: owner/action@main }, + { uses: actions/checkout@v7.0.1 }, + """ + + swapped = """ + @@ -1,1 +1,1 @@ + - - uses: actions/checkout@v7.0.1 + + - uses: evil/checkout@v7.0.1 + """ + + for patch <- [added, swapped] do + f = file(".github/workflows/ci.yml", patch) + refute PA.pin_lines_only?(f) + assert PA.classify(pr([f]), @policy).safety == "flag" + end + + # Control: the same action moved to another ref is a re-pin. + assert PA.pin_lines_only?( + file(".github/workflows/ci.yml", checkout_patch("v7.0.0", "v7.0.1")) + ) + end + + test "a pin inside an input's mapping is data, not a site" do + line = "+ with: { uses: github/codeql-action/init@#{@poison} }" + assert PA.parse_pins(line) == [] + end + + test "a pin quoted inside another scalar or a comment is not a site" do + line = + ~s(+ - { name: "see uses: github/codeql-action/init@#{@poison}", uses: actions/checkout@v4.1.7 } # uses: x/y@#{@poison}) + + assert [%{action: "actions/checkout", ref: "v4.1.7"}] = PA.parse_pins(line) + end + + test "repeated sites of one action pair one-to-one, so the second site's major bump counts" do + # The first checkout stays at v4.1.7 and the second moves to v7.0.1. + # Pairing both removals with the first addition (the unchanged one) + # hid the major bump and armed an excision instead of a close. + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/checkout@v4.1.7 }, { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/checkout@v4.1.7 }, { uses: actions/checkout@v7.0.1 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v7.0.1"} => "7.0.1" + } + + f = file(".github/workflows/ci.yml", patch) + + assert [%{action: "actions/checkout", from: "4.1.7", to: "7.0.1", major?: true}] = + f + |> List.wrap() + |> PA.pin_deltas(resolution, []) + |> Enum.filter(&(&1.action == "actions/checkout")) + + decision = PA.classify(pr([f], resolution: resolution), @policy) + assert decision.disposition == :close_poison_and_majors + end + + test "a hunk that opens inside a script, header out of view, flags a poison pin rather than closing" do + # More than three lines below `run: |`, the hunk shows no header: the + # pin may be fixture text. Its neighbours at its own indentation are + # shell, so the context is unknown and the PR goes to an owner. + patch = """ + @@ -40,3 +40,3 @@ + cat > fixture.yml <<'EOF' + - - uses: github/codeql-action/init@#{@good} + + - uses: github/codeql-action/init@#{@poison} + EOF + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: false}] = PA.poison_sites([f], @policy) + refute PA.pin_lines_only?(f) + + decision = PA.classify(pr([f]), @policy) + assert decision.disposition == :flag + assert decision.blocked_by == "denylisted_pin_context_unknown" + + # Control: the same change among YAML siblings is a live pin. + live = """ + @@ -12,3 +12,3 @@ + - name: Init CodeQL + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + with: + """ + + live_decision = PA.classify(pr([file(".github/workflows/ci.yml", live)]), @policy) + assert live_decision.disposition == :close_poison_only + end + + test "an input named uses under a step is data: no poison site, no pin-only change" do + patch = """ + @@ -12,3 +12,3 @@ + - uses: some/wrapper@v1 + with: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/ci.yml", patch) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + refute PA.classify(pr([f]), @policy).disposition == :close_poison_only + + # Control: the step's own key at the same depth is the pin it was. + live = """ + @@ -12,3 +12,3 @@ + - name: init + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + assert PA.classify(pr([file(".github/workflows/ci.yml", live)]), @policy).disposition == + :close_poison_only + end + + test "a re-pinned anchor an unchanged uses: *alias runs is a poison site" do + patch = """ + @@ -1,7 +1,7 @@ + env: + - BAD: &bad github/codeql-action/init@#{@good} + + BAD: &bad github/codeql-action/init@#{@poison} + jobs: + a: + steps: + - uses: *bad + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy) + + decision = PA.classify(pr([f]), @policy) + assert decision.disposition == :flag + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + end + + test "a step given whole as an alias makes its anchored mapping's re-pin a poison site" do + # The mapping is anchored in a matrix entry (data); `- *step` in the + # job's steps runs it. + for {anchor, changed} <- [ + {" - &step", " uses: github/codeql-action/init@"}, + {nil, " - step: &step { uses: github/codeql-action/init@"} + ] do + close = if anchor, do: "", else: " }" + head = if anchor, do: " #{anchor}\n", else: "" + + patch = + "@@ -1,9 +1,9 @@\n jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + head <> + "-#{changed}#{@good}#{close}\n+#{changed}#{@poison}#{close}\n" <> + " steps:\n - *step\n" + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy), patch + end + end + + test "a uses: key under a root data key at column 0 is data: no poison site, no pin-only change" do + patch = """ + @@ -1,3 +1,3 @@ + env: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/ci.yml", patch) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + + # Control: a job's own key is the pin it was. + job = """ + @@ -1,4 +1,4 @@ + jobs: + a: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + assert [%{denylist_id: "PIN-001"}] = + PA.poison_sites([file(".github/workflows/ci.yml", job)], @policy) + end + + test "a re-pinned block scalar a uses: key holds is the site it resolves to" do + patch = """ + @@ -1,3 +1,3 @@ + - uses: >- + - github/codeql-action/init@#{@good} + + github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + assert PA.pin_lines_only?(f) + + # Control: the same block scalar under `run:` is script text. + run = String.replace(patch, "- uses: >-", "- run: >-") + assert PA.poison_sites([file(".github/workflows/codeql.yml", run)], @policy) == [] + refute PA.pin_lines_only?(file(".github/workflows/codeql.yml", run)) + end + + test "a step alias whose mapping names its action by alias makes the anchor's re-pin a poison site" do + patch = """ + @@ -1,11 +1,11 @@ + env: + - BAD: &bad github/codeql-action/init@#{@good} + + BAD: &bad github/codeql-action/init@#{@poison} + jobs: + a: + strategy: + matrix: + include: + - &step + uses: *bad + steps: + - *step + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy) + + # Control: with no step alias the matrix entry, alias and all, is data. + data = String.replace(patch, " - *step", " - run: echo") + assert PA.poison_sites([file(".github/workflows/ci.yml", data)], @policy) == [] + end + + test "a pin written with escapes is the reference it decodes to, and no plain re-pin" do + escaped = "\\x31" <> String.slice(@poison, 1..-1//1) + + patch = """ + @@ -1,3 +1,3 @@ + steps: + - - uses: "github/codeql-action/init@#{@good}" + + - uses: "github/codeql-action/init@#{escaped}" + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + refute PA.pin_lines_only?(f) + + # Control: the same re-pin written plainly is the pin-only change it was. + plain = String.replace(patch, escaped, @poison) + assert PA.pin_lines_only?(file(".github/workflows/ci.yml", plain)) + end + + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do + # The quote opens on a context line and closes inside the hunk: every + # line between is string text, never a site. + closed = """ + @@ -1,7 +1,7 @@ + - run: "cat > fixture.yml < fixture.yml < Enum.filter(fn {repo, _} -> repo == "hyperpolymath/cicd-suite" end) + |> Enum.map(fn {_, path} -> path end) + + for gate <- ~w(main-estate-audit rust-ci creusot-gate spark-gate kanren-gate) do + assert ".github/workflows/#{gate}.yml" in paths + end + end + end + describe "check/1 — sensitivity" do test "fires on a known reusable workflow change" do [finding] = ShaBumpPropagation.check(event(%{})) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs new file mode 100644 index 00000000..7cdfaabb --- /dev/null +++ b/test/rules/workflow_audit_pins_test.exs @@ -0,0 +1,1714 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Rules.WorkflowAuditPinsTest do + use ExUnit.Case, async: true + alias Hypatia.Rules.WorkflowAudit + + # The two anchors of the 2026-09-29..10-14 incident (plasma-parser-writer#105): + # the SHA standards' rust-ci-reusable.yml pinned dtolnay/rust-toolchain at, + # and the standards revision plasma was pinned to. + @toolchain_sha "6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772" + @broken_standards "d135b05bfc647d0c0fbfedc7e80f37ea50f49236" + @good_standards "75bd5994470661a099df562d6d5ad2fe744dbd66" + @rust_ci "hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml" + + # A SHA-pinned toolchain step with no inputs, for a `with:` to follow. + @rust_step "jobs:\n b:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" + + defp wf026(wf), do: WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + describe "check_sha_pinned_toolchain_without_input/1 (WF026)" do + test "fires on a SHA-pinned step with no with: block at all" do + wf = """ + jobs: + check: + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - run: cargo check --locked + """ + + [f] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + assert f.rule == "WF026" + # Advisory: below the CLI's default `--severity medium`, so a scan + # reports and routes it without failing (AGENTIC must-not-promote-to-blocking). + assert f.severity == :low + assert f.advisory == true + assert f.file == "rust.yml" + assert f.line == 5 + assert f.current_sha == @toolchain_sha + assert f.auto_fixable + assert f.fix_script == "fix-rust-toolchain-input.sh" + assert f.reason =~ "'toolchain' is a required input" + end + + test "fires when the with: block names components but no toolchain" do + wf = """ + jobs: + check: + steps: + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@#{@toolchain_sha} # stable + with: + components: clippy, rustfmt + - run: cargo check + """ + + assert [%{rule: "WF026", line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "silent when the step passes a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: stable + components: clippy + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) == [] + end + + test "silent on a tag-pinned step: the ref carries the channel" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: dtolnay/rust-toolchain@1.85.0 + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) == [] + end + + test "a later step's toolchain: does not rescue an earlier bare step" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: stable + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a shallower key ends the step: a job-level toolchain env is not an input" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + env: + toolchain: stable + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "silent when with: is written above uses: (YAML keys carry no order)" do + wf = """ + jobs: + check: + steps: + - with: + toolchain: stable + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "silent on a flow with: that names a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: stable, components: clippy } + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "fires on a flow with: without a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy } + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "env.toolchain is not an input, below or above uses:" do + wf = """ + jobs: + check: + steps: + - name: toolchain via env only + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + env: + toolchain: stable + - env: + toolchain: beta + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + components: clippy + """ + + assert [%{line: 5}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a quoted \"uses\" key is the same key: it fires like the bare one" do + wf = """ + jobs: + check: + steps: + - "uses": "dtolnay/rust-toolchain@#{@toolchain_sha}" + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "the same text inside a run: | script or under env: is not the step's action" do + wf = """ + jobs: + check: + steps: + - name: write a fixture + run: | + cat > fixture.yml <<'EOF' + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + EOF + - uses: dtolnay/rust-toolchain@stable + env: + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a bare - opener is a step: with: above uses: rescues it, a bare uses: still fires" do + wf = """ + jobs: + check: + steps: + - + with: + toolchain: nightly + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 9}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "toolchain: inside another input's block scalar is text, not an input: still fires" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + components: | + toolchain: stable + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a flow with: spread over several lines is read to its closing brace" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + toolchain: nightly, + } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + components: clippy, + } + """ + + assert [%{line: 8}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an expression's braces inside a multi-line flow with: do not close it early" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + components: ${{ matrix.component }}, + toolchain: nightly, + } + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a list item is a step only under steps: — fixtures in env: and run: | are text" do + wf = """ + jobs: + check: + env: + FIXTURE: | + jobs: + a: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - run: | + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 12}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a KYAML workflow is judged as its block equivalent, line for line" do + # The literal shape of `yq -o kyaml` (standards YAML-POLICY Y-3): every + # value quoted, every mapping braced, step items opening with `{`. + wf = """ + { + jobs: { + check: { + runs-on: "ubuntu-latest", + steps: [ + { + uses: "dtolnay/rust-toolchain@#{@toolchain_sha}", + with: { + toolchain: "stable", + }, + }, + { + uses: "dtolnay/rust-toolchain@#{@toolchain_sha}", + }, + { uses: "dtolnay/rust-toolchain@#{@toolchain_sha}" }, + ], + }, + }, + } + """ + + assert [%{line: 13} = a, %{line: 15} = b] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # The repair script rewrites block-style steps only: a KYAML finding is + # real but routes to review, never to the fixer. + refute a.auto_fixable + assert a.fix_script == nil + assert a.reason =~ "KYAML" + refute b.auto_fixable + end + + test "a flow-style step inline in a block document is read through its braces" do + wf = """ + jobs: + check: + steps: + - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } } + - { name: "rust", uses: "dtolnay/rust-toolchain@#{@toolchain_sha}" } + - { with: { components: "clippy, rustfmt" }, uses: dtolnay/rust-toolchain@#{@toolchain_sha} } + - run: echo "{ uses: dtolnay/rust-toolchain@#{@toolchain_sha} }" + """ + + assert [%{line: 5} = a, %{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + assert a.reason =~ "flow style" + end + + test "steps: [ { uses: ... } ] on one line is a step; a flow list in env: is not" do + wf = """ + jobs: + check: + env: + fixture: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + """ + + assert [%{line: 5, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "the action is matched in either case; a spelling the fixer cannot match routes to review" do + wf = """ + jobs: + check: + steps: + - uses: Dtolnay/Rust-Toolchain@#{@toolchain_sha} + """ + + assert [%{rule: "WF026", line: 4, auto_fixable: false, fix_script: nil, reason: reason}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert reason =~ "not spelled" + end + + test "a workflow written in flow style on one line is read through its keys" do + sha2 = String.duplicate("a", 40) + + docs = %{ + "jobs.yml" => + "jobs: { build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "root.yml" => + "{ on: push, jobs: { build: { runs-on: ubuntu-latest, steps: [ " <> + "{ uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } }, " <> + "{ name: \"CI, build\", uses: dtolnay/rust-toolchain@#{sha2} } ] } } }\n", + "jobline.yml" => + "jobs:\n build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] }\n" + } + + findings = WorkflowAudit.check_sha_pinned_toolchain_without_input(docs) + + assert [%{line: 1, auto_fixable: false}] = Enum.filter(findings, &(&1.file == "jobs.yml")) + + assert [%{line: 1, current_sha: ^sha2, auto_fixable: false}] = + Enum.filter(findings, &(&1.file == "root.yml")) + + assert [%{line: 2, auto_fixable: false}] = + Enum.filter(findings, &(&1.file == "jobline.yml")) + end + + test "a one-line flow workflow inside a run heredoc is not a toolchain step" do + for opener <- ["", "{ "], closer = if(opener == "", do: "", else: " }") do + wf = """ + jobs: + build: + steps: + - run: | + cat <<'YAML' + #{opener}jobs: { fixture: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }#{closer} + YAML + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"x.yml" => wf}) == [] + end + end + + test "a one-line flow path that is not steps in a job in jobs is not a step" do + docs = %{ + "env.yml" => + "env: { build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "fixture.yml" => + "jobs: { build: { fixture: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "quoted.yml" => + "jobs: { build: { steps: [ { run: \"jobs: { b: { steps: [ { uses: " <> + "dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\" } ] } }\n" + } + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(docs) == [] + end + + test "a - { item spread over lines: its with entry decides, nested braces respected" do + wf = """ + jobs: + check: + steps: + - { + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + with: { components: "${{ matrix.component }}", toolchain: nightly }, + } + - { + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + with: { components: "${{ matrix.component }}" }, + } + - { + env: { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }, + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + } + """ + + assert [%{line: 9, auto_fixable: false}, %{line: 14, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "with: through an anchor or alias is resolved; the finding is not auto-fixable" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &rust-inputs + toolchain: stable + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *rust-inputs + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &bare-inputs { components: clippy } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *bare-inputs + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *rust-inputs + components: rustfmt + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *bare-inputs + """ + + assert [%{line: 10} = a, %{line: 12} = b, %{line: 18} = c] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + assert a.reason =~ "anchor or alias" + refute b.auto_fixable + # A block `with:` whose merged mapping lacks the input is still the + # fixer's: an explicit `toolchain:` child overrides a merge key, so + # inserting one is a sound repair. + assert c.auto_fixable + end + + test "a merge sequence is resolved alias by alias; a merge it cannot read routes to review" do + wf = """ + x-base: &base { components: clippy } + x-rust: &rust { toolchain: nightly } + x-wasm: &wasm { targets: wasm32-unknown-unknown } + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*base, *rust] + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*base, *wasm] + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: { toolchain: nightly } + """ + + # The first step's channel comes from `*rust`: no finding, and nothing + # for the fixer to override with `stable`. + assert [%{line: 10} = none, %{line: 13} = inline] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert none.auto_fixable + refute inline.auto_fixable + end + + test "in a merge sequence the first mapping that defines toolchain decides, null included" do + wf = """ + x-empty: &empty { toolchain: null } + x-rust: &rust { toolchain: nightly } + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*empty, *rust] + """ + + assert [%{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a toolchain value continued on the next line is the value, not null" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: + nightly + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an anchor written inside a run: | script is text, not the anchor an alias names" do + wf = """ + jobs: + check: + steps: + - run: | + fake: &inputs + toolchain: nightly + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs {} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + """ + + assert [%{line: 7}, %{line: 9}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an anchor written after a tag (`with: !!map &inputs`) is still an anchored mapping" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: !!map &inputs + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs2 !!map + components: clippy + """ + + assert [%{line: 4} = a, %{line: 7} = b] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + refute b.auto_fixable + end + + test "a uses: line inside a quoted scalar spread over lines is string text" do + # The `run:` value opens a double-quoted scalar that runs to `echo end"`; + # the `uses:` line between is part of the string, not the step's key. + quoted = """ + jobs: + check: + steps: + - run: "echo start + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + echo end" + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => quoted}) + + # The same string opened JSON-style in a flow step (`{"run":"echo`), or + # past an anchor (`run: &cmd "echo`), is a string all the same. + for text <- [ + "jobs:\n check:\n steps: [{\"run\":\"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"}]\n", + "jobs:\n check:\n steps: [{\"run\": &cmd \"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"}]\n", + "jobs:\n check:\n steps:\n - run: &cmd \"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n", + # The flow collections open on earlier lines (`steps: [`, `{`). + "jobs:\n check:\n steps: [\n {\n \"run\":\"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n" <> + " }\n ]\n" + ] do + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => text}) + end + + # Controls: a quote closed on its own line, and a quote inside a + # `run: |` script, open nothing — the bare step still fires. + closed = """ + jobs: + check: + steps: + - name: "it's the toolchain" + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + script = """ + jobs: + check: + steps: + - run: | + echo "unterminated + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + for wf <- [closed, script] do + assert [%{rule: "WF026"}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"c.yml" => wf}) + end + end + + test "anchor text inside a quoted scalar or a comment is not the anchor" do + # The quoted `&rust-inputs` comes first; reading children from its line + # found none and reported the aliased step as missing `toolchain`. + wf = """ + env: + NOTE: "see &rust-inputs below" # and &rust-inputs here + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &rust-inputs + toolchain: stable + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *rust-inputs + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an alias with no anchor cannot be read: reported, never passed silently" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *nowhere + """ + + assert [%{line: 4, auto_fixable: false} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "no `&nowhere` anchor" + end + + test "an alias names the latest anchor before it, not the first in the file" do + # YAML lets a document define `&inputs` again; `*inputs` is the + # empty mapping here, so the step passes no toolchain. + wf = """ + defaults: &inputs {toolchain: nightly} + other: &inputs {} + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + """ + + assert [%{line: 6, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # Controls: the other order is configured, and an anchor defined only + # after the alias defines nothing for it. + configured = String.replace(wf, "&inputs {toolchain: nightly}", "&inputs {}", global: false) + + configured = + String.replace(configured, "other: &inputs {}", "other: &inputs {toolchain: nightly}") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => configured}) + + later = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + defaults: &inputs {toolchain: nightly} + """ + + assert [%{line: 4} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => later}) + + assert f.reason =~ "no `&inputs` anchor is defined before it" + end + + test "a uses: value continued below its key, past a comment, is the step's pin" do + wf = """ + jobs: + check: + steps: + - uses: + # pinned + dtolnay/rust-toolchain@#{@toolchain_sha} + - run: cargo test + """ + + assert [%{line: 4, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "continued below its key" + + # Controls: the same step with its input is configured, and the + # reference inside a `run: |` script continues nothing. + with_input = + String.replace( + wf, + " - run: cargo test", + " with: { toolchain: stable }\n - run: cargo test" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + script = """ + jobs: + check: + steps: + - run: | + uses: + dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => script}) + end + + test "a uses: value carrying an anchor or tag is the step's pin; routed to review" do + wf = """ + jobs: + check: + steps: + - uses: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: !!str dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [ + %{line: 4, auto_fixable: false, fix_script: nil} = a, + %{line: 5, auto_fixable: false} = b + ] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert a.reason =~ "anchor or tag" + assert b.reason =~ "anchor or tag" + end + + test "a flow step whose `{` stands on its own line is a flow step; routed to review" do + wf = """ + jobs: + check: + steps: [ + { + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + } + ] + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "flow style" + end + + test "a flow step's uses: continued on the next line is the step's pin" do + wf = """ + jobs: + check: + steps: [ { uses: + dtolnay/rust-toolchain@#{@toolchain_sha} } ] + """ + + assert [%{line: 3, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a uses: alias of an anchored toolchain reference is the step's pin; routed to review" do + wf = """ + x: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + jobs: + check: + steps: + - uses: *rust + - run: cargo test + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.current_sha == @toolchain_sha + assert f.reason =~ "alias of an anchored reference" + + # Controls: with its input the aliased step is configured; an alias + # of another action, or of an anchor defined after it, is not this pin. + with_input = + String.replace(wf, " - run:", " with: { toolchain: stable }\n - run:") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + other = String.replace(wf, "dtolnay/rust-toolchain", "actions/checkout") + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => other}) + + later = """ + jobs: + check: + steps: + - uses: *rust + x: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => later}) + end + + test "a flow step written without indentation, opener by opener, is still a step" do + # Flow content may start in column 0 on every line: indentation says + # nothing, so the ancestry is the openers the lines above left open. + wf = """ + jobs: { + build: { + steps: [ + { + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + } + ] + } + } + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "flow style" + + # Controls: the step's own input configures it, and the same layout + # under `env:` or a job's non-step key is data, not a step. + with_input = + String.replace( + wf, + "rust-toolchain@#{@toolchain_sha}\n", + "rust-toolchain@#{@toolchain_sha},\nwith: { toolchain: stable }\n" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + env = String.replace(wf, "jobs: {", "env: {") + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => env}) + + fixture = String.replace(wf, "steps: [", "fixture: [") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => fixture}) + + # The step's `uses:` after its `{`, or after a sibling entry, on a + # column-0 line is the same step. + for {step, line} <- [ + {"{ uses: dtolnay/rust-toolchain@#{@toolchain_sha} }\n", 4}, + {"{\nname: x, uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n}\n", 5} + ] do + inline = "jobs: {\nbuild: {\nsteps: [\n" <> step <> "]\n}\n}\n" + + assert [%{line: ^line, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => inline}) + + configured = + String.replace( + inline, + "@#{@toolchain_sha}", + "@#{@toolchain_sha}, with: { toolchain: stable }" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => configured + }) + end + + shared = + "jobs: {\nbuild: {\nsteps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ]\n}\n}\n" + + assert [%{line: 3, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => shared}) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => String.replace(shared, "steps: [", "fixture: [") + }) + end + + test "node properties before a step's mapping leave it a flow step" do + for {wf, line} <- [ + {"jobs:\n b:\n steps: [ &step { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ]\n", + 3}, + {"jobs:\n b:\n steps:\n - &step { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }\n", + 4} + ] do + assert [%{line: ^line, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + end + + test "a quoted sibling continued past a `#` line does not hide the step" do + # `# world",` continues the string; it is no comment, and its quote + # closes the scalar before the step's own `uses:`. + wf = + "jobs: {\nb: {\nsteps: [\n{\nname: \"hello\n# world\",\n" <> + "uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n}\n]\n}\n}\n" + + assert [%{line: 7, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # The same quote closing on the `uses:` line's own start. + inline = + "jobs:\n b:\n steps: [{ name: \"hello\n world\", " <> + "uses: dtolnay/rust-toolchain@#{@toolchain_sha} }]\n" + + assert [%{line: 4, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => inline}) + end + + test "a bare sequence item's anchor holds the mapping under its dash" do + item = + "x:\n - &inputs\n toolchain: nightly\n" <> @rust_step <> " with: *inputs\n" + + assert [] == wf026(item) + + unconfigured = + "x:\n - &inputs\n components: clippy\n" <> @rust_step <> " with: *inputs\n" + + assert [%{line: 7}] = wf026(unconfigured) + end + + test "an & inside a plain scalar is text, not the anchor an alias names" do + # `*channel` is the empty value anchored before `foo&channel`. + text = + "x: &channel \"\"\ny: foo&channel nightly\n" <> + @rust_step <> " with:\n toolchain: *channel\n" + + assert [%{line: 6}] = wf026(text) + end + + test "a block input's alias names the anchor on a child line above it" do + sibling = + @rust_step <> + " with:\n source: &channel nightly\n toolchain: *channel\n" + + assert [] == wf026(sibling) + end + + test "an empty block scalar toolchain is the empty string; one with content is set" do + empty = + @rust_step <> " with:\n toolchain: >-\n components: clippy\n" + + assert [%{line: 4}] = wf026(empty) + + folded = @rust_step <> " with:\n toolchain: >-\n stable\n" + assert [] == wf026(folded) + end + + test "toolchain: *alias is the scalar its anchor holds" do + step = fn tc -> + "x: &empty \"\"\ny: &chan stable\njobs:\n b:\n steps:\n" <> + " - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n with:\n" <> + " toolchain: #{tc}\n" + end + + # An empty anchored value is no input, as `toolchain: ""` is not. + assert [%{line: 6, auto_fixable: true}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*empty") + }) + + # An alias of no anchor cannot be read: reported, routed to review. + assert [%{line: 6, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*nope") + }) + + assert f.reason =~ "`toolchain:` input is the alias `*nope`" + + # Control: an anchored channel is the input. + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*chan") + }) + + flow = + "x: &empty ~\njobs:\n b:\n steps:\n" <> + " - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: *empty } }\n" + + assert [%{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => flow}) + end + + test "a flow step's alias names the anchor before it, not one later on its line" do + # `*inputs` is the empty mapping defined first; the configured + # `&inputs` after it on the same line is not what it names. + wf = """ + x: &inputs {} + jobs: + check: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: *inputs }, { run: echo, env: &inputs {toolchain: nightly} } ] + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # Control: with the configured anchor first, the alias names it. + configured = String.replace(wf, "x: &inputs {}", "x: &inputs {toolchain: stable}") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => configured}) + end + + test "node properties on a toolchain value are not the value" do + # An anchor names the node and `!!str ""` is still the empty string: + # none of these is an input. `!!str null` is the string null, one. + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: &channel "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: !!str "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: &channel null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: !!str null + """ + + assert [%{line: 4}, %{line: 7}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a toolchain value of nothing but an anchor is null, or the value below it" do + bare = @rust_step <> " with:\n toolchain: &channel\n" + assert [%{line: 4}] = wf026(bare) + + # Control: the anchored scalar continued on the line below is the input. + assert [] == wf026(bare <> " nightly\n") + end + + test "an anchored toolchain value keeps its tag through an alias" do + aliased = fn value -> + "x: &e #{value}\n" <> @rust_step <> " with:\n toolchain: *e\n" + end + + # `!!str null` is the string null, an input; `!!str ""` is still empty. + assert [] == wf026(aliased.("!!str null")) + assert [%{line: 5}] = wf026(aliased.(~s(!!str ""))) + end + + test "an escaped quote in a flow step's name hides none of its entries" do + # `"a\"b, c"` is one scalar: `with: *inputs` is the step's entry and + # names the empty mapping before it, not the configured `&inputs` + # defined later on the same line. + wf = """ + x: &inputs {} + jobs: + check: + steps: [ { name: "a\\"b, c", uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: *inputs }, { run: echo, env: &inputs {toolchain: nightly} } ] + """ + + assert [%{line: 4}] = wf026(wf) + + # Control: with the configured anchor first, the alias names it. + assert [] == wf026(String.replace(wf, "x: &inputs {}", "x: &inputs {toolchain: stable}")) + end + + test "a steps: list some other mapping holds is data, not a job's steps" do + wf = """ + jobs: + check: + strategy: + matrix: + include: + - fixture: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - run: echo + """ + + assert [] == wf026(wf) + + # Control: the job's own steps. + assert [%{line: 4}] = wf026(@rust_step) + end + + test "an alias name runs to whitespace or a flow indicator" do + wf = + "x: &e/x \"\"\n" <> @rust_step <> " with:\n toolchain: *e/x\n" + + assert [%{line: 5, auto_fixable: true}] = wf026(wf) + end + + test "an anchor directly after a quoted key's colon in a flow mapping is the anchor" do + wf = + "x: {\"e\":&e \"\"}\n" <> @rust_step <> " with:\n toolchain: *e\n" + + # The anchor is read, so the empty value it holds routes to the fixer + # rather than to review as an alias of no anchor. + assert [%{line: 5, auto_fixable: true}] = wf026(wf) + end + + test "a step given whole as an alias is judged where its mapping is anchored" do + wf = """ + jobs: + check: + strategy: + matrix: + include: + - &step + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - *step + """ + + # The anchored mapping is matrix data until the alias runs it: then it + # is a live step with no input, routed to review (not the fixer). + assert [%{line: 7, auto_fixable: false} = f] = wf026(wf) + assert f.reason =~ "given as an alias" + + # Control: with no alias, the matrix entry is data. + assert [] == wf026(String.replace(wf, " - *step\n", " - run: echo\n")) + end + + test "a step alias whose mapping names the action by alias is judged at its uses: key" do + wf = """ + env: + TC: &tc dtolnay/rust-toolchain@#{@toolchain_sha} + jobs: + check: + strategy: + matrix: + include: + - &step + uses: *tc + steps: + - *step + """ + + assert [%{line: 9, auto_fixable: false}] = wf026(wf) + assert [] == wf026(String.replace(wf, " - *step\n", " - run: echo\n")) + end + + test "a uses: block scalar whose one line is the reference is the step's key; routed to review" do + wf = """ + jobs: + check: + steps: + - uses: >- + dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 4, auto_fixable: false} = f] = wf026(wf) + assert f.reason =~ "continued below its key" + + # Control: with the input given, the block-scalar step is silent. + assert [] == wf026(wf <> " with:\n toolchain: stable\n") + end + + test "a toolchain: key with no value is no input: null and ~ included" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy, toolchain: null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: ~ # chosen later + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: "1.85.0" } + """ + + assert [%{line: 4, auto_fixable: true}, %{line: 8, auto_fixable: true}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "YAML null is unquoted: \"null\" is a string, Null/NULL/\"\"/'' are not inputs" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: "null" } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: Null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: NULL + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: '' } + """ + + assert [%{line: 6}, %{line: 8}, %{line: 11}, %{line: 14}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an explicit toolchain: key wins over a merged anchor, empty or not" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs { toolchain: stable, components: clippy } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *inputs + toolchain: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs, toolchain: null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs, toolchain: nightly } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs } + """ + + assert [%{line: 6, auto_fixable: true}, %{line: 10, auto_fixable: true}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "every flow-style step sharing a line is judged, not only the first" do + wf = """ + jobs: + check: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } }, { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + other: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }, { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { components: clippy } } ] + """ + + assert [%{line: 3, auto_fixable: false}, %{line: 5}, %{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "upper-case hex is the same commit: it fires like lower-case" do + sha = String.upcase(@toolchain_sha) + + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{sha} + - { uses: dtolnay/rust-toolchain@#{sha} } + """ + + assert [%{line: 4, current_sha: ^sha}, %{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a tag that merely begins with a SHA is a tag: it carries its channel" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha}-patched + - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}-patched } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} # pinned + """ + + assert [%{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a plain block-style finding is still the fixer's" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy } # a flow mapping the fixer extends + """ + + assert [%{line: 4, auto_fixable: true, fix_script: "fix-rust-toolchain-input.sh"}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "the audit carries WF026 findings and a count" do + wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" + result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) + assert Enum.any?(result.findings, &(&1[:rule] == "WF026")) + assert result.toolchain_input_missing_count == 1 + end + end + + describe "check_known_broken_reusable_revision/1 (WF027)" do + test "fires on rust-ci-reusable at a revision inside the toolchain window" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" + [f] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + assert f.rule == "WF027" + assert f.severity == :low + assert f.advisory == true + assert f.reusable == @rust_ci + assert f.current_sha == @broken_standards + assert f.replacement_sha == @good_standards + assert f.auto_fixable + assert f.fix_script == "fix-stale-reusable-pin.sh" + assert f.reason =~ "standards#439" + end + + test "silent at the replacement revision" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@good_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) == [] + end + + test "a job's uses: value continued below its key is the call; routed to review" do + # The fleet fixer refuses a continued value (exit 2), so the finding + # is not handed to it. + wf = + "jobs:\n rust-ci:\n uses:\n\n # dead window\n #{@rust_ci}@#{@broken_standards}\n" + + assert [%{line: 3, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.current_sha == @broken_standards + assert f.reason =~ "continued below its key" + + # Control: under `env:` the same layout is data, not a call. + env = "env:\n CALL:\n #{@rust_ci}@#{@broken_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => env}) == [] + end + + test "a job's uses: with an anchor or tag, or a flow-style job, is the call; routed to review" do + # The fleet fixer refuses all three (exit 2), so none is handed to it. + wf = + "jobs:\n a:\n uses: !!str #{@rust_ci}@#{@broken_standards}\n" <> + " b:\n uses: &call #{@rust_ci}@#{@broken_standards}\n" <> + " c: { uses: #{@rust_ci}@#{@broken_standards} }\n" + + assert [ + %{line: 3, auto_fixable: false, fix_script: nil} = a, + %{line: 5, auto_fixable: false} = b, + %{line: 6, auto_fixable: false} = c + ] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert a.reason =~ "anchor or tag" + assert b.reason =~ "anchor or tag" + assert c.reason =~ "flow mapping" + + # Control: the plain block call is still handed to the fixer. + plain = "jobs:\n a:\n uses: #{@rust_ci}@#{@broken_standards}\n" + + assert [%{auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) + end + + test "a job's uses: alias of an anchored reusable reference is the call; routed to review" do + wf = + "x: &call #{@rust_ci}@#{@broken_standards}\n" <> + "jobs:\n rust-ci:\n uses: *call\n" + + assert [%{line: 4, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.current_sha == @broken_standards + assert f.reason =~ "alias of an anchored reference" + + # Control: an alias of the replacement revision is silent. + good = String.replace(wf, @broken_standards, @good_standards) + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => good}) == [] + end + + test "a job given whole as an alias calls the reusable its anchored mapping names; routed to review" do + for template <- [ + "template: &job\n uses: #{@rust_ci}@#{@broken_standards}\n", + "template: &job { uses: #{@rust_ci}@#{@broken_standards} }\n" + ] do + wf = template <> "jobs:\n rust: *job\n" + + assert [%{auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "given as an alias" + + # Control: with no job alias the anchored mapping is data. + plain = template <> "jobs:\n rust:\n uses: #{@rust_ci}@#{@good_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) == [] + end + + # A job anchored under `jobs:` and aliased is still the fixer's: the + # anchored job's own key is the one it rewrites. + own = "jobs:\n base: &job\n uses: #{@rust_ci}@#{@broken_standards}\n rust: *job\n" + + assert [%{line: 3, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => own}) + end + + test "a job's uses: block scalar whose one line is the call is the call; routed to review" do + wf = "jobs:\n rust-ci:\n uses: >-\n #{@rust_ci}@#{@broken_standards}\n" + + assert [%{line: 3, auto_fixable: false} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "continued below its key" + end + + test "a flow-style job written without indentation is still a job; routed to review" do + # Every line in column 0: the brackets, not the indentation, say the + # `uses:` is a job's key. The fleet fixer places a key by indentation + # and cannot place these, so none is handed to it. + own_line = "jobs: {\nrust: {\nuses: #{@rust_ci}@#{@broken_standards}\n}\n}\n" + inline = "jobs: {\nrust: { uses: #{@rust_ci}@#{@broken_standards} }\n}\n" + + sibling = + "jobs: {\na: { runs-on: x }, rust: { uses: #{@rust_ci}@#{@broken_standards} }\n}\n" + + for {wf, line} <- [{own_line, 3}, {inline, 2}, {sibling, 2}] do + assert [%{line: ^line, current_sha: @broken_standards, auto_fixable: false} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "flow mapping" + end + + # Controls: indented as its nesting reads, the own-line key is one the + # fixer rewrites; under `env:`, or inside a job's `with: { }`, the + # same layout is data, not a call. + indented = "jobs: {\n rust: {\n uses: #{@rust_ci}@#{@broken_standards}\n }\n}\n" + + assert [%{auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => indented}) + + for wf <- [own_line, inline], + data <- [ + String.replace(wf, "jobs: {", "env: {"), + String.replace(wf, "rust: {", "rust: {\nwith: {") <> "}\n" + ] do + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => data}) == [] + end + end + + test "a job's mapping carrying node properties, or past a `#` string line, is the call" do + anchored = "jobs: { rust: &job { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{line: 1, auto_fixable: false}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => anchored}) + + quoted = + "jobs: {\nrust: {\nname: \"hello\n# world\",\nuses: #{@rust_ci}@#{@broken_standards}\n}\n}\n" + + assert [%{line: 5, auto_fixable: false}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => quoted}) + + # Control: a `uses:` line inside that string is text. + text = + "jobs: {\nrust: {\nname: \"hello\nuses: #{@rust_ci}@#{@broken_standards}\n\",\n}\n}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => text}) == [] + end + + test "a flow-style job's uses: continued on the next line is the call" do + wf = "jobs: { rust: { uses:\n #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{line: 1, auto_fixable: false, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + end + + test "silent for another reusable at the same revision: the window is per reusable" do + wf = + "jobs:\n mirror:\n uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@#{@broken_standards}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"mirror.yml" => wf}) == [] + end + + test "a commented-out pin does not fire" do + wf = + "jobs:\n rust-ci:\n # uses: #{@rust_ci}@#{@broken_standards}\n uses: #{@rust_ci}@#{@good_standards}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) == [] + end + + test "the table is the measured window: 83 revisions, edges included, replacement excluded" do + [row] = WorkflowAudit.broken_reusable_revisions() + assert row.reusable == @rust_ci + assert length(row.shas) == 83 + assert "d73c93e0e7d38f673b68e616d51a62f5860081fd" in row.shas + assert "49ed4715" |> then(&Enum.any?(row.shas, fn s -> String.starts_with?(s, &1) end)) + refute row.replacement in row.shas + assert Enum.all?(row.shas, &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) + end + + test "the path inside a run: script or a string value is not a pin" do + wf = """ + jobs: + note: + steps: + - run: echo "uses: #{@rust_ci}@#{@broken_standards}" + - name: "uses: #{@rust_ci}@#{@broken_standards}" + run: true + """ + + assert [] = WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "a quoted uses: key with a quoted value still fires" do + wf = """ + jobs: + rust: + "uses": "#{@rust_ci}@#{@broken_standards}" + """ + + assert [%{rule: "WF027", current_sha: @broken_standards, line: 3}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "only a key of a job is a call: run: | and env: copies are not" do + wf = """ + jobs: + fixture: + env: + uses: #{@rust_ci}@#{@broken_standards} + steps: + - run: | + cat > caller.yml <<'EOF' + uses: #{@rust_ci}@#{@broken_standards} + EOF + rust-ci: + uses: #{@rust_ci}@#{@broken_standards} + """ + + assert [%{rule: "WF027", line: 11}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "a tag that merely begins with a broken SHA is a tag, not the pin" do + wf = """ + jobs: + rust-ci: + uses: #{@rust_ci}@#{@broken_standards}-patched + other: + uses: #{@rust_ci}@#{@broken_standards} + """ + + assert [%{rule: "WF027", line: 5}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "upper-case hex is the same revision: it fires, reported as written" do + sha = String.upcase(@broken_standards) + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{sha}\n" + + assert [%{rule: "WF027", current_sha: ^sha, replacement_sha: @good_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "a job-level uses: inside a quoted scalar spread over lines is string text" do + wf = """ + jobs: + rust-ci: + name: "first line + uses: #{@rust_ci}@#{@broken_standards} + last line" + """ + + assert [] == WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "the owner and repository match in either case; the workflow path as written" do + # GitHub resolves `Hyperpolymath/Standards` to the same repository. + mixed = "Hyperpolymath/Standards/.github/workflows/rust-ci-reusable.yml" + wf = "jobs:\n rust-ci:\n uses: #{mixed}@#{@broken_standards}\n" + + assert [%{rule: "WF027", line: 3, replacement_sha: @good_standards, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + + inline = "jobs: { rust: { uses: #{mixed}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => inline}) + + # A file path inside the repository is case-sensitive: another file. + other = "hyperpolymath/standards/.github/workflows/Rust-CI-Reusable.yml" + wf2 = "jobs:\n rust-ci:\n uses: #{other}@#{@broken_standards}\n" + assert [] == WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf2}) + end + + test "a job written as a flow mapping is a call: on its own line or inside jobs: { }" do + wf = """ + jobs: + rust-ci: { uses: #{@rust_ci}@#{@broken_standards}, secrets: inherit } + note: + steps: + - run: echo "{ uses: #{@rust_ci}@#{@broken_standards} }" + """ + + assert [%{rule: "WF027", line: 2, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + + one_line = + "jobs: { a: { uses: #{@rust_ci}@#{@good_standards} }, b: { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => one_line}) + + nested_under_env = + "jobs:\n fixture:\n env: { uses: #{@rust_ci}@#{@broken_standards} }\n" + + assert [] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => nested_under_env}) + end + + test "a quoted comma or brace in a sibling entry does not hide a flow-style job" do + one_line = + "jobs: { a: { name: \"x, y }\", uses: #{@rust_ci}@#{@good_standards} }, " <> + "b: { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => one_line}) + + own_line = """ + jobs: + rust-ci: { name: "CI, build", uses: #{@rust_ci}@#{@broken_standards} } + note: + env: { name: "x, y", uses: #{@rust_ci}@#{@broken_standards} } + """ + + assert [%{rule: "WF027", line: 2}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => own_line}) + end + + test "a one-line flow workflow inside a run heredoc is not a reusable call" do + for opener <- ["", "{ "], closer = if(opener == "", do: "", else: " }") do + wf = """ + jobs: + build: + steps: + - run: | + cat <<'YAML' + #{opener}jobs: { fixture: { uses: #{@rust_ci}@#{@broken_standards} } }#{closer} + YAML + """ + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) == [] + end + end + + test "a root-braced one-line workflow is read through its keys" do + root = + "{ on: push, jobs: { rust: { uses: #{@rust_ci}@#{@broken_standards} }, " <> + "build: { steps: [ { uses: #{@rust_ci}@#{@broken_standards} } ] } } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => root}) + end + + test "a KYAML caller is read as written: the quoted job-level uses: fires, fixtures do not" do + wf = """ + { + name: "Rust", + on: { push: { branches: ["main"] } }, + jobs: { + note: { + "runs-on": "ubuntu-latest", + env: { FIXTURE: "uses: #{@rust_ci}@#{@broken_standards}" }, + steps: [{ + run: "echo uses: #{@rust_ci}@#{@broken_standards}", + }], + }, + "rust-ci": { + uses: "#{@rust_ci}@#{@broken_standards}", + secrets: "inherit", + }, + }, + } + """ + + assert [%{rule: "WF027", line: 13, current_sha: @broken_standards, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "the audit carries WF027 findings and a count" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" + result = WorkflowAudit.audit(["rust-ci.yml"], %{"rust-ci.yml" => wf}) + assert Enum.any?(result.findings, &(&1[:rule] == "WF027")) + assert result.broken_reusable_revision_count == 1 + end + end +end