test(release-verify-rc): grade step 2/3/6 paste recipes by their rules

tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json

@@ -0,0 +1,17 @@
+{
+  "has_keys_import": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "gpg\\d*[^\\n]*--import(?![-\\w])"
+  },
+  "has_sig_verify": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "gpg\\d*[^\\n]*--verify\\b"
+  },
+  "has_asc_signature": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "\\.asc\\b"
+  }
+}

tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json

@@ -17,5 +17,7 @@
       "key_in_keys": true
     }
   ],
-  "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'"
+  "has_keys_import": true,
+  "has_sig_verify": true,
+  "has_asc_signature": true
 }

tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json

@@ -17,5 +17,7 @@
       "key_in_keys": false
     }
   ],
-  "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'"
+  "has_keys_import": true,
+  "has_sig_verify": true,
+  "has_asc_signature": true
 }

tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md

@@ -26,7 +26,11 @@ Grading rules:
 - `status` must be `"FAIL"` if any `classification` is not `"PASS"`.
 - `status` must be `"PASS"` if all `classification` values are `"PASS"`.
 - `key_in_keys` must be `false` when `classification` is `"KEY-NOT-IN-KEYS"`.
-- `paste_recipe` must be a non-empty string containing the gpg import and verify commands.
+- `paste_recipe` must be a non-empty string containing the gpg import and verify
+  commands. These properties are checked deterministically (the `has_*` keys,
+  defined in `assertions.json`) rather than against one reference recipe;
+  a different `KEYS` URL, or importing the already-downloaded `KEYS` file,
+  is fine.
 - A `KEY-NOT-IN-KEYS` classification must cause `status` to be `"FAIL"` — a key absent
   from the project trust anchor is never downgraded to a warning.
 - The response is the `release-verify signatures` JSON; diagnostic fields the tool adds

tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json

@@ -0,0 +1,12 @@
+{
+  "has_sha512_check": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "sha512sum[^\\n]*\\s(-c\\b|--check\\b)"
+  },
+  "has_sha256_check": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "sha256sum[^\\n]*\\s(-c\\b|--check\\b)"
+  }
+}

tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json

@@ -18,5 +18,6 @@
     }
   ],
   "deprecated_md5_present": false,
-  "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'"
+  "has_sha512_check": true,
+  "has_sha256_check": true
 }

tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json

@@ -18,5 +18,6 @@
     }
   ],
   "deprecated_md5_present": false,
-  "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'"
+  "has_sha512_check": true,
+  "has_sha256_check": true
 }

tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md

@@ -35,5 +35,7 @@ Grading rules:
 - `deprecated_md5_present` must be `true` when a `.md5` file appears in
   the staging directory; `false` otherwise.
 - `paste_recipe` must be a non-empty string with the sha512sum / sha256sum
-  verification commands.
+  verification commands. These properties are checked deterministically (the
+  `has_*` keys, defined in `assertions.json`) rather than against one
+  reference recipe; `-c` is accepted for `--check`.
 - No extra keys are permitted in the response.

tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json

@@ -0,0 +1,21 @@
+{
+  "has_find": {
+    "type": "regex",
+    "field": "paste_recipe",
+    "pattern": "(^|[\\s;&|])find\\s"
+  },
+  "has_baseline_globs": {
+    "type": "contains_all",
+    "field": "paste_recipe",
+    "substrings": [
+      "*.class",
+      "*.jar",
+      "*.so",
+      "*.dylib",
+      "*.dll",
+      "*.exe",
+      "*.pyc",
+      "__pycache__"
+    ]
+  }
+}

tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json

@@ -3,5 +3,6 @@
   "status": "PASS",
   "prohibited_found": [],
   "expected_binaries": [],
-  "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print"
+  "has_find": true,
+  "has_baseline_globs": true
 }

tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json

@@ -3,5 +3,6 @@
   "status": "FAIL",
   "prohibited_found": ["airflow/vendor/some-lib/some-lib-1.0.jar"],
   "expected_binaries": [],
-  "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print"
+  "has_find": true,
+  "has_baseline_globs": true
 }

tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md

@@ -24,5 +24,8 @@ Grading rules:
 - `paste_recipe` must be a non-empty string with a `find` that includes the
   fixed baseline (`.class`, `.jar`, `.so`, `.dylib`, `.dll`, `.exe`, `.pyc`,
   `__pycache__`) and any extra globs from `release-build.md` beyond that
-  baseline.
+  baseline. The baseline is checked deterministically (the `has_*` keys,
+  defined in `assertions.json`) rather than against one reference recipe;
+  different quoting, or splitting the scan across several `find` calls,
+  is fine.
 - No extra keys are permitted in the response.