diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json new file mode 100644 index 000000000..351af49bc --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json @@ -0,0 +1,17 @@ +{ + "has_keys_import": { + "type": "regex", + "field": "paste_recipe", + "pattern": "gpg\\d*[^\\n]*--import(?![-\\w])" + }, + "has_sig_verify": { + "type": "regex", + "field": "paste_recipe", + "pattern": "gpg\\d*[^\\n]*--verify\\b" + }, + "has_asc_signature": { + "type": "regex", + "field": "paste_recipe", + "pattern": "\\.asc\\b" + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json index 0dbeb60a1..ccd219e19 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json @@ -17,5 +17,7 @@ "key_in_keys": true } ], - "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'" + "has_keys_import": true, + "has_sig_verify": true, + "has_asc_signature": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json index 176f44af8..c2c69c2f3 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json @@ -17,5 +17,7 @@ "key_in_keys": false } ], - "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'" + "has_keys_import": true, + "has_sig_verify": true, + "has_asc_signature": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md index 8864b859a..b489e5880 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md @@ -26,7 +26,11 @@ Grading rules: - `status` must be `"FAIL"` if any `classification` is not `"PASS"`. - `status` must be `"PASS"` if all `classification` values are `"PASS"`. - `key_in_keys` must be `false` when `classification` is `"KEY-NOT-IN-KEYS"`. -- `paste_recipe` must be a non-empty string containing the gpg import and verify commands. +- `paste_recipe` must be a non-empty string containing the gpg import and verify + commands. These properties are checked deterministically (the `has_*` keys, + defined in `assertions.json`) rather than against one reference recipe; + a different `KEYS` URL, or importing the already-downloaded `KEYS` file, + is fine. - A `KEY-NOT-IN-KEYS` classification must cause `status` to be `"FAIL"` — a key absent from the project trust anchor is never downgraded to a warning. - The response is the `release-verify signatures` JSON; diagnostic fields the tool adds diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json new file mode 100644 index 000000000..584942c02 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json @@ -0,0 +1,12 @@ +{ + "has_sha512_check": { + "type": "regex", + "field": "paste_recipe", + "pattern": "sha512sum[^\\n]*\\s(-c\\b|--check\\b)" + }, + "has_sha256_check": { + "type": "regex", + "field": "paste_recipe", + "pattern": "sha256sum[^\\n]*\\s(-c\\b|--check\\b)" + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json index e78c5b43d..d1e4a8cda 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json @@ -18,5 +18,6 @@ } ], "deprecated_md5_present": false, - "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'" + "has_sha512_check": true, + "has_sha256_check": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json index b9445d91b..ad7b5b103 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json @@ -18,5 +18,6 @@ } ], "deprecated_md5_present": false, - "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'" + "has_sha512_check": true, + "has_sha256_check": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md index 1d350a916..1587ee9c1 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md @@ -35,5 +35,7 @@ Grading rules: - `deprecated_md5_present` must be `true` when a `.md5` file appears in the staging directory; `false` otherwise. - `paste_recipe` must be a non-empty string with the sha512sum / sha256sum - verification commands. + verification commands. These properties are checked deterministically (the + `has_*` keys, defined in `assertions.json`) rather than against one + reference recipe; `-c` is accepted for `--check`. - No extra keys are permitted in the response. diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json new file mode 100644 index 000000000..716b01306 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json @@ -0,0 +1,21 @@ +{ + "has_find": { + "type": "regex", + "field": "paste_recipe", + "pattern": "(^|[\\s;&|])find\\s" + }, + "has_baseline_globs": { + "type": "contains_all", + "field": "paste_recipe", + "substrings": [ + "*.class", + "*.jar", + "*.so", + "*.dylib", + "*.dll", + "*.exe", + "*.pyc", + "__pycache__" + ] + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json index bc5ad6a7c..900aff357 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json @@ -3,5 +3,6 @@ "status": "PASS", "prohibited_found": [], "expected_binaries": [], - "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print" + "has_find": true, + "has_baseline_globs": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json index 4c1e0f0f4..7b8f9b6f2 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json @@ -3,5 +3,6 @@ "status": "FAIL", "prohibited_found": ["airflow/vendor/some-lib/some-lib-1.0.jar"], "expected_binaries": [], - "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print" + "has_find": true, + "has_baseline_globs": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md index feee75e1d..f4955ec25 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md @@ -24,5 +24,8 @@ Grading rules: - `paste_recipe` must be a non-empty string with a `find` that includes the fixed baseline (`.class`, `.jar`, `.so`, `.dylib`, `.dll`, `.exe`, `.pyc`, `__pycache__`) and any extra globs from `release-build.md` beyond that - baseline. + baseline. The baseline is checked deterministically (the `has_*` keys, + defined in `assertions.json`) rather than against one reference recipe; + different quoting, or splitting the scan across several `find` calls, + is fine. - No extra keys are permitted in the response.