From b7a15f2f5874d0ced83bc34b2e77592aea673557 Mon Sep 17 00:00:00 2001 From: liwenjie <3133746534@qq.com> Date: Tue, 6 Oct 2026 11:41:53 +0800 Subject: [PATCH 1/2] test(release-verify-rc): grade step 2/3/6 paste recipes by their rules The last three release-verify-rc suites still grade `paste_recipe` against one reference recipe: Step 2 (signatures), Step 3 (checksums) and Step 6 (binary exclusion). Correct answers fail for quoting the gpg arguments differently, importing the already-downloaded KEYS file, passing `-c` for `--check`, or splitting the binary scan across several find calls, while the step only requires properties of the recipe. Replace the exact `paste_recipe` in every case with structural checks in a new `assertions.json`, encoding the output-spec rules: Step 2 - a gpg key import and a `gpg --verify` of the staged `.asc` signatures; Step 3 - `sha512sum`/`sha256sum` in check mode (`-c` accepted for `--check`); Step 6 - a `find` whose globs cover the eight-entry fixed baseline. Every original reference recipe satisfies them. Same treatment as the Step 6c suite in PR #1505 and the Step 6b suite in PR #1536. Generated-by: ZCode (GLM-5.3-Flash) --- .../fixtures/assertions.json | 12 +++++++++++ .../case-1-all-sigs-pass/expected.json | 3 ++- .../case-2-tampered-sig/expected.json | 3 ++- .../fixtures/output-spec.md | 6 +++++- .../fixtures/assertions.json | 12 +++++++++++ .../case-1-checksums-pass/expected.json | 3 ++- .../case-2-checksum-mismatch/expected.json | 3 ++- .../fixtures/output-spec.md | 4 +++- .../fixtures/assertions.json | 21 +++++++++++++++++++ .../expected.json | 3 ++- .../expected.json | 3 ++- .../fixtures/output-spec.md | 5 ++++- 12 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json create mode 100644 tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json create mode 100644 tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json new file mode 100644 index 000000000..e473b0f40 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json @@ -0,0 +1,12 @@ +{ + "has_keys_import": { + "type": "regex", + "field": "paste_recipe", + "pattern": "gpg\\d*[^\\n]*--import(?![-\\w])" + }, + "has_sig_verify": { + "type": "regex", + "field": "paste_recipe", + "pattern": "gpg\\d*[^\\n]*--verify\\b[^\\n]*\\.asc" + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json index 0dbeb60a1..4cfe77aef 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json @@ -17,5 +17,6 @@ "key_in_keys": true } ], - "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'" + "has_keys_import": true, + "has_sig_verify": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json index 176f44af8..3336c984b 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json @@ -17,5 +17,6 @@ "key_in_keys": false } ], - "paste_recipe": "curl -s 'https://dist.apache.org/repos/dist/release/airflow/KEYS' | gpg --import\ngpg --verify 'apache-airflow-2.11.0-source-release.tar.gz.asc' 'apache-airflow-2.11.0-source-release.tar.gz'\ngpg --verify 'apache-airflow-2.11.0-bin.tar.gz.asc' 'apache-airflow-2.11.0-bin.tar.gz'" + "has_keys_import": true, + "has_sig_verify": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md index 8864b859a..b489e5880 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/output-spec.md @@ -26,7 +26,11 @@ Grading rules: - `status` must be `"FAIL"` if any `classification` is not `"PASS"`. - `status` must be `"PASS"` if all `classification` values are `"PASS"`. - `key_in_keys` must be `false` when `classification` is `"KEY-NOT-IN-KEYS"`. -- `paste_recipe` must be a non-empty string containing the gpg import and verify commands. +- `paste_recipe` must be a non-empty string containing the gpg import and verify + commands. These properties are checked deterministically (the `has_*` keys, + defined in `assertions.json`) rather than against one reference recipe; + a different `KEYS` URL, or importing the already-downloaded `KEYS` file, + is fine. - A `KEY-NOT-IN-KEYS` classification must cause `status` to be `"FAIL"` — a key absent from the project trust anchor is never downgraded to a warning. - The response is the `release-verify signatures` JSON; diagnostic fields the tool adds diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json new file mode 100644 index 000000000..584942c02 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/assertions.json @@ -0,0 +1,12 @@ +{ + "has_sha512_check": { + "type": "regex", + "field": "paste_recipe", + "pattern": "sha512sum[^\\n]*\\s(-c\\b|--check\\b)" + }, + "has_sha256_check": { + "type": "regex", + "field": "paste_recipe", + "pattern": "sha256sum[^\\n]*\\s(-c\\b|--check\\b)" + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json index e78c5b43d..d1e4a8cda 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-1-checksums-pass/expected.json @@ -18,5 +18,6 @@ } ], "deprecated_md5_present": false, - "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'" + "has_sha512_check": true, + "has_sha256_check": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json index b9445d91b..ad7b5b103 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/case-2-checksum-mismatch/expected.json @@ -18,5 +18,6 @@ } ], "deprecated_md5_present": false, - "paste_recipe": "sha512sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-source-release.tar.gz.sha256'\nsha512sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha512'\nsha256sum --check 'apache-airflow-2.11.0-bin.tar.gz.sha256'" + "has_sha512_check": true, + "has_sha256_check": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md index 1d350a916..1587ee9c1 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-3-verify-checksums/fixtures/output-spec.md @@ -35,5 +35,7 @@ Grading rules: - `deprecated_md5_present` must be `true` when a `.md5` file appears in the staging directory; `false` otherwise. - `paste_recipe` must be a non-empty string with the sha512sum / sha256sum - verification commands. + verification commands. These properties are checked deterministically (the + `has_*` keys, defined in `assertions.json`) rather than against one + reference recipe; `-c` is accepted for `--check`. - No extra keys are permitted in the response. diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json new file mode 100644 index 000000000..716b01306 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/assertions.json @@ -0,0 +1,21 @@ +{ + "has_find": { + "type": "regex", + "field": "paste_recipe", + "pattern": "(^|[\\s;&|])find\\s" + }, + "has_baseline_globs": { + "type": "contains_all", + "field": "paste_recipe", + "substrings": [ + "*.class", + "*.jar", + "*.so", + "*.dylib", + "*.dll", + "*.exe", + "*.pyc", + "__pycache__" + ] + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json index bc5ad6a7c..900aff357 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-1-no-prohibited-binaries/expected.json @@ -3,5 +3,6 @@ "status": "PASS", "prohibited_found": [], "expected_binaries": [], - "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print" + "has_find": true, + "has_baseline_globs": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json index 4c1e0f0f4..7b8f9b6f2 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/case-2-prohibited-binary-found/expected.json @@ -3,5 +3,6 @@ "status": "FAIL", "prohibited_found": ["airflow/vendor/some-lib/some-lib-1.0.jar"], "expected_binaries": [], - "paste_recipe": "find 'apache-airflow-2.11.0-source-release' \\( -type f \\( -name '*.class' -o -name '*.jar' -o -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.exe' -o -name '*.pyc' \\) -o -type d -name '__pycache__' \\) -print" + "has_find": true, + "has_baseline_globs": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md index feee75e1d..f4955ec25 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-6-binary-exclusion/fixtures/output-spec.md @@ -24,5 +24,8 @@ Grading rules: - `paste_recipe` must be a non-empty string with a `find` that includes the fixed baseline (`.class`, `.jar`, `.so`, `.dylib`, `.dll`, `.exe`, `.pyc`, `__pycache__`) and any extra globs from `release-build.md` beyond that - baseline. + baseline. The baseline is checked deterministically (the `has_*` keys, + defined in `assertions.json`) rather than against one reference recipe; + different quoting, or splitting the scan across several `find` calls, + is fine. - No extra keys are permitted in the response. From 57cd887dc0966c846490a8192d3d20b272a514cb Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 6 Oct 2026 09:54:40 +0200 Subject: [PATCH 2/2] test(release-verify-rc): accept looped and wrapped gpg --verify in step 2 has_sig_verify required `--verify` and `.asc` on one line in that order, so a `for f in *.asc; do gpg --verify "$f"; done` loop or a backslash-continued `gpg --verify` still failed. Check the verify call and the `.asc` signature as two separate assertions. Generated-by: Claude Opus 5 --- .../step-2-verify-signatures/fixtures/assertions.json | 7 ++++++- .../fixtures/case-1-all-sigs-pass/expected.json | 3 ++- .../fixtures/case-2-tampered-sig/expected.json | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json index e473b0f40..351af49bc 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/assertions.json @@ -7,6 +7,11 @@ "has_sig_verify": { "type": "regex", "field": "paste_recipe", - "pattern": "gpg\\d*[^\\n]*--verify\\b[^\\n]*\\.asc" + "pattern": "gpg\\d*[^\\n]*--verify\\b" + }, + "has_asc_signature": { + "type": "regex", + "field": "paste_recipe", + "pattern": "\\.asc\\b" } } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json index 4cfe77aef..ccd219e19 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-1-all-sigs-pass/expected.json @@ -18,5 +18,6 @@ } ], "has_keys_import": true, - "has_sig_verify": true + "has_sig_verify": true, + "has_asc_signature": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json index 3336c984b..c2c69c2f3 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-2-verify-signatures/fixtures/case-2-tampered-sig/expected.json @@ -18,5 +18,6 @@ } ], "has_keys_import": true, - "has_sig_verify": true + "has_sig_verify": true, + "has_asc_signature": true }