From 59b68c221d8e7bb4fd0c33ae56789df4db044bbe Mon Sep 17 00:00:00 2001 From: Danny van Dyk Date: Thu, 17 Sep 2026 14:26:26 +0100 Subject: [PATCH 1/3] Add missing eos_version to 2025-05 and 2025-06 Both entries lacked the field, which eos.DataSetDescription requires, so eos.Datasets could not deserialise the registry at all. The version is the one stated in each data set's README. --- datasets.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasets.yaml b/datasets.yaml index bf774ab..0df64c3 100644 --- a/datasets.yaml +++ b/datasets.yaml @@ -8,6 +8,7 @@ keywords: - Semileptonic decays - Form factors + eos_version: 1.0.19 2025-05: authors: @@ -20,6 +21,7 @@ - Kaon decays - Semileptonic decays - Form factors + eos_version: 1.0.19 2025-04: authors: From a163b245671f81409a7f2e8aa22fd409813e6082 Mon Sep 17 00:00:00 2001 From: Danny van Dyk Date: Thu, 17 Sep 2026 14:39:17 +0100 Subject: [PATCH 2/3] Validate analysis files of all released data sets Replace the hand-maintained matrix in the validation workflow with one built from a new record, .github/analyses.yaml, which names the analysis files of each data set at its highest released revision. A first job turns that record into a job matrix; a second checks out each data set's branch and validates one analysis file per job. Validation now runs in docker.io/eoshep/release:latest rather than against a pinned prerelease of eoshep installed with pip. eos-analysis validate exits 0 whether or not it finds anything, so the check runs through a driver that reports the errors found as JSON. Releases up to 1.0.21 have no notion of severity and print one line per error; later EOS returns diagnostics carrying a severity, of which only the errors are reported. Warnings are expected for released data sets and must not raise an alarm. Errors open an issue per data set and analysis file, or update the body of the issue already open for it. Issues are never closed automatically. The composite action is dropped, having become unused. --- .github/actions/validate-analysis/action.yml | 28 ----- .github/analyses.yaml | 83 +++++++++++++++ .github/scripts/build_matrix.py | 105 +++++++++++++++++++ .github/scripts/report_issue.sh | 74 +++++++++++++ .github/scripts/validate_analysis.py | 63 +++++++++++ .github/workflows/validate.yaml | 103 ++++++++++++++---- 6 files changed, 408 insertions(+), 48 deletions(-) delete mode 100644 .github/actions/validate-analysis/action.yml create mode 100644 .github/analyses.yaml create mode 100755 .github/scripts/build_matrix.py create mode 100755 .github/scripts/report_issue.sh create mode 100755 .github/scripts/validate_analysis.py diff --git a/.github/actions/validate-analysis/action.yml b/.github/actions/validate-analysis/action.yml deleted file mode 100644 index e49e8ec..0000000 --- a/.github/actions/validate-analysis/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Validate an EOS analysis' -description: 'Checks out an EOS tag or branch and runs ``eos-analysis validate``' - -inputs: - ref: - description: The git reference from which to check out the EOS analysis, either a branch or a tag name. - required: true - analysis_file: - description: The name of the main analysis file contained in the branch or tag - default: analysis.yaml - -runs: - using: composite - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - path: analysis/ - clean: false - set-safe-directory: false - - - name: Update EOS - shell: bash - run: pip3 install --upgrade --pre eoshep>=1.0.13.dev482 - - - name: Validate the analysis file - shell: bash - run: eos-analysis validate --analysis-file analysis/${{ inputs.analysis_file }} diff --git a/.github/analyses.yaml b/.github/analyses.yaml new file mode 100644 index 0000000..a56cbfc --- /dev/null +++ b/.github/analyses.yaml @@ -0,0 +1,83 @@ +# Analysis files to be validated by .github/workflows/validate.yaml. +# +# One entry per data set, keyed by the data set id *without* its revision suffix. That key doubles +# as the name of the branch carrying the data set: the workflow validates the tip of that branch, +# which is expected to correspond to the tag of the most recent revision. The files listed for a +# data set are therefore those of its highest revision presently released. +# +# An entry takes two optional keys: +# +# analysis_files The analysis files to validate. An entry without them -- an empty entry -- +# describes a data set that carries no analysis file, and nothing is validated +# for it. +# skip When true, the data set is recorded but not validated. +# +# datasets.yaml is deliberately not used for this record. It is the public registry, deserialised +# by eos.Datasets into eos.DataSetDescription -- a dataclass that rejects unknown keys -- so adding +# an 'analysis_files' key there would break every already-released EOS that refreshes the registry. + +2023-01: + # highest revision: 2023-01v2 + skip: True + analysis_files: + - b-to-u-l-nu.yaml + +2023-02: + # highest revision: 2023-02v2 + skip: True + analysis_files: + - BToK_FF.yaml + - BToKstar_FF.yaml + - BsToPhi_FF.yaml + +2023-03: + skip: True + analysis_files: + - VubVcb-bs-to-k-ds-l-nu.yaml + - ff-bs-to-k.yaml + - lcsr-bs-to-k.yaml + +2024-01: + # highest revision: 2024-01v2 + skip: True + analysis_files: + - analysis.yaml + +2024-02: + skip: True + analysis_files: + - analysis.yaml + +2025-01: + skip: True + analysis_files: + - analysis.yaml + - form-factor-analysis.yaml + +2025-02: + skip: True + analysis_files: + - analysis.yaml + +2025-03: + skip: True + analysis_files: + - analysis_200.yaml + - analysis_210.yaml + - analysis_321.yaml + +2025-05: + analysis_files: + - analysis_BSM.yaml + - analysis_CKM.yaml + - analysis_FF.yaml + +2025-06: + analysis_files: + - analysis.yaml + +2026-01: + +2026-03: + analysis_files: + - analysis.yaml diff --git a/.github/scripts/build_matrix.py b/.github/scripts/build_matrix.py new file mode 100755 index 0000000..1e54112 --- /dev/null +++ b/.github/scripts/build_matrix.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Turn the record in .github/analyses.yaml into a job matrix, one row per analysis file. + +A data set's key is also the name of the branch carrying it. Branches that do not exist are +reported as workflow warnings and dropped, so that a data set awaiting its branch does not block +validation of every other data set. + +A data set contributes no row if it is marked 'skip', or if it records no analysis file at all -- +an empty entry, describing a data set that carries none. +""" + +import argparse +import json +import os +import subprocess +import sys + +import yaml + +KNOWN_KEYS = {'analysis_files', 'skip'} + + +def branch_exists(remote: str, branch: str) -> bool: + result = subprocess.run( + ['git', 'ls-remote', '--exit-code', '--heads', remote, f'refs/heads/{branch}'], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True + ) + if result.returncode not in (0, 2): + raise RuntimeError(f'git ls-remote failed for branch \'{branch}\': {result.stderr.strip()}') + + return result.returncode == 0 + + +def emit(name: str, value: str) -> None: + github_output = os.environ.get('GITHUB_OUTPUT') + if github_output is None: + print(f'{name}={value}') + return + + with open(github_output, 'a', encoding='utf-8') as f: + f.write(f'{name}={value}\n') + + +def summarize(lines: list[str]) -> None: + github_step_summary = os.environ.get('GITHUB_STEP_SUMMARY') + if github_step_summary is None: + print('\n'.join(lines), file=sys.stderr) + return + + with open(github_step_summary, 'a', encoding='utf-8') as f: + f.write('\n'.join(lines) + '\n') + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('record', help='the YAML file recording the analysis files of each data set') + parser.add_argument('--remote', default='origin', help='the remote whose branches are checked out') + args = parser.parse_args() + + with open(args.record, encoding='utf-8') as f: + record = yaml.safe_load(f) or {} + + include = [] + excluded = {'skipped': [], 'no analysis file': [], 'no branch': []} + for data_set, description in sorted(record.items()): + description = description or {} + if (unknown := description.keys() - KNOWN_KEYS): + raise ValueError(f'data set \'{data_set}\' records unknown key(s): {", ".join(sorted(unknown))}') + + if description.get('skip'): + excluded['skipped'].append(data_set) + continue + + analysis_files = description.get('analysis_files') or [] + if not analysis_files: + excluded['no analysis file'].append(data_set) + continue + + if not branch_exists(args.remote, data_set): + excluded['no branch'].append(data_set) + print(f'::warning title=Missing branch::Data set {data_set} is recorded in ' + f'{args.record} but has no branch \'{data_set}\'; it will not be validated.') + continue + + include.extend( + {'data_set': data_set, 'branch': data_set, 'analysis_file': analysis_file} + for analysis_file in analysis_files + ) + + emit('matrix', json.dumps({'include': include})) + emit('empty', 'true' if not include else 'false') + + summary = ['## Analysis files to validate', '', '| Data set | Analysis file |', '| --- | --- |'] + summary += [f'| `{row["data_set"]}` | `{row["analysis_file"]}` |' for row in include] or ['| _none_ | |'] + summary += [ + f'\nNot validated ({reason}): {", ".join(f"`{d}`" for d in data_sets)}.' + for reason, data_sets in excluded.items() if data_sets + ] + summarize(summary) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/report_issue.sh b/.github/scripts/report_issue.sh new file mode 100755 index 0000000..be1474b --- /dev/null +++ b/.github/scripts/report_issue.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Report the outcome recorded by validate_analysis.py: open an issue for the errors found, or +# update the body of the issue already open for this data set and analysis file. Issues are never +# closed automatically; a human decides when a report has been dealt with. +# +# Where EOS reports the line an error refers to, the error is also emitted as a workflow +# annotation, and linked from the issue body. The annotation names a file that does not exist on +# the branch this workflow runs from, so it is listed in the run rather than bound to the source; +# the links in the issue body are pinned to the data set's own commit and do resolve. + +set -euo pipefail + +: "${DATA_SET:?}" "${ANALYSIS_FILE:?}" "${BRANCH:?}" "${RESULT_FILE:?}" "${GH_REPO:?}" "${GH_TOKEN:?}" +: "${LABEL:=validation}" +: "${SERVER_URL:=https://github.com}" +: "${MAX_LINKS:=20}" + +count=$(jq '.errors | length' "${RESULT_FILE}") +version=$(jq -r '.version' "${RESULT_FILE}") + +if [[ "${count}" -eq 0 ]]; then + echo "::notice title=Validated::${DATA_SET}: ${ANALYSIS_FILE} validates cleanly with EOS v${version}." + exit 0 +fi + +title="EOS-DATA-${DATA_SET}: Validation failed for analysis file \`${ANALYSIS_FILE}\`" +blob="${SERVER_URL}/${GH_REPO}/blob/${COMMIT:-${BRANCH}}/${ANALYSIS_FILE}" + +# One annotation per error, so the Actions UI lists them with their position. +jq -r --arg file "${ANALYSIS_FILE}" --arg set "${DATA_SET}" ' + .errors[] + | "::error file=\($file)\(if .line then ",line=\(.line)" else "" end),title=EOS-DATA-\($set)::" + + (.message | gsub("%";"%25") | gsub("\n";"%0A") | gsub("\r";"%0D")) +' "${RESULT_FILE}" + +body=$(mktemp) +{ + printf 'One or more issues were found when validating `%s` with EOS v%s.\n\n' "${ANALYSIS_FILE}" "${version}" + printf '```\n' + jq -r '.errors[] | (if .line then "line \(.line): " else "" end) + .message' "${RESULT_FILE}" + printf '```\n\n' + + located=$(jq '[.errors[] | select(.line)] | length' "${RESULT_FILE}") + if [[ "${located}" -gt 0 ]]; then + printf '### Locations\n\n' + # Consecutive lines are collapsed into ranges, since one fault usually spans a whole block. + jq -r --arg blob "${blob}" --arg file "${ANALYSIS_FILE}" --argjson max "${MAX_LINKS}" ' + [.errors[] | select(.line) | .line] | unique + | reduce .[] as $l ([]; + if (length > 0 and .[-1][1] == $l - 1) then .[:-1] + [[.[-1][0], $l]] else . + [[$l, $l]] end) + | (.[:$max] | .[] | if .[0] == .[1] + then "- [`\($file):\(.[0])`](\($blob)#L\(.[0]))" + else "- [`\($file):\(.[0])-\(.[1])`](\($blob)#L\(.[0])-L\(.[1]))" end), + (if length > $max then "- … and \(length - $max) further ranges" else empty end) + ' "${RESULT_FILE}" + printf '\n' + fi + + printf -- '---\nBranch `%s` at %s, checked by [this workflow run](%s).\n' \ + "${BRANCH}" "${COMMIT:-unknown commit}" "${RUN_URL:-}" +} > "${body}" + +# Match on the exact title: one issue per data set and analysis file. +number=$(gh issue list --repo "${GH_REPO}" --state open --label "${LABEL}" --limit 200 --json number,title \ + | jq -r --arg title "${title}" 'map(select(.title == $title)) | .[0].number // empty') + +if [[ -n "${number}" ]]; then + gh issue edit "${number}" --repo "${GH_REPO}" --body-file "${body}" + echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); updated issue #${number}." +else + gh issue create --repo "${GH_REPO}" --title "${title}" --label "${LABEL}" --body-file "${body}" + echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); opened a new issue." +fi diff --git a/.github/scripts/validate_analysis.py b/.github/scripts/validate_analysis.py new file mode 100755 index 0000000..a8ed50d --- /dev/null +++ b/.github/scripts/validate_analysis.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Validate a single EOS analysis file and record the errors found. + +Runs inside the EOS container, with the working directory set to the checked-out data set so that +the deep validation phase resolves relative data paths as it would for a user. The result is +written as JSON, since ``eos-analysis validate`` exits 0 whether or not it finds anything. + +``eos.validate`` returns one ``eos.Diagnostic`` per finding, carrying a severity and, where EOS can +resolve one, the line of the analysis file it refers to. Only the errors are reported: warnings +(an unused prior, say, or a constraint file that has not been produced yet) are expected for +released data sets and must not raise an alarm. This requires EOS 1.1 or later; up to 1.0.21 +``eos.validate`` returned nothing and knew no severities. +""" + +import argparse +import contextlib +import io +import json +import sys +import traceback + + +def errors_from(diagnostics) -> list[dict]: + """Return one {'line', 'message'} per error. 'line' is None where EOS resolved no position.""" + from eos.diagnostic import Severity + + return [ + {'line': d.line, 'message': f'{d.location()}: {d.message}'} + for d in diagnostics if d.severity is Severity.ERROR + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('analysis_file', help='the analysis file to validate, relative to the working directory') + parser.add_argument('--output', required=True, help='the JSON file to write the result to') + args = parser.parse_args() + + import eos + + result = {'analysis_file': args.analysis_file, 'version': eos.__version__, 'errors': [], 'log': ''} + + # eos.validate() prints as it goes; capture that so only the filtered errors are reported. + printed = io.StringIO() + try: + with contextlib.redirect_stdout(printed): + diagnostics = eos.validate(analysis_file=args.analysis_file) + result['errors'] = errors_from(diagnostics) + except Exception: + result['errors'] = [{'line': None, + 'message': f'Validation aborted with an unhandled exception:\n{traceback.format_exc()}'}] + result['log'] = printed.getvalue() + + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) + + print(f'{args.analysis_file}: EOS v{result["version"]}, {len(result["errors"])} error(s)', file=sys.stderr) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 3e37146..68ec4a8 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -1,35 +1,98 @@ -on: - pull_request: - branches: - - main - paths: - - '.github/workflows/validate.yaml' - - '.github/actions/validate-analysis/**' +name: Validate analysis files +on: workflow_dispatch: schedule: - cron: '0 8 * * WED' +# Runs are serialised: reporting looks for an open issue and then creates one if it finds none, +# which two overlapping runs would both do for the same finding. Queue rather than cancel, so that +# a scheduled run is not dropped because someone started one by hand. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + issues: write + +env: + EOS_IMAGE: docker.io/eoshep/eos:latest + LABEL: validation + jobs: + matrix: + name: Determine the analysis files to validate + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + empty: ${{ steps.build.outputs.empty }} + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: pip3 install --quiet pyyaml + + - name: Ensure the issue label exists + env: + GH_TOKEN: ${{ github.token }} + run: >- + gh label create "${LABEL}" --repo "${{ github.repository }}" --color BFD4F2 + --description 'Reported by the analysis-file validation workflow' --force + + - name: Build the job matrix + id: build + run: python3 .github/scripts/build_matrix.py .github/analyses.yaml + validate: + name: ${{ matrix.data_set }} / ${{ matrix.analysis_file }} + needs: matrix + if: needs.matrix.outputs.empty != 'true' runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - { ref: '2023-01', analysis_file: 'b-to-u-l-nu.yaml' } - - { ref: '2023-03', analysis_file: 'VubVcb-bs-to-k-ds-l-nu.yaml' } - - { ref: '2023-03', analysis_file: 'lcsr-bs-to-k.yaml' } - - { ref: '2023-03', analysis_file: 'ff-bs-to-k.yaml' } - - { ref: '2024-01', analysis_file: 'analysis.yaml' } + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} steps: - - uses: actions/checkout@v4 + - name: Check out the workflow scripts + uses: actions/checkout@v4 with: - clean: false - set-safe-directory: false + path: workflow - - uses: ./.github/actions/validate-analysis + - name: Check out the data set + uses: actions/checkout@v4 with: - ref: ${{ matrix.ref }} - analysis_file: ${{ matrix.analysis_file }} + # Fully qualified: a data set's name is both a tag and a branch, and it is the tip of + # the branch that is to be validated. + ref: refs/heads/${{ matrix.branch }} + path: analysis + + - name: Validate the analysis file + env: + ANALYSIS_FILE: ${{ matrix.analysis_file }} + run: | + # The image runs as its own user, which does not own the checkout, so the result is + # written to a directory of its own that any user may write to. + mkdir -p "${GITHUB_WORKSPACE}/result" + chmod 777 "${GITHUB_WORKSPACE}/result" + docker run --rm \ + --volume "${GITHUB_WORKSPACE}/analysis:/analysis:ro" \ + --volume "${GITHUB_WORKSPACE}/workflow/.github/scripts:/scripts:ro" \ + --volume "${GITHUB_WORKSPACE}/result:/result" \ + --workdir /analysis \ + "${EOS_IMAGE}" \ + python3 /scripts/validate_analysis.py "${ANALYSIS_FILE}" --output /result/validation.json + + - name: Report the outcome + env: + DATA_SET: ${{ matrix.data_set }} + ANALYSIS_FILE: ${{ matrix.analysis_file }} + BRANCH: ${{ matrix.branch }} + RESULT_FILE: ${{ github.workspace }}/result/validation.json + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + SERVER_URL: ${{ github.server_url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + COMMIT=$(git -C "${GITHUB_WORKSPACE}/analysis" rev-parse HEAD) \ + workflow/.github/scripts/report_issue.sh From 4762c2acff7b958ed48eafb05ee1121aedcc91c8 Mon Sep 17 00:00:00 2001 From: Danny van Dyk Date: Thu, 17 Sep 2026 14:53:02 +0100 Subject: [PATCH 3/3] Update pre-commit hooks to v6.0.0 v4.4.0 declared the deprecated stage names 'commit' and 'push', which a future pre-commit will reject. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9407937..8a32284 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace