diff --git a/.github/actions/validate-analysis/action.yml b/.github/actions/validate-analysis/action.yml deleted file mode 100644 index e49e8ec..0000000 --- a/.github/actions/validate-analysis/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Validate an EOS analysis' -description: 'Checks out an EOS tag or branch and runs ``eos-analysis validate``' - -inputs: - ref: - description: The git reference from which to check out the EOS analysis, either a branch or a tag name. - required: true - analysis_file: - description: The name of the main analysis file contained in the branch or tag - default: analysis.yaml - -runs: - using: composite - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - path: analysis/ - clean: false - set-safe-directory: false - - - name: Update EOS - shell: bash - run: pip3 install --upgrade --pre eoshep>=1.0.13.dev482 - - - name: Validate the analysis file - shell: bash - run: eos-analysis validate --analysis-file analysis/${{ inputs.analysis_file }} diff --git a/.github/analyses.yaml b/.github/analyses.yaml new file mode 100644 index 0000000..a56cbfc --- /dev/null +++ b/.github/analyses.yaml @@ -0,0 +1,83 @@ +# Analysis files to be validated by .github/workflows/validate.yaml. +# +# One entry per data set, keyed by the data set id *without* its revision suffix. That key doubles +# as the name of the branch carrying the data set: the workflow validates the tip of that branch, +# which is expected to correspond to the tag of the most recent revision. The files listed for a +# data set are therefore those of its highest revision presently released. +# +# An entry takes two optional keys: +# +# analysis_files The analysis files to validate. An entry without them -- an empty entry -- +# describes a data set that carries no analysis file, and nothing is validated +# for it. +# skip When true, the data set is recorded but not validated. +# +# datasets.yaml is deliberately not used for this record. It is the public registry, deserialised +# by eos.Datasets into eos.DataSetDescription -- a dataclass that rejects unknown keys -- so adding +# an 'analysis_files' key there would break every already-released EOS that refreshes the registry. + +2023-01: + # highest revision: 2023-01v2 + skip: True + analysis_files: + - b-to-u-l-nu.yaml + +2023-02: + # highest revision: 2023-02v2 + skip: True + analysis_files: + - BToK_FF.yaml + - BToKstar_FF.yaml + - BsToPhi_FF.yaml + +2023-03: + skip: True + analysis_files: + - VubVcb-bs-to-k-ds-l-nu.yaml + - ff-bs-to-k.yaml + - lcsr-bs-to-k.yaml + +2024-01: + # highest revision: 2024-01v2 + skip: True + analysis_files: + - analysis.yaml + +2024-02: + skip: True + analysis_files: + - analysis.yaml + +2025-01: + skip: True + analysis_files: + - analysis.yaml + - form-factor-analysis.yaml + +2025-02: + skip: True + analysis_files: + - analysis.yaml + +2025-03: + skip: True + analysis_files: + - analysis_200.yaml + - analysis_210.yaml + - analysis_321.yaml + +2025-05: + analysis_files: + - analysis_BSM.yaml + - analysis_CKM.yaml + - analysis_FF.yaml + +2025-06: + analysis_files: + - analysis.yaml + +2026-01: + +2026-03: + analysis_files: + - analysis.yaml diff --git a/.github/scripts/build_matrix.py b/.github/scripts/build_matrix.py new file mode 100755 index 0000000..1e54112 --- /dev/null +++ b/.github/scripts/build_matrix.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Turn the record in .github/analyses.yaml into a job matrix, one row per analysis file. + +A data set's key is also the name of the branch carrying it. Branches that do not exist are +reported as workflow warnings and dropped, so that a data set awaiting its branch does not block +validation of every other data set. + +A data set contributes no row if it is marked 'skip', or if it records no analysis file at all -- +an empty entry, describing a data set that carries none. +""" + +import argparse +import json +import os +import subprocess +import sys + +import yaml + +KNOWN_KEYS = {'analysis_files', 'skip'} + + +def branch_exists(remote: str, branch: str) -> bool: + result = subprocess.run( + ['git', 'ls-remote', '--exit-code', '--heads', remote, f'refs/heads/{branch}'], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True + ) + if result.returncode not in (0, 2): + raise RuntimeError(f'git ls-remote failed for branch \'{branch}\': {result.stderr.strip()}') + + return result.returncode == 0 + + +def emit(name: str, value: str) -> None: + github_output = os.environ.get('GITHUB_OUTPUT') + if github_output is None: + print(f'{name}={value}') + return + + with open(github_output, 'a', encoding='utf-8') as f: + f.write(f'{name}={value}\n') + + +def summarize(lines: list[str]) -> None: + github_step_summary = os.environ.get('GITHUB_STEP_SUMMARY') + if github_step_summary is None: + print('\n'.join(lines), file=sys.stderr) + return + + with open(github_step_summary, 'a', encoding='utf-8') as f: + f.write('\n'.join(lines) + '\n') + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('record', help='the YAML file recording the analysis files of each data set') + parser.add_argument('--remote', default='origin', help='the remote whose branches are checked out') + args = parser.parse_args() + + with open(args.record, encoding='utf-8') as f: + record = yaml.safe_load(f) or {} + + include = [] + excluded = {'skipped': [], 'no analysis file': [], 'no branch': []} + for data_set, description in sorted(record.items()): + description = description or {} + if (unknown := description.keys() - KNOWN_KEYS): + raise ValueError(f'data set \'{data_set}\' records unknown key(s): {", ".join(sorted(unknown))}') + + if description.get('skip'): + excluded['skipped'].append(data_set) + continue + + analysis_files = description.get('analysis_files') or [] + if not analysis_files: + excluded['no analysis file'].append(data_set) + continue + + if not branch_exists(args.remote, data_set): + excluded['no branch'].append(data_set) + print(f'::warning title=Missing branch::Data set {data_set} is recorded in ' + f'{args.record} but has no branch \'{data_set}\'; it will not be validated.') + continue + + include.extend( + {'data_set': data_set, 'branch': data_set, 'analysis_file': analysis_file} + for analysis_file in analysis_files + ) + + emit('matrix', json.dumps({'include': include})) + emit('empty', 'true' if not include else 'false') + + summary = ['## Analysis files to validate', '', '| Data set | Analysis file |', '| --- | --- |'] + summary += [f'| `{row["data_set"]}` | `{row["analysis_file"]}` |' for row in include] or ['| _none_ | |'] + summary += [ + f'\nNot validated ({reason}): {", ".join(f"`{d}`" for d in data_sets)}.' + for reason, data_sets in excluded.items() if data_sets + ] + summarize(summary) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/report_issue.sh b/.github/scripts/report_issue.sh new file mode 100755 index 0000000..be1474b --- /dev/null +++ b/.github/scripts/report_issue.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Report the outcome recorded by validate_analysis.py: open an issue for the errors found, or +# update the body of the issue already open for this data set and analysis file. Issues are never +# closed automatically; a human decides when a report has been dealt with. +# +# Where EOS reports the line an error refers to, the error is also emitted as a workflow +# annotation, and linked from the issue body. The annotation names a file that does not exist on +# the branch this workflow runs from, so it is listed in the run rather than bound to the source; +# the links in the issue body are pinned to the data set's own commit and do resolve. + +set -euo pipefail + +: "${DATA_SET:?}" "${ANALYSIS_FILE:?}" "${BRANCH:?}" "${RESULT_FILE:?}" "${GH_REPO:?}" "${GH_TOKEN:?}" +: "${LABEL:=validation}" +: "${SERVER_URL:=https://github.com}" +: "${MAX_LINKS:=20}" + +count=$(jq '.errors | length' "${RESULT_FILE}") +version=$(jq -r '.version' "${RESULT_FILE}") + +if [[ "${count}" -eq 0 ]]; then + echo "::notice title=Validated::${DATA_SET}: ${ANALYSIS_FILE} validates cleanly with EOS v${version}." + exit 0 +fi + +title="EOS-DATA-${DATA_SET}: Validation failed for analysis file \`${ANALYSIS_FILE}\`" +blob="${SERVER_URL}/${GH_REPO}/blob/${COMMIT:-${BRANCH}}/${ANALYSIS_FILE}" + +# One annotation per error, so the Actions UI lists them with their position. +jq -r --arg file "${ANALYSIS_FILE}" --arg set "${DATA_SET}" ' + .errors[] + | "::error file=\($file)\(if .line then ",line=\(.line)" else "" end),title=EOS-DATA-\($set)::" + + (.message | gsub("%";"%25") | gsub("\n";"%0A") | gsub("\r";"%0D")) +' "${RESULT_FILE}" + +body=$(mktemp) +{ + printf 'One or more issues were found when validating `%s` with EOS v%s.\n\n' "${ANALYSIS_FILE}" "${version}" + printf '```\n' + jq -r '.errors[] | (if .line then "line \(.line): " else "" end) + .message' "${RESULT_FILE}" + printf '```\n\n' + + located=$(jq '[.errors[] | select(.line)] | length' "${RESULT_FILE}") + if [[ "${located}" -gt 0 ]]; then + printf '### Locations\n\n' + # Consecutive lines are collapsed into ranges, since one fault usually spans a whole block. + jq -r --arg blob "${blob}" --arg file "${ANALYSIS_FILE}" --argjson max "${MAX_LINKS}" ' + [.errors[] | select(.line) | .line] | unique + | reduce .[] as $l ([]; + if (length > 0 and .[-1][1] == $l - 1) then .[:-1] + [[.[-1][0], $l]] else . + [[$l, $l]] end) + | (.[:$max] | .[] | if .[0] == .[1] + then "- [`\($file):\(.[0])`](\($blob)#L\(.[0]))" + else "- [`\($file):\(.[0])-\(.[1])`](\($blob)#L\(.[0])-L\(.[1]))" end), + (if length > $max then "- … and \(length - $max) further ranges" else empty end) + ' "${RESULT_FILE}" + printf '\n' + fi + + printf -- '---\nBranch `%s` at %s, checked by [this workflow run](%s).\n' \ + "${BRANCH}" "${COMMIT:-unknown commit}" "${RUN_URL:-}" +} > "${body}" + +# Match on the exact title: one issue per data set and analysis file. +number=$(gh issue list --repo "${GH_REPO}" --state open --label "${LABEL}" --limit 200 --json number,title \ + | jq -r --arg title "${title}" 'map(select(.title == $title)) | .[0].number // empty') + +if [[ -n "${number}" ]]; then + gh issue edit "${number}" --repo "${GH_REPO}" --body-file "${body}" + echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); updated issue #${number}." +else + gh issue create --repo "${GH_REPO}" --title "${title}" --label "${LABEL}" --body-file "${body}" + echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); opened a new issue." +fi diff --git a/.github/scripts/validate_analysis.py b/.github/scripts/validate_analysis.py new file mode 100755 index 0000000..a8ed50d --- /dev/null +++ b/.github/scripts/validate_analysis.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Validate a single EOS analysis file and record the errors found. + +Runs inside the EOS container, with the working directory set to the checked-out data set so that +the deep validation phase resolves relative data paths as it would for a user. The result is +written as JSON, since ``eos-analysis validate`` exits 0 whether or not it finds anything. + +``eos.validate`` returns one ``eos.Diagnostic`` per finding, carrying a severity and, where EOS can +resolve one, the line of the analysis file it refers to. Only the errors are reported: warnings +(an unused prior, say, or a constraint file that has not been produced yet) are expected for +released data sets and must not raise an alarm. This requires EOS 1.1 or later; up to 1.0.21 +``eos.validate`` returned nothing and knew no severities. +""" + +import argparse +import contextlib +import io +import json +import sys +import traceback + + +def errors_from(diagnostics) -> list[dict]: + """Return one {'line', 'message'} per error. 'line' is None where EOS resolved no position.""" + from eos.diagnostic import Severity + + return [ + {'line': d.line, 'message': f'{d.location()}: {d.message}'} + for d in diagnostics if d.severity is Severity.ERROR + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('analysis_file', help='the analysis file to validate, relative to the working directory') + parser.add_argument('--output', required=True, help='the JSON file to write the result to') + args = parser.parse_args() + + import eos + + result = {'analysis_file': args.analysis_file, 'version': eos.__version__, 'errors': [], 'log': ''} + + # eos.validate() prints as it goes; capture that so only the filtered errors are reported. + printed = io.StringIO() + try: + with contextlib.redirect_stdout(printed): + diagnostics = eos.validate(analysis_file=args.analysis_file) + result['errors'] = errors_from(diagnostics) + except Exception: + result['errors'] = [{'line': None, + 'message': f'Validation aborted with an unhandled exception:\n{traceback.format_exc()}'}] + result['log'] = printed.getvalue() + + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) + + print(f'{args.analysis_file}: EOS v{result["version"]}, {len(result["errors"])} error(s)', file=sys.stderr) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 3e37146..68ec4a8 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -1,35 +1,98 @@ -on: - pull_request: - branches: - - main - paths: - - '.github/workflows/validate.yaml' - - '.github/actions/validate-analysis/**' +name: Validate analysis files +on: workflow_dispatch: schedule: - cron: '0 8 * * WED' +# Runs are serialised: reporting looks for an open issue and then creates one if it finds none, +# which two overlapping runs would both do for the same finding. Queue rather than cancel, so that +# a scheduled run is not dropped because someone started one by hand. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + issues: write + +env: + EOS_IMAGE: docker.io/eoshep/eos:latest + LABEL: validation + jobs: + matrix: + name: Determine the analysis files to validate + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + empty: ${{ steps.build.outputs.empty }} + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: pip3 install --quiet pyyaml + + - name: Ensure the issue label exists + env: + GH_TOKEN: ${{ github.token }} + run: >- + gh label create "${LABEL}" --repo "${{ github.repository }}" --color BFD4F2 + --description 'Reported by the analysis-file validation workflow' --force + + - name: Build the job matrix + id: build + run: python3 .github/scripts/build_matrix.py .github/analyses.yaml + validate: + name: ${{ matrix.data_set }} / ${{ matrix.analysis_file }} + needs: matrix + if: needs.matrix.outputs.empty != 'true' runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - { ref: '2023-01', analysis_file: 'b-to-u-l-nu.yaml' } - - { ref: '2023-03', analysis_file: 'VubVcb-bs-to-k-ds-l-nu.yaml' } - - { ref: '2023-03', analysis_file: 'lcsr-bs-to-k.yaml' } - - { ref: '2023-03', analysis_file: 'ff-bs-to-k.yaml' } - - { ref: '2024-01', analysis_file: 'analysis.yaml' } + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} steps: - - uses: actions/checkout@v4 + - name: Check out the workflow scripts + uses: actions/checkout@v4 with: - clean: false - set-safe-directory: false + path: workflow - - uses: ./.github/actions/validate-analysis + - name: Check out the data set + uses: actions/checkout@v4 with: - ref: ${{ matrix.ref }} - analysis_file: ${{ matrix.analysis_file }} + # Fully qualified: a data set's name is both a tag and a branch, and it is the tip of + # the branch that is to be validated. + ref: refs/heads/${{ matrix.branch }} + path: analysis + + - name: Validate the analysis file + env: + ANALYSIS_FILE: ${{ matrix.analysis_file }} + run: | + # The image runs as its own user, which does not own the checkout, so the result is + # written to a directory of its own that any user may write to. + mkdir -p "${GITHUB_WORKSPACE}/result" + chmod 777 "${GITHUB_WORKSPACE}/result" + docker run --rm \ + --volume "${GITHUB_WORKSPACE}/analysis:/analysis:ro" \ + --volume "${GITHUB_WORKSPACE}/workflow/.github/scripts:/scripts:ro" \ + --volume "${GITHUB_WORKSPACE}/result:/result" \ + --workdir /analysis \ + "${EOS_IMAGE}" \ + python3 /scripts/validate_analysis.py "${ANALYSIS_FILE}" --output /result/validation.json + + - name: Report the outcome + env: + DATA_SET: ${{ matrix.data_set }} + ANALYSIS_FILE: ${{ matrix.analysis_file }} + BRANCH: ${{ matrix.branch }} + RESULT_FILE: ${{ github.workspace }}/result/validation.json + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + SERVER_URL: ${{ github.server_url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + COMMIT=$(git -C "${GITHUB_WORKSPACE}/analysis" rev-parse HEAD) \ + workflow/.github/scripts/report_issue.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9407937..8a32284 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace diff --git a/datasets.yaml b/datasets.yaml index bf774ab..0df64c3 100644 --- a/datasets.yaml +++ b/datasets.yaml @@ -8,6 +8,7 @@ keywords: - Semileptonic decays - Form factors + eos_version: 1.0.19 2025-05: authors: @@ -20,6 +21,7 @@ - Kaon decays - Semileptonic decays - Form factors + eos_version: 1.0.19 2025-04: authors: