Validate released analysis files against the current EOS release

.github/analyses.yaml

@@ -0,0 +1,83 @@
+# Analysis files to be validated by .github/workflows/validate.yaml.
+#
+# One entry per data set, keyed by the data set id *without* its revision suffix. That key doubles
+# as the name of the branch carrying the data set: the workflow validates the tip of that branch,
+# which is expected to correspond to the tag of the most recent revision. The files listed for a
+# data set are therefore those of its highest revision presently released.
+#
+# An entry takes two optional keys:
+#
+#   analysis_files  The analysis files to validate. An entry without them -- an empty entry --
+#                   describes a data set that carries no analysis file, and nothing is validated
+#                   for it.
+#   skip            When true, the data set is recorded but not validated.
+#
+# datasets.yaml is deliberately not used for this record. It is the public registry, deserialised
+# by eos.Datasets into eos.DataSetDescription -- a dataclass that rejects unknown keys -- so adding
+# an 'analysis_files' key there would break every already-released EOS that refreshes the registry.
+
+2023-01:
+  # highest revision: 2023-01v2
+  skip: True
+  analysis_files:
+    - b-to-u-l-nu.yaml
+
+2023-02:
+  # highest revision: 2023-02v2
+  skip: True
+  analysis_files:
+    - BToK_FF.yaml
+    - BToKstar_FF.yaml
+    - BsToPhi_FF.yaml
+
+2023-03:
+  skip: True
+  analysis_files:
+    - VubVcb-bs-to-k-ds-l-nu.yaml
+    - ff-bs-to-k.yaml
+    - lcsr-bs-to-k.yaml
+
+2024-01:
+  # highest revision: 2024-01v2
+  skip: True
+  analysis_files:
+    - analysis.yaml
+
+2024-02:
+  skip: True
+  analysis_files:
+    - analysis.yaml
+
+2025-01:
+  skip: True
+  analysis_files:
+    - analysis.yaml
+    - form-factor-analysis.yaml
+
+2025-02:
+  skip: True
+  analysis_files:
+    - analysis.yaml
+
+2025-03:
+  skip: True
+  analysis_files:
+    - analysis_200.yaml
+    - analysis_210.yaml
+    - analysis_321.yaml
+
+2025-05:
+  analysis_files:
+    - analysis_BSM.yaml
+    - analysis_CKM.yaml
+    - analysis_FF.yaml
+
+2025-06:
+  analysis_files:
+    - analysis.yaml
+
+2026-01:
+
+2026-03:
+  analysis_files:
+    - analysis.yaml

.github/scripts/build_matrix.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Turn the record in .github/analyses.yaml into a job matrix, one row per analysis file.
+
+A data set's key is also the name of the branch carrying it. Branches that do not exist are
+reported as workflow warnings and dropped, so that a data set awaiting its branch does not block
+validation of every other data set.
+
+A data set contributes no row if it is marked 'skip', or if it records no analysis file at all --
+an empty entry, describing a data set that carries none.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+import yaml
+
+KNOWN_KEYS = {'analysis_files', 'skip'}
+
+
+def branch_exists(remote: str, branch: str) -> bool:
+    result = subprocess.run(
+        ['git', 'ls-remote', '--exit-code', '--heads', remote, f'refs/heads/{branch}'],
+        stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
+    )
+    if result.returncode not in (0, 2):
+        raise RuntimeError(f'git ls-remote failed for branch \'{branch}\': {result.stderr.strip()}')
+
+    return result.returncode == 0
+
+
+def emit(name: str, value: str) -> None:
+    github_output = os.environ.get('GITHUB_OUTPUT')
+    if github_output is None:
+        print(f'{name}={value}')
+        return
+
+    with open(github_output, 'a', encoding='utf-8') as f:
+        f.write(f'{name}={value}\n')
+
+
+def summarize(lines: list[str]) -> None:
+    github_step_summary = os.environ.get('GITHUB_STEP_SUMMARY')
+    if github_step_summary is None:
+        print('\n'.join(lines), file=sys.stderr)
+        return
+
+    with open(github_step_summary, 'a', encoding='utf-8') as f:
+        f.write('\n'.join(lines) + '\n')
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('record', help='the YAML file recording the analysis files of each data set')
+    parser.add_argument('--remote', default='origin', help='the remote whose branches are checked out')
+    args = parser.parse_args()
+
+    with open(args.record, encoding='utf-8') as f:
+        record = yaml.safe_load(f) or {}
+
+    include = []
+    excluded = {'skipped': [], 'no analysis file': [], 'no branch': []}
+    for data_set, description in sorted(record.items()):
+        description = description or {}
+        if (unknown := description.keys() - KNOWN_KEYS):
+            raise ValueError(f'data set \'{data_set}\' records unknown key(s): {", ".join(sorted(unknown))}')
+
+        if description.get('skip'):
+            excluded['skipped'].append(data_set)
+            continue
+
+        analysis_files = description.get('analysis_files') or []
+        if not analysis_files:
+            excluded['no analysis file'].append(data_set)
+            continue
+
+        if not branch_exists(args.remote, data_set):
+            excluded['no branch'].append(data_set)
+            print(f'::warning title=Missing branch::Data set {data_set} is recorded in '
+                  f'{args.record} but has no branch \'{data_set}\'; it will not be validated.')
+            continue
+
+        include.extend(
+            {'data_set': data_set, 'branch': data_set, 'analysis_file': analysis_file}
+            for analysis_file in analysis_files
+        )
+
+    emit('matrix', json.dumps({'include': include}))
+    emit('empty', 'true' if not include else 'false')
+
+    summary = ['## Analysis files to validate', '', '| Data set | Analysis file |', '| --- | --- |']
+    summary += [f'| `{row["data_set"]}` | `{row["analysis_file"]}` |' for row in include] or ['| _none_ | |']
+    summary += [
+        f'\nNot validated ({reason}): {", ".join(f"`{d}`" for d in data_sets)}.'
+        for reason, data_sets in excluded.items() if data_sets
+    ]
+    summarize(summary)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

.github/scripts/report_issue.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+#
+# Report the outcome recorded by validate_analysis.py: open an issue for the errors found, or
+# update the body of the issue already open for this data set and analysis file. Issues are never
+# closed automatically; a human decides when a report has been dealt with.
+#
+# Where EOS reports the line an error refers to, the error is also emitted as a workflow
+# annotation, and linked from the issue body. The annotation names a file that does not exist on
+# the branch this workflow runs from, so it is listed in the run rather than bound to the source;
+# the links in the issue body are pinned to the data set's own commit and do resolve.
+
+set -euo pipefail
+
+: "${DATA_SET:?}" "${ANALYSIS_FILE:?}" "${BRANCH:?}" "${RESULT_FILE:?}" "${GH_REPO:?}" "${GH_TOKEN:?}"
+: "${LABEL:=validation}"
+: "${SERVER_URL:=https://github.com}"
+: "${MAX_LINKS:=20}"
+
+count=$(jq '.errors | length' "${RESULT_FILE}")
+version=$(jq -r '.version' "${RESULT_FILE}")
+
+if [[ "${count}" -eq 0 ]]; then
+    echo "::notice title=Validated::${DATA_SET}: ${ANALYSIS_FILE} validates cleanly with EOS v${version}."
+    exit 0
+fi
+
+title="EOS-DATA-${DATA_SET}: Validation failed for analysis file \`${ANALYSIS_FILE}\`"
+blob="${SERVER_URL}/${GH_REPO}/blob/${COMMIT:-${BRANCH}}/${ANALYSIS_FILE}"
+
+# One annotation per error, so the Actions UI lists them with their position.
+jq -r --arg file "${ANALYSIS_FILE}" --arg set "${DATA_SET}" '
+    .errors[]
+    | "::error file=\($file)\(if .line then ",line=\(.line)" else "" end),title=EOS-DATA-\($set)::"
+      + (.message | gsub("%";"%25") | gsub("\n";"%0A") | gsub("\r";"%0D"))
+' "${RESULT_FILE}"
+
+body=$(mktemp)
+{
+    printf 'One or more issues were found when validating `%s` with EOS v%s.\n\n' "${ANALYSIS_FILE}" "${version}"
+    printf '```\n'
+    jq -r '.errors[] | (if .line then "line \(.line): " else "" end) + .message' "${RESULT_FILE}"
+    printf '```\n\n'
+
+    located=$(jq '[.errors[] | select(.line)] | length' "${RESULT_FILE}")
+    if [[ "${located}" -gt 0 ]]; then
+        printf '### Locations\n\n'
+        # Consecutive lines are collapsed into ranges, since one fault usually spans a whole block.
+        jq -r --arg blob "${blob}" --arg file "${ANALYSIS_FILE}" --argjson max "${MAX_LINKS}" '
+            [.errors[] | select(.line) | .line] | unique
+            | reduce .[] as $l ([];
+                if (length > 0 and .[-1][1] == $l - 1) then .[:-1] + [[.[-1][0], $l]] else . + [[$l, $l]] end)
+            | (.[:$max] | .[] | if .[0] == .[1]
+                  then "- [`\($file):\(.[0])`](\($blob)#L\(.[0]))"
+                  else "- [`\($file):\(.[0])-\(.[1])`](\($blob)#L\(.[0])-L\(.[1]))" end),
+              (if length > $max then "- … and \(length - $max) further ranges" else empty end)
+        ' "${RESULT_FILE}"
+        printf '\n'
+    fi
+
+    printf -- '---\nBranch `%s` at %s, checked by [this workflow run](%s).\n' \
+        "${BRANCH}" "${COMMIT:-unknown commit}" "${RUN_URL:-}"
+} > "${body}"
+
+# Match on the exact title: one issue per data set and analysis file.
+number=$(gh issue list --repo "${GH_REPO}" --state open --label "${LABEL}" --limit 200 --json number,title \
+    | jq -r --arg title "${title}" 'map(select(.title == $title)) | .[0].number // empty')
+
+if [[ -n "${number}" ]]; then
+    gh issue edit "${number}" --repo "${GH_REPO}" --body-file "${body}"
+    echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); updated issue #${number}."
+else
+    gh issue create --repo "${GH_REPO}" --title "${title}" --label "${LABEL}" --body-file "${body}"
+    echo "${DATA_SET}: ${ANALYSIS_FILE} reported ${count} error(s); opened a new issue."
+fi

.github/scripts/validate_analysis.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Validate a single EOS analysis file and record the errors found.
+
+Runs inside the EOS container, with the working directory set to the checked-out data set so that
+the deep validation phase resolves relative data paths as it would for a user. The result is
+written as JSON, since ``eos-analysis validate`` exits 0 whether or not it finds anything.
+
+``eos.validate`` returns one ``eos.Diagnostic`` per finding, carrying a severity and, where EOS can
+resolve one, the line of the analysis file it refers to. Only the errors are reported: warnings
+(an unused prior, say, or a constraint file that has not been produced yet) are expected for
+released data sets and must not raise an alarm. This requires EOS 1.1 or later; up to 1.0.21
+``eos.validate`` returned nothing and knew no severities.
+"""
+
+import argparse
+import contextlib
+import io
+import json
+import sys
+import traceback
+
+
+def errors_from(diagnostics) -> list[dict]:
+    """Return one {'line', 'message'} per error. 'line' is None where EOS resolved no position."""
+    from eos.diagnostic import Severity
+
+    return [
+        {'line': d.line, 'message': f'{d.location()}: {d.message}'}
+        for d in diagnostics if d.severity is Severity.ERROR
+    ]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('analysis_file', help='the analysis file to validate, relative to the working directory')
+    parser.add_argument('--output', required=True, help='the JSON file to write the result to')
+    args = parser.parse_args()
+
+    import eos
+
+    result = {'analysis_file': args.analysis_file, 'version': eos.__version__, 'errors': [], 'log': ''}
+
+    # eos.validate() prints as it goes; capture that so only the filtered errors are reported.
+    printed = io.StringIO()
+    try:
+        with contextlib.redirect_stdout(printed):
+            diagnostics = eos.validate(analysis_file=args.analysis_file)
+        result['errors'] = errors_from(diagnostics)
+    except Exception:
+        result['errors'] = [{'line': None,
+                             'message': f'Validation aborted with an unhandled exception:\n{traceback.format_exc()}'}]
+    result['log'] = printed.getvalue()
+
+    with open(args.output, 'w', encoding='utf-8') as f:
+        json.dump(result, f, indent=2)
+
+    print(f'{args.analysis_file}: EOS v{result["version"]}, {len(result["errors"])} error(s)', file=sys.stderr)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())