diff --git a/.github/scripts/check_pr_review_threads.py b/.github/scripts/check_pr_review_threads.py new file mode 100755 index 0000000..e836046 --- /dev/null +++ b/.github/scripts/check_pr_review_threads.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Fail a PR when high-severity review feedback is still unresolved. + +Port of .github/scripts/check-pr-review-threads.rb. The Ruby original could not +run on the ARC runner image, which ships no Ruby interpreter. This port uses the +Python standard library only -- json, re, subprocess, argparse -- so it has no +dependency on anything beyond the interpreter itself. + +Behaviour is intentionally identical to the Ruby version: same severity ranking, +same informational-summary suppression, same GraphQL queries and pagination, +same annotations, same exit codes (0 clean, 1 blocking feedback, 2 bad usage). +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys + +SEVERITY_RANK = { + "none": 0, + "low": 1, + "medium": 2, + "high": 3, + "p1": 4, + "p0": 5, +} + +INFORMATIONAL_HEADING = re.compile(r"\A##\s+(PR\s+Summary|Summary|Walkthrough)\b", re.IGNORECASE) +INFORMATIONAL_AUTHOR = re.compile(r"\A(cursor|coderabbitai|chatgpt-codex-connector)\b", re.IGNORECASE) + +SEVERITY_PATTERNS = ( + ("p0", (re.compile(r"\bP0\b", re.IGNORECASE),)), + ("p1", (re.compile(r"\bP1\b", re.IGNORECASE),)), + ("high", (re.compile(r"\bHigh Severity\b", re.IGNORECASE), re.compile(r"!\[High Badge\]", re.IGNORECASE))), + ("medium", (re.compile(r"\bMedium Severity\b", re.IGNORECASE), re.compile(r"!\[Medium Badge\]", re.IGNORECASE))), + ("low", (re.compile(r"\bLow Severity\b", re.IGNORECASE), re.compile(r"!\[Low Badge\]", re.IGNORECASE))), +) + + +def _str(value) -> str: + return "" if value is None else str(value) + + +def _dig(node, *keys): + for key in keys: + if not isinstance(node, dict): + return None + node = node.get(key) + return node + + +def _nodes(value): + return value if isinstance(value, list) else [] + + +def first_nonblank_line(body) -> str: + for line in _str(body).splitlines(): + stripped = line.strip() + if stripped: + return stripped + return "" + + +def informational_summary(body, author=None) -> bool: + if not INFORMATIONAL_HEADING.search(first_nonblank_line(body)): + return False + return bool(INFORMATIONAL_AUTHOR.search(_str(author))) + + +def severity(body) -> str: + text = _str(body) + for level, patterns in SEVERITY_PATTERNS: + if any(pattern.search(text) for pattern in patterns): + return level + return "none" + + +def severity_comment(comments): + """Return (severity, comment) for the highest-severity comment, or None.""" + candidates = [] + for comment in _nodes(comments): + detected = severity(_dig(comment, "body")) + if SEVERITY_RANK[detected] <= SEVERITY_RANK["none"]: + continue + candidates.append((detected, comment)) + if not candidates: + return None + # max() keeps the first maximal element, matching Ruby's max_by. + return max(candidates, key=lambda pair: SEVERITY_RANK[pair[0]]) + + +def unresolved_threads(payload, min_severity="high", include_outdated=False): + threshold = SEVERITY_RANK[min_severity] + threads = _nodes(_dig(payload, "data", "repository", "pullRequest", "reviewThreads", "nodes")) + matches = [] + for thread in threads: + if not isinstance(thread, dict): + continue + if thread.get("isResolved"): + continue + if thread.get("isOutdated") and not include_outdated: + continue + + found = severity_comment(_dig(thread, "comments", "nodes")) + detected, comment = found if found else ("none", {}) + if SEVERITY_RANK[detected] < threshold: + continue + + matches.append( + { + "kind": "review_thread", + "id": thread.get("id"), + "path": thread.get("path"), + "line": thread.get("line"), + "is_outdated": thread.get("isOutdated"), + "severity": detected, + "url": comment.get("url"), + "body": _str(comment.get("body")), + } + ) + return matches + + +def top_level_feedback(payload, min_severity="high"): + threshold = SEVERITY_RANK[min_severity] + pull_request = _dig(payload, "data", "repository", "pullRequest") or {} + current_head_oid = _str(pull_request.get("headRefOid")) + feedback = [] + + for comment in _nodes(_dig(pull_request, "comments", "nodes")): + if not isinstance(comment, dict): + continue + author = _dig(comment, "author", "login") + if informational_summary(comment.get("body"), author=author): + continue + detected = severity(comment.get("body")) + if SEVERITY_RANK[detected] < threshold: + continue + feedback.append( + { + "kind": "pr_comment", + "severity": detected, + "url": comment.get("url"), + "body": _str(comment.get("body")), + "author": author, + } + ) + + for review in _nodes(_dig(pull_request, "reviews", "nodes")): + if not isinstance(review, dict): + continue + author = _dig(review, "author", "login") + if informational_summary(review.get("body"), author=author): + continue + detected = severity(review.get("body")) + if SEVERITY_RANK[detected] < threshold: + continue + review_commit_oid = _str(_dig(review, "commit", "oid")) + if current_head_oid and review_commit_oid and review_commit_oid != current_head_oid: + continue + feedback.append( + { + "kind": "pr_review", + "severity": detected, + "url": review.get("url"), + "body": _str(review.get("body")), + "author": author, + "state": review.get("state"), + } + ) + + return feedback + + +def blocking_feedback(payload, min_severity="high", include_outdated=False): + return unresolved_threads( + payload, min_severity=min_severity, include_outdated=include_outdated + ) + top_level_feedback(payload, min_severity=min_severity) + + +GRAPHQL_QUERY = """ +query($owner:String!,$repo:String!,$number:Int!) { + repository(owner:$owner, name:$repo) { + pullRequest(number:$number) { + headRefOid + comments(first:100) { + pageInfo { hasNextPage endCursor } + nodes { author { login } body url } + } + reviews(first:100) { + pageInfo { hasNextPage endCursor } + nodes { author { login } body commit { oid } state url } + } + reviewThreads(first:100) { + pageInfo { hasNextPage endCursor } + nodes { + id + isResolved + isOutdated + path + line + comments(first:20) { nodes { body url } } + } + } + } + } +} +""" + +COMMENTS_PAGE_QUERY = """ +query($owner:String!,$repo:String!,$number:Int!,$after:String) { + repository(owner:$owner, name:$repo) { + pullRequest(number:$number) { + comments(first:100, after:$after) { + pageInfo { hasNextPage endCursor } + nodes { author { login } body url } + } + } + } +} +""" + +REVIEWS_PAGE_QUERY = """ +query($owner:String!,$repo:String!,$number:Int!,$after:String) { + repository(owner:$owner, name:$repo) { + pullRequest(number:$number) { + reviews(first:100, after:$after) { + pageInfo { hasNextPage endCursor } + nodes { author { login } body commit { oid } state url } + } + } + } +} +""" + +REVIEW_THREADS_PAGE_QUERY = """ +query($owner:String!,$repo:String!,$number:Int!,$after:String) { + repository(owner:$owner, name:$repo) { + pullRequest(number:$number) { + reviewThreads(first:100, after:$after) { + pageInfo { hasNextPage endCursor } + nodes { + id + isResolved + isOutdated + path + line + comments(first:20) { nodes { body url } } + } + } + } + } +} +""" + + +class GraphQLError(RuntimeError): + pass + + +def fetch_graphql(owner, name, pr, query, cursor=None): + args = [ + "gh", "api", "graphql", + "-f", f"owner={owner}", + "-f", f"repo={name}", + "-F", f"number={pr}", + "-f", f"query={query}", + ] + if cursor: + args += ["-f", f"after={cursor}"] + + try: + completed = subprocess.run(args, capture_output=True, text=True, check=False) + except FileNotFoundError as exc: + # Never degrade to "no feedback found" when the GitHub CLI is absent. + raise GraphQLError(f"gh is not installed on this runner: {exc}") from exc + + if completed.returncode != 0: + raise GraphQLError(f"gh api graphql failed: {completed.stderr.strip()}") + return json.loads(completed.stdout) + + +def fetch_connection_tail(owner, name, pr, query, connection_name, first_connection): + nodes = [] + connection = first_connection or {} + page_info = connection.get("pageInfo") or {} + while page_info.get("hasNextPage"): + cursor = _str(page_info.get("endCursor")) + if not cursor: + raise GraphQLError(f"gh api graphql failed: missing {connection_name} endCursor") + payload = fetch_graphql(owner, name, pr, query, cursor=cursor) + connection = _dig(payload, "data", "repository", "pullRequest", connection_name) or {} + nodes.extend(_nodes(connection.get("nodes"))) + page_info = connection.get("pageInfo") or {} + return nodes + + +def merge_pull_request_connections(payload, comments=None, reviews=None, review_threads=None): + merged = payload if isinstance(payload, dict) else {} + if not isinstance(merged.get("data"), dict): + merged["data"] = {} + if not isinstance(merged["data"].get("repository"), dict): + merged["data"]["repository"] = {} + if not isinstance(merged["data"]["repository"].get("pullRequest"), dict): + merged["data"]["repository"]["pullRequest"] = {} + pull_request = merged["data"]["repository"]["pullRequest"] + + for key, value in (("comments", comments), ("reviews", reviews), ("reviewThreads", review_threads)): + if value is None: + continue + if not isinstance(pull_request.get(key), dict): + pull_request[key] = {} + pull_request[key]["nodes"] = value + return merged + + +def fetch_payload(repo, pr): + owner, _, name = repo.partition("/") + payload = fetch_graphql(owner, name, pr, GRAPHQL_QUERY) + pull_request = _dig(payload, "data", "repository", "pullRequest") or {} + comments_connection = pull_request.get("comments") or {} + reviews_connection = pull_request.get("reviews") or {} + threads_connection = pull_request.get("reviewThreads") or {} + + comments = _nodes(comments_connection.get("nodes")) + fetch_connection_tail( + owner, name, pr, COMMENTS_PAGE_QUERY, "comments", comments_connection + ) + reviews = _nodes(reviews_connection.get("nodes")) + fetch_connection_tail( + owner, name, pr, REVIEWS_PAGE_QUERY, "reviews", reviews_connection + ) + review_threads = _nodes(threads_connection.get("nodes")) + fetch_connection_tail( + owner, name, pr, REVIEW_THREADS_PAGE_QUERY, "reviewThreads", threads_connection + ) + + return merge_pull_request_connections( + payload, comments=comments, reviews=reviews, review_threads=review_threads + ) + + +def annotation(thread) -> str: + path = thread.get("path") + if not path: + title = f"unresolved {thread['severity'].upper()} {thread['kind'].replace('_', ' ')}" + return f"::error title={title}::{thread['url']}" + + line = thread.get("line") + location = f"{path}:{line}" if line is not None else path + title = f"unresolved {thread['severity'].upper()} review thread" + return f"::error file={path},line={line if line is not None else 1},title={title}::{location} {thread['url']}" + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(add_help=True) + parser.add_argument("--repo", help="Repository to inspect, as OWNER/REPO") + parser.add_argument("--pr", type=int, help="Pull request number") + parser.add_argument("--min-severity", default="high", help="Minimum severity: low, medium, high, p1, p0") + parser.add_argument("--include-outdated", action="store_true", help="Include outdated unresolved threads") + parser.add_argument("--json", dest="json_path", help="Read GraphQL payload from a file instead of gh") + options = parser.parse_args(argv) + + min_severity = options.min_severity.lower() + if min_severity not in SEVERITY_RANK: + print(f"invalid --min-severity {min_severity!r}", file=sys.stderr) + return 2 + + if options.json_path: + with open(options.json_path, encoding="utf-8") as handle: + payload = json.load(handle) + else: + missing = [name for name in ("repo", "pr") if not _str(getattr(options, name))] + if missing: + print(f"missing required options: {', '.join(missing)}", file=sys.stderr) + return 2 + payload = fetch_payload(options.repo, options.pr) + + feedback = blocking_feedback( + payload, min_severity=min_severity, include_outdated=options.include_outdated + ) + + if not feedback: + print(f"No unresolved PR feedback at or above {min_severity} severity.") + return 0 + + print( + f"Found {len(feedback)} unresolved PR feedback item(s) at or above {min_severity} severity:", + file=sys.stderr, + ) + for thread in feedback: + if thread.get("path"): + location = f"{thread['path']}:{thread.get('line') if thread.get('line') is not None else '?'}" + else: + location = _str(thread.get("kind")) + print(f"- [{thread['severity']}] {location} {thread.get('url')}", file=sys.stderr) + print(annotation(thread)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/fixtures/review-threads-blocking.json b/.github/scripts/fixtures/review-threads-blocking.json new file mode 100644 index 0000000..740c8b0 --- /dev/null +++ b/.github/scripts/fixtures/review-threads-blocking.json @@ -0,0 +1,43 @@ +{ + "data": { + "repository": { + "pullRequest": { + "headRefOid": "1111111111111111111111111111111111111111", + "comments": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [] + }, + "reviews": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [] + }, + "reviewThreads": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [ + { + "id": "RT_blocking", + "isResolved": false, + "isOutdated": false, + "path": "src/lib.rs", + "line": 42, + "comments": { + "nodes": [ + { + "body": "High Severity: unchecked index can panic.", + "url": "https://example.invalid/t/3" + } + ] + } + } + ] + } + } + } + } +} diff --git a/.github/scripts/fixtures/review-threads-clean.json b/.github/scripts/fixtures/review-threads-clean.json new file mode 100644 index 0000000..df73d68 --- /dev/null +++ b/.github/scripts/fixtures/review-threads-clean.json @@ -0,0 +1,78 @@ +{ + "data": { + "repository": { + "pullRequest": { + "headRefOid": "0000000000000000000000000000000000000000", + "comments": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [ + { + "author": { + "login": "coderabbitai" + }, + "body": "## PR Summary\n\nEverything looks fine.", + "url": "https://example.invalid/c/1" + } + ] + }, + "reviews": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [ + { + "author": { + "login": "reviewer" + }, + "body": "Low Severity: naming nit.", + "commit": { + "oid": "0000000000000000000000000000000000000000" + }, + "state": "COMMENTED", + "url": "https://example.invalid/r/1" + } + ] + }, + "reviewThreads": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [ + { + "id": "RT_clean_resolved", + "isResolved": true, + "isOutdated": false, + "path": "src/lib.rs", + "line": 10, + "comments": { + "nodes": [ + { + "body": "High Severity: fixed and resolved.", + "url": "https://example.invalid/t/1" + } + ] + } + }, + { + "id": "RT_clean_low", + "isResolved": false, + "isOutdated": false, + "path": "src/lib.rs", + "line": 22, + "comments": { + "nodes": [ + { + "body": "Low Severity: optional cleanup.", + "url": "https://example.invalid/t/2" + } + ] + } + } + ] + } + } + } + } +} diff --git a/.github/workflows/codex-rails-check.yml b/.github/workflows/codex-rails-check.yml new file mode 100644 index 0000000..089ae87 --- /dev/null +++ b/.github/workflows/codex-rails-check.yml @@ -0,0 +1,247 @@ +name: codex-rails-check + +# Reusable org rails check, called by repositories across the organisation. +# It runs on whatever runner the caller names, so it may only depend on tools +# proven to exist on every runner image we operate. Ruby exists on the older +# self-hosted image and not on the ARC image, so every parse step below runs on +# python3 + PyYAML, asserted up front by the preflight. + +on: + pull_request: + paths: + - "AGENTS.md" + - "**/AGENTS.md" + - ".agents/skills/**" + - ".github/scripts/**" + - ".github/ISSUE_TEMPLATE/**" + - ".github/pull_request_template.md" + - ".github/workflows/**" + - ".github/workflow-templates/**" + - "profile/**" + - "test/**" + workflow_call: + inputs: + require_agents: + description: "Fail when the repository does not have AGENTS.md" + required: false + type: boolean + default: false + runner_label: + description: "Runner label used for the validation job" + required: false + type: string + default: ubuntu-latest + +permissions: + contents: read + +jobs: + validate: + runs-on: ${{ inputs.runner_label || 'ubuntu-latest' }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - name: Preflight the YAML toolchain + shell: bash + run: | + set -euo pipefail + + if ! command -v python3 >/dev/null 2>&1; then + echo "::error::python3 is not on this runner (${RUNNER_NAME:-unknown}). codex-rails-check cannot parse YAML without it." + exit 1 + fi + python3 -V + + if ! python3 -c 'import yaml' >/dev/null 2>&1; then + echo "::error::PyYAML is not importable on this runner (${RUNNER_NAME:-unknown}). Install python3-yaml on the runner image or add actions/setup-python + 'pip install pyyaml' before this workflow." + exit 1 + fi + python3 -c 'import yaml; print("PyYAML", yaml.__version__)' + + cat > "${RUNNER_TEMP}/yamlcheck.py" <<'PY' + """Parse each argument as YAML. Exit 1 naming the first file that fails.""" + import sys + import yaml + + failed = False + for path in sys.argv[1:]: + try: + with open(path, encoding="utf-8") as handle: + yaml.safe_load(handle) + except (yaml.YAMLError, OSError, UnicodeDecodeError) as exc: + detail = " ".join(str(exc).split()) + print(f"::error file={path}::{path} is not valid YAML: {detail}") + failed = True + else: + print(f"ok {path}") + sys.exit(1 if failed else 0) + PY + + # Prove the validator still rejects bad input. Without this, a broken + # parser would let every later step pass while checking nothing. + selftest_dir="${RUNNER_TEMP}/yamlcheck-selftest" + mkdir -p "${selftest_dir}" + printf 'name: ok\nvalue: 1\n' > "${selftest_dir}/good.yml" + printf 'name: [unclosed\n bad: : :\n' > "${selftest_dir}/bad.yml" + + python3 "${RUNNER_TEMP}/yamlcheck.py" "${selftest_dir}/good.yml" >/dev/null + if python3 "${RUNNER_TEMP}/yamlcheck.py" "${selftest_dir}/bad.yml" >/dev/null 2>&1; then + echo "::error::YAML validator self-test failed: invalid YAML was accepted. This gate would have passed without checking anything." + exit 1 + fi + echo "YAML validator self-test passed (accepts valid YAML, rejects invalid YAML)." + + - name: Validate issue template YAML + shell: bash + run: | + set -euo pipefail + shopt -s globstar nullglob + files=(.github/ISSUE_TEMPLATE/*.yml .github/ISSUE_TEMPLATE/*.yaml) + if [ "${#files[@]}" -eq 0 ]; then + echo "No issue template YAML files found." + exit 0 + fi + python3 "${RUNNER_TEMP}/yamlcheck.py" "${files[@]}" + + - name: Validate workflow YAML + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + files=(.github/workflows/*.yml .github/workflows/*.yaml .github/workflow-templates/*.yml .github/workflow-templates/*.yaml) + if [ "${#files[@]}" -eq 0 ]; then + echo "No workflow YAML files found." + exit 0 + fi + python3 "${RUNNER_TEMP}/yamlcheck.py" "${files[@]}" + + - name: Validate workflow template metadata + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + files=(.github/workflow-templates/*.properties.json) + if [ "${#files[@]}" -eq 0 ]; then + echo "No workflow template metadata files found." + exit 0 + fi + python3 - "${files[@]}" <<'PY' + import json + import os + import re + import sys + + OCTICON = re.compile(r"\Aocticon [a-z0-9-]+\Z") + + for path in sys.argv[1:]: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + icon = str(data.get("iconName") or "") + if icon and not OCTICON.match(icon) and not os.path.exists( + f".github/workflow-templates/{icon}.svg" + ): + print( + f'{path}: iconName must be an octicon reference like "octicon tag" ' + "or a local SVG basename", + file=sys.stderr, + ) + sys.exit(1) + print(f"ok {path}") + PY + + - name: Check AGENTS.md files + shell: bash + env: + REQUIRE_AGENTS: ${{ inputs.require_agents || false }} + run: | + set -euo pipefail + shopt -s globstar nullglob + if [ "${REQUIRE_AGENTS}" = "true" ] && [ ! -f AGENTS.md ]; then + echo "::error::AGENTS.md is required for this repository." + exit 1 + fi + + agents=(**/AGENTS.md) + if [ "${#agents[@]}" -eq 0 ]; then + echo "No AGENTS.md files found." + exit 0 + fi + + for agent in "${agents[@]}"; do + test -s "${agent}" || { echo "::error::${agent} is empty"; exit 1; } + echo "ok ${agent}" + done + + - name: Validate skill frontmatter + shell: bash + run: | + set -euo pipefail + shopt -s globstar nullglob + skills=(.agents/skills/**/SKILL.md) + if [ "${#skills[@]}" -eq 0 ]; then + echo "No repo skills found." + exit 0 + fi + python3 - "${skills[@]}" <<'PY' + import re + import sys + + import yaml + + NAME = re.compile(r"\A[a-z0-9][a-z0-9_-]*\Z") + SPLIT = re.compile(r"^---\s*$", re.MULTILINE) + + failed = False + for path in sys.argv[1:]: + print(f"checking {path}") + with open(path, encoding="utf-8") as handle: + text = handle.read() + + if not text.startswith("---\n"): + print(f"{path}: missing YAML frontmatter", file=sys.stderr) + failed = True + continue + + parts = SPLIT.split(text, 2) + front = parts[1] if len(parts) > 1 else "" + try: + data = yaml.safe_load(front) + except yaml.YAMLError as exc: + detail = " ".join(str(exc).split()) + print(f"{path}: frontmatter is not valid YAML: {detail}", file=sys.stderr) + failed = True + continue + + if not isinstance(data, dict) or not NAME.match(str(data.get("name") or "")): + print(f"{path}: frontmatter must include kebab/snake-safe name", file=sys.stderr) + failed = True + continue + + if len(str(data.get("description") or "").strip()) < 20: + print(f"{path}: frontmatter must include a useful description", file=sys.stderr) + failed = True + + sys.exit(1 if failed else 0) + PY + + - name: Run repo Ruby tests + shell: bash + run: | + set -euo pipefail + shopt -s globstar nullglob + tests=(test/**/*_test.rb test/*_test.rb) + if [ "${#tests[@]}" -eq 0 ]; then + echo "No Ruby tests found." + exit 0 + fi + + # Reached only by repos that actually ship a Ruby suite. Fail loudly + # rather than skipping: a silently skipped test suite is worse than a + # red check, and the ARC image has no Ruby interpreter. + if ! command -v ruby >/dev/null 2>&1; then + echo "::error::This repository has ${#tests[@]} Ruby test file(s) but the runner (${RUNNER_NAME:-unknown}) has no ruby. Add ruby/setup-ruby to the calling workflow or run this job on a runner image that ships Ruby." + exit 1 + fi + + ruby -Itest "${tests[@]}" diff --git a/.github/workflows/review-thread-guard.yml b/.github/workflows/review-thread-guard.yml new file mode 100644 index 0000000..fcef288 --- /dev/null +++ b/.github/workflows/review-thread-guard.yml @@ -0,0 +1,152 @@ +name: review-thread-guard + +# Produces the `unresolved-review-threads` check, which callers wire up as a +# required status check on protected branches. It runs on the caller's runner, +# so it may only depend on tools present on every runner image we operate. The +# guard used to shell out to Ruby; the ARC image ships no Ruby, so it now runs +# on python3 (standard library only) with an explicit preflight and a fixture +# self-test that proves the gate can still fail. + +on: + workflow_call: + inputs: + pr_number: + description: "Pull request number to inspect" + required: true + type: string + min_severity: + description: "Minimum severity to block on" + required: false + default: "high" + type: string + runner_label: + description: "Runner label used for the validation job" + required: false + type: string + default: ubuntu-latest + guard_ref: + description: "evalops/.github ref to checkout for guard scripts" + required: false + type: string + settle_seconds: + description: "Seconds to wait before checking review feedback so bot reviews can finish before auto-merge" + required: false + type: string + default: "90" + workflow_dispatch: + inputs: + repo: + description: "Repository to inspect, as owner/name" + required: false + type: string + pr_number: + description: "Pull request number to inspect" + required: true + type: string + min_severity: + description: "Minimum severity to block on" + required: false + default: "high" + type: choice + options: + - high + - p1 + - p0 + guard_ref: + description: "evalops/.github ref to checkout for guard scripts" + required: false + type: string + settle_seconds: + description: "Seconds to wait before checking review feedback so bot reviews can finish before auto-merge" + required: false + type: string + default: "90" + +permissions: + contents: read + pull-requests: read + +jobs: + unresolved-review-threads: + runs-on: ${{ inputs.runner_label || 'ubuntu-latest' }} + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + repository: evalops/.github + ref: ${{ inputs.guard_ref || github.workflow_sha }} + path: evalops-github + + - name: Preflight the guard toolchain + shell: bash + run: | + set -euo pipefail + + if ! command -v python3 >/dev/null 2>&1; then + echo "::error::python3 is not on this runner (${RUNNER_NAME:-unknown}). The review-thread guard cannot run, and it must not be skipped." + exit 1 + fi + python3 -V + + if ! command -v gh >/dev/null 2>&1; then + echo "::error::gh is not on this runner (${RUNNER_NAME:-unknown}). The guard reads review threads over the GitHub GraphQL API and cannot verify anything without it." + exit 1 + fi + gh --version | head -1 + + test -f evalops-github/.github/scripts/check_pr_review_threads.py + + - name: Prove the guard can still fail + shell: bash + run: | + set -euo pipefail + guard=evalops-github/.github/scripts/check_pr_review_threads.py + fixtures=evalops-github/.github/scripts/fixtures + log="${RUNNER_TEMP}/guard-selftest.log" + + # A clean PR must pass. Output is captured so the fixtures never emit + # ::error:: annotations against the real PR under review. + if ! python3 "${guard}" --json "${fixtures}/review-threads-clean.json" --min-severity high >"${log}" 2>&1; then + echo "::error::review-thread guard self-test failed: the clean fixture was reported as blocking." + sed 's/^::/ /' "${log}" + exit 1 + fi + + # A PR with an unresolved high-severity thread must fail. If this ever + # passes, the guard has stopped verifying and every PR would merge + # with a green check. + if python3 "${guard}" --json "${fixtures}/review-threads-blocking.json" --min-severity high >"${log}" 2>&1; then + echo "::error::review-thread guard self-test failed: a blocking fixture was reported clean. This gate is not verifying anything." + sed 's/^::/ /' "${log}" + exit 1 + fi + + echo "Guard self-test passed: clean fixture exits 0, blocking fixture exits non-zero." + + - name: Let asynchronous review bots settle + env: + SETTLE_SECONDS: ${{ inputs.settle_seconds || '90' }} + run: | + set -euo pipefail + case "${SETTLE_SECONDS}" in + ''|*[!0-9]*) + echo "::error::settle_seconds must be a non-negative integer" + exit 2 + ;; + esac + if [ "${SETTLE_SECONDS}" -gt 0 ]; then + echo "Waiting ${SETTLE_SECONDS}s before checking review feedback." + sleep "${SETTLE_SECONDS}" + fi + + - name: Fail on unresolved high-priority review threads + env: + GH_TOKEN: ${{ github.token }} + TARGET_REPO: ${{ inputs.repo || github.repository }} + PR_NUMBER: ${{ inputs.pr_number }} + MIN_SEVERITY: ${{ inputs.min_severity || 'high' }} + run: | + python3 evalops-github/.github/scripts/check_pr_review_threads.py \ + --repo "${TARGET_REPO}" \ + --pr "${PR_NUMBER}" \ + --min-severity "${MIN_SEVERITY}"