diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index 69f21f7b..6f42eb7c 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -51,6 +51,8 @@ jobs: runs-on: ${{ github.repository_owner == 'intel' && github.event_name == 'pull_request' && 'dokimi-ec2' || 'ubuntu-latest' }} permissions: contents: read + # The changes step reads this workflow's earlier runs (gh api). + actions: read timeout-minutes: 15 env: # Pinned, not `main` -- an upstream rename/restructure of @@ -64,24 +66,72 @@ jobs: with: persist-credentials: false fetch-depth: 0 - # Whether a PR changes anything besides Markdown. A docs-only PR skips the - # steps below (the Docker smoke test cannot be affected by it); the benchmark - # job measures the docs, so it always runs. The job itself still takes a - # runner, as its steps are skipped, not the job. + # Docs-only runs skip the code steps (the benchmark still runs). A push diffs from the last verified smoke run, else all; docs are the Markdown in DOCS below. + # dokimi/data/, tests and .agent-eval count as code, and a base that moved since that run's merge also runs everything. Force pushes run all. - id: changes working-directory: . + # Explicit, so the step runs as `bash --noprofile --norc -eo pipefail`, the flags the test harness uses. + shell: bash env: EVENT: ${{ github.event_name }} + ACTION: ${{ github.event.action }} BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REF: ${{ github.head_ref }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + DOCS: '^([^/]+|dokimi/[^/]+|dokimi/docs/(.+/)?[^/]+|dokimi/changelog\.d/[^/]+|\.github/(.+/)?[^/]+)\.md$' run: | - if [ "$EVENT" != pull_request ]; then echo code=true >> "$GITHUB_OUTPUT"; exit 0; fi - # The base branch's current tip (fetched by the checkout), not base.sha - # (the base when the PR was opened). - if git diff --name-only "origin/$BASE_REF"...HEAD | grep -qv '\.md$'; then - echo code=true >> "$GITHUB_OUTPUT" + set_code() { echo "code=$1" >> "$GITHUB_OUTPUT"; echo "changes: code=$1 ($2)"; } + # Newest commit reachable from AFTER whose smoke run passed (Verify pass rate succeeded): prints " ", + # or nothing if none. An unreadable run list also prints nothing; the caller then runs everything. + last_verified() { + runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs" --method GET \ + -f branch="$HEAD_REF" -f event=pull_request -f status=completed -f per_page=20 \ + --jq '.workflow_runs[] | "\(.id) \(.head_sha)"') || return 0 + while read -r id sha; do + [ -n "$id" ] || continue + git cat-file -e "$sha^{commit}" 2>/dev/null && git merge-base --is-ancestor "$sha" "$AFTER" 2>/dev/null || continue + verdict=$(gh api "repos/$REPO/actions/runs/$id/jobs" \ + --jq '.jobs[] | select(.name | startswith("External catalog smoke test")) | .steps[] | select(.name == "Verify pass rate") | .conclusion') || continue + if [ "$verdict" = success ]; then echo "$sha $id"; return 0; fi + done <<< "$runs" + } + if [ "$EVENT" != pull_request ]; then set_code true "not a pull request"; exit 0; fi + # --no-renames: a rename from code to Markdown must count as the code it removes. + if [ "$ACTION" = synchronize ]; then + if ! { [ -n "$BEFORE" ] && [ -n "$AFTER" ] \ + && git cat-file -e "$BEFORE^{commit}" && git cat-file -e "$AFTER^{commit}" \ + && git merge-base --is-ancestor "$BEFORE" "$AFTER"; } 2>/dev/null; then + set_code true "force push, or the commits are not fetched"; exit 0 + fi + # Diffed from the last passed run's commit, not from BEFORE: a run this push + # cancelled or that failed verified nothing, so its code is diffed again. + read -r from verified_run <<< "$(last_verified)" || true + if [ -z "$from" ]; then set_code true "push: no passed smoke run to diff from"; exit 0; fi + # The diff above does not show base changes: the run tested its merge with the base it was built on. + # Skip only if origin/$BASE_REF is still that base (the smoke job records it; no record means run everything). + dl=$(mktemp -d) + tested_base=$(gh run download "$verified_run" -R "$REPO" -n external-catalog-smoke-runs -D "$dl" >/dev/null 2>&1 && cat "$dl/base.sha") || tested_base= + base=$(git rev-parse --verify -q "origin/$BASE_REF") || base= + if [ -z "$tested_base" ] || [ "$tested_base" != "$base" ]; then + set_code true "push: the base moved since ${from:0:7}, or that run did not record it"; exit 0 + fi + scope="push since ${from:0:7}" + files=$(git -c core.quotepath=false diff --no-renames --name-only "$from" "$AFTER") \ + || { set_code true "push diff failed"; exit 0; } else - echo code=false >> "$GITHUB_OUTPUT" + # The base branch's current tip (fetched by the checkout), not the event's pull_request.base.sha + # (the base when the PR was opened). + scope=pr + files=$(git -c core.quotepath=false diff --no-renames --name-only "origin/$BASE_REF"...HEAD) \ + || { set_code true "PR diff failed"; exit 0; } fi + if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi + # A here-string, not a pipe: grep -q may exit early, and under pipefail the pipe's printf would get SIGPIPE. + if grep -qvE "$DOCS" <<< "$files"; then set_code true "$scope changes code"; exit 0; fi + set_code false "$scope changes only documentation" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 if: steps.changes.outputs.code == 'true' with: @@ -121,6 +171,22 @@ jobs: --tasks-root "${{ runner.temp }}/uxl-skills/evaluation/harbor/tasks" \ --harnesses docker-oracle,docker-probe \ --out-json "${{ github.workspace }}/logs/external-catalog-smoke/runs.json" + # The base the merge commit (refs/pull/N/merge) was built on, its first parent; not origin/$BASE_REF, which may have moved since. + # A PR that does not merge records nothing. A later docs-only push skips only while origin/$BASE_REF is still this one. + - name: Record the base this run tested + if: steps.changes.outputs.code == 'true' && github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + RECORD: ${{ github.workspace }}/logs/external-catalog-smoke/base.sha + run: | + # The checkout is the PR's merge commit; its first parent is the base tip the merge was built on. + # Recorded only if that parent is on origin/; else nothing, so the next push runs everything. + rm -f "$RECORD" + if git rev-parse --verify -q HEAD^2 > /dev/null \ + && parent=$(git rev-parse --verify HEAD^1) \ + && git merge-base --is-ancestor "$parent" "origin/$BASE_REF"; then + echo "$parent" > "$RECORD" + fi # Every cell passed and none errored (an errored cell leaves pass_rate's # denominator, so pass_rate alone would read 1.0), and the probe's fetch # was denied by the egress proxy; see scripts/check_smoke_runs.py. diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py new file mode 100644 index 00000000..511d98b3 --- /dev/null +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -0,0 +1,527 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""agent-quality's `changes` step decides whether a push runs the smoke test. + +A docs-only push may skip the code steps only when everything since the last commit +whose smoke run passed its code steps is documentation, and the base branch is still the +one that run's merge was built on (the smoke job records it). A cancelled or failed run +does not count as passed, so the code it carried is diffed again. The changes step and +the record step are run here as the workflow runs them (their scripts, read from the +YAML); the record step runs on a merge the test builds. `gh` is replaced by a stub that +serves run and job fixtures. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/agent-quality.yml" +SMOKE_JOB = "external-catalog-smoke" +SMOKE_NAME = "External catalog smoke test (uxlfoundation/skills)" +BULK_FILES = 2000 + +# The stub `gh`: `gh api` serves runs.json and jobs-.json from $FAKE_GH_DIR (--jq as jq -r; a missing fixture fails). +# `gh run download -D ` copies artifact-/ from $FAKE_GH_DIR into . +GH_STUB = r"""#!/bin/sh +if [ "$1" = run ] && [ "$2" = download ]; then + id=$3; shift 3 + while [ $# -gt 0 ]; do case "$1" in -D) dir=$2; shift 2 ;; *) shift ;; esac; done + [ -d "$FAKE_GH_DIR/artifact-$id" ] || { echo "no artifact for run $id" >&2; exit 1; } + mkdir -p "$dir" && cp -R "$FAKE_GH_DIR/artifact-$id/." "$dir/" + exit $? +fi +[ "$1" = api ] || { echo "unexpected: $*" >&2; exit 2; } +path=$2 +# The runs request's arguments, one per line, for the test to read back. +case "$path" in repos/*/actions/workflows/agent-quality.yml/runs*) printf '%s\n' "$@" > "$FAKE_GH_DIR/runs-request" ;; esac +shift 2 +filter="" +while [ $# -gt 0 ]; do + case "$1" in --jq) filter=$2; shift 2 ;; *) shift ;; esac +done +case "$path" in + repos/*/actions/workflows/agent-quality.yml/runs*) file="$FAKE_GH_DIR/runs.json" ;; + repos/*/actions/runs/*/jobs) + id=${path#*/actions/runs/}; id=${id%/jobs}; file="$FAKE_GH_DIR/jobs-$id.json" ;; + *) echo "no fixture for $path" >&2; exit 1 ;; +esac +[ -f "$file" ] || { echo "no fixture file $file" >&2; exit 1; } +if [ -n "$filter" ]; then jq -r "$filter" "$file"; else cat "$file"; fi +""" + + +def _step() -> dict: + wf = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = wf["jobs"][SMOKE_JOB]["steps"] + return next(s for s in steps if s.get("id") == "changes") + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@example.com", *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +def _write_bulk(repo: Path, name: str) -> None: + # Enough changed paths to overflow a pipe buffer: the step must still read a + # code change as code (a `printf | grep -q` pipeline fails under pipefail here). + (repo / "src/generated").mkdir() + for i in range(BULK_FILES): + (repo / f"src/generated/module_{i:05d}_padding_to_make_the_path_long.py").write_text( + "x\n", encoding="utf-8" + ) + + +def _write_docs_non_ascii(repo: Path, name: str) -> None: + # git quotes a non-ASCII path in --name-only output unless core.quotepath is off, + # and a quoted name does not match the DOCS pattern. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/é.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_docs_txt(repo: Path, name: str) -> None: + # Only Markdown is docs: a .txt under dokimi/docs/ is code. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/x.txt").write_text(f"{name}\n", encoding="utf-8") + + +def _write_markdown_outside_docs(repo: Path, name: str) -> None: + # Markdown under data/ and tests/ is not the docs set: the smoke test reads it. + for d in ("dokimi/data", "dokimi/tests"): + (repo / d).mkdir(parents=True, exist_ok=True) + (repo / d / "x.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_nested_docs(repo: Path, name: str) -> None: + # dokimi/docs/ is docs at any depth; changelog.d/ is docs one level deep. + (repo / "dokimi/docs/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/sub/a.md").write_text(f"{name}\n", encoding="utf-8") + (repo / "dokimi/changelog.d").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/1.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_nested_changelog(repo: Path, name: str) -> None: + (repo / "dokimi/changelog.d/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/sub/1.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_rename_code_to_md(repo: Path, name: str) -> None: + # A pure rename of code to a docs-set path: only the new name is listed unless + # renames are off. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + _git(repo, "mv", "src/app.py", "dokimi/docs/app.md") + + +# The kinds that write something other than the plain edit in _write_change. +_WRITERS: dict[str, Callable[[Path, str], None]] = { + "bulk": _write_bulk, + "docs_non_ascii": _write_docs_non_ascii, + "docs_txt": _write_docs_txt, + "markdown_outside_docs": _write_markdown_outside_docs, + "nested_docs": _write_nested_docs, + "nested_changelog": _write_nested_changelog, + "rename_code_to_md": _write_rename_code_to_md, +} + + +def _write_change(repo: Path, name: str, kind: str) -> None: + writer = _WRITERS.get(kind) + if writer is not None: + writer(repo, name) + return + # "code" edits src/app.py; "docs" and "offbranch" edit README.md. + path = "src/app.py" if kind == "code" else "README.md" + (repo / path).write_text(f"{name}\n", encoding="utf-8") + + +def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: + """A linear history on main: `base` (code and docs), then each (name, "code" | "docs").""" + _git(repo, "init", "-q", "-b", "main") + (repo / "src").mkdir() + (repo / "src/app.py").write_text("base\n", encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + shas: dict[str, str] = {"base": _commit(repo, "base")} + for name, kind in names: + if kind == "offbranch": + # A commit on a side branch from the last commit of main (only docs on top of it), so + # that it is not an ancestor of the push but the diff from it reads as docs. + _git(repo, "checkout", "-q", "-b", f"side-{name}", list(shas.values())[-1]) + _write_change(repo, name, kind) + shas[name] = _commit(repo, name) + if kind == "offbranch": + _git(repo, "checkout", "-q", "main") + return shas + + +def _job(verify: str) -> dict: + return { + "jobs": [ + { + "name": SMOKE_NAME, + "steps": [{"name": "Verify pass rate", "conclusion": verify}], + }, + {"name": "benchmark", "steps": []}, + ] + } + + +def _run_changes( + tmp_path: Path, + history, + runs, + before: str, + after: str, + head_ref: str = "feature", + action: str = "synchronize", + tested_base: str | None = "base", +) -> str: + """Run the changes step's script for a push (or an opened PR); return the code= output. + + tested_base names the commit that a passed run recorded as the base it tested (None: it recorded none). + """ + repo = tmp_path / "repo" + repo.mkdir() + shas = _history(repo, history) + # The base branch's tip, as the checkout fetches it: the diff base of an opened PR. + _git(repo, "update-ref", "refs/remotes/origin/main", shas["base"]) + fake = tmp_path / "fake" + fake.mkdir() + gh = fake / "gh" + gh.write_text(GH_STUB, encoding="utf-8") + gh.chmod(0o755) + if runs is not None: + listing = [] + for run_id, (name, verify) in enumerate(runs, start=1): + listing.append({"id": run_id, "head_sha": shas[name]}) + (fake / f"jobs-{run_id}.json").write_text(json.dumps(_job(verify)), encoding="utf-8") + if verify == "success" and tested_base is not None: + artifact = fake / f"artifact-{run_id}" + artifact.mkdir() + (artifact / "base.sha").write_text(shas[tested_base] + "\n", encoding="utf-8") + (fake / "runs.json").write_text(json.dumps({"workflow_runs": listing}), encoding="utf-8") + out = tmp_path / "github_output" + step = _step() + env = { + **os.environ, + **{k: str(v) for k, v in step["env"].items()}, + "EVENT": "pull_request", + "ACTION": action, + "BASE_REF": "main", + "HEAD_REF": head_ref, + "REPO": "intel/dokimi", + "GH_TOKEN": "test-token", + "BEFORE": shas[before], + "AFTER": shas[after], + "GITHUB_OUTPUT": str(out), + "FAKE_GH_DIR": str(fake), + "PATH": f"{fake}{os.pathsep}{os.environ['PATH']}", + } + proc = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step["run"]], + cwd=repo, + env=env, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + lines = [line for line in out.read_text(encoding="utf-8").splitlines() if line] + assert len(lines) == 1, lines + return lines[0] + + +# (history after base, runs newest first as (commit, Verify pass rate conclusion) or None +# for an unreadable API, before, after, the code output expected) +SCENARIOS = { + # Push B (docs) cancels push A's run (code), so A's smoke run never passed: A's code + # must still run. The only passed run is V's, so the diff starts there. + "docs_push_after_cancelled_code_run": ( + [("V", "code"), ("A", "code"), ("B", "docs")], + [("A", "cancelled"), ("V", "success")], + "A", + "B", + "true", + ), + "docs_push_after_failed_code_run": ( + [("V", "code"), ("A", "code"), ("B", "docs")], + [("A", "failure"), ("V", "success")], + "A", + "B", + "true", + ), + "no_passed_run_on_the_pr": ( + [("V", "code"), ("B", "docs")], + [("V", "cancelled")], + "V", + "B", + "true", + ), + "api_unreadable": ( + [("V", "code"), ("B", "docs")], + None, + "V", + "B", + "true", + ), + # Everything since the last passed run (V) is documentation: skip. + "docs_push_after_passed_code_run": ( + [("V", "code"), ("B", "docs")], + [("V", "success")], + "V", + "B", + "false", + ), + # A docs-only run in between skipped its code steps (Verify pass rate skipped), so it + # does not count, and the docs it carries are still docs: skip. + "docs_push_after_skipped_docs_run": ( + [("V", "code"), ("D", "docs"), ("B", "docs")], + [("D", "skipped"), ("V", "success")], + "D", + "B", + "false", + ), + "code_push_after_passed_code_run": ( + [("V", "code"), ("A", "code")], + [("V", "success")], + "V", + "A", + "true", + ), + # Many changed paths, the first one code: still code, not a docs-only push. + "code_push_with_many_paths_after_passed_run": ( + [("V", "code"), ("A", "bulk")], + [("V", "success")], + "V", + "A", + "true", + ), + # Only the Markdown in the docs set is docs: a .txt under docs/, or Markdown under data/ and tests/, is code. + "non_markdown_under_docs_after_passed_run_is_code": ( + [("V", "code"), ("A", "docs_txt")], + [("V", "success")], + "V", + "A", + "true", + ), + "markdown_under_data_and_tests_after_passed_run_is_code": ( + [("V", "code"), ("A", "markdown_outside_docs")], + [("V", "success")], + "V", + "A", + "true", + ), + # docs/ is docs at any depth; changelog.d/ is docs only one level deep. + "nested_docs_and_flat_changelog_after_passed_run_are_docs": ( + [("V", "code"), ("B", "nested_docs")], + [("V", "success")], + "V", + "B", + "false", + ), + "nested_changelog_entry_after_passed_run_is_code": ( + [("V", "code"), ("A", "nested_changelog")], + [("V", "success")], + "V", + "A", + "true", + ), + # A rename from code to Markdown counts as the code it removes (--no-renames). + "rename_from_code_to_markdown_after_passed_run_is_code": ( + [("V", "code"), ("A", "rename_code_to_md")], + [("V", "success")], + "V", + "A", + "true", + ), + # A passed run on a commit outside the push's history is not a diff base. + "passed_run_off_the_history_is_not_a_base": ( + [("V", "code"), ("X", "offbranch"), ("B", "docs")], + [("X", "success")], + "V", + "B", + "true", + ), + "docs_push_with_non_ascii_doc_name_after_passed_run": ( + [("V", "code"), ("B", "docs_non_ascii")], + [("V", "success")], + "V", + "B", + "false", + ), +} + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +@pytest.mark.parametrize("name", sorted(SCENARIOS)) +def test_docs_only_push_skips_only_when_all_since_the_last_passed_run_is_docs(tmp_path, name): + history, runs, before, after, expect = SCENARIOS[name] + assert _run_changes(tmp_path, history, runs, before, after) == f"code={expect}" + + +def _runs_request(fake: Path) -> tuple[str, dict[str, list[str]]]: + """The method and the query of the runs request: the URL's own query, plus the -f fields.""" + args = (fake / "runs-request").read_text(encoding="utf-8").splitlines() + # gh api sends a request with fields as POST unless --method says otherwise. + field_flags = ("-f", "-F", "--raw-field", "--field") + default_method = "POST" if any(flag in field_flags for flag in args) else "GET" + method = args[args.index("--method") + 1] if "--method" in args else default_method + query = parse_qs(urlsplit(args[1]).query, keep_blank_values=True) + for i, flag in enumerate(args[:-1]): + if flag in field_flags: + key, _, value = args[i + 1].partition("=") + query.setdefault(key, []).append(value) + return method, query + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_a_branch_name_with_query_characters_reaches_the_runs_query_whole(tmp_path): + # & # and + are legal in a branch name. In the URL they would end the branch value; as + # GET fields gh encodes them. gh sends fields as POST unless --method GET is given. + branch = "a&per_page=1#b+c" + code = _run_changes( + tmp_path, [("V", "code"), ("B", "docs")], [("V", "success")], "V", "B", head_ref=branch + ) + assert code == "code=false" + method, query = _runs_request(tmp_path / "fake") + assert method == "GET" + assert query["branch"] == [branch] + assert query["event"] == ["pull_request"] + + +def test_an_opened_pr_with_only_a_non_ascii_doc_name_is_docs_only(tmp_path): + # The PR diff (origin/main...HEAD) takes the same quoting path as the push diff. + history = [("B", "docs_non_ascii")] + assert _run_changes(tmp_path, history, None, "base", "B", action="opened") == "code=false" + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_docs_push_runs_the_code_again_when_the_base_moved_since_the_passed_run(tmp_path): + # The passed run tested its merge with another base ("V" stands for it): main has moved on + # since, so the docs-only push's merge holds base changes that no run tested. + history = [("V", "code"), ("B", "docs")] + code = _run_changes(tmp_path, history, [("V", "success")], "V", "B", tested_base="V") + assert code == "code=true" + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_docs_push_runs_the_code_again_when_the_passed_run_recorded_no_base(tmp_path): + history = [("V", "code"), ("B", "docs")] + code = _run_changes(tmp_path, history, [("V", "success")], "V", "B", tested_base=None) + assert code == "code=true" + + +def _record_step() -> dict: + wf = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = wf["jobs"][SMOKE_JOB]["steps"] + return next(s for s in steps if s.get("name") == "Record the base this run tested") + + +def _record_base(tmp_path: Path, shape: str, main_moved: bool, stale: str | None = None) -> str: + """Run the record step on a checkout as the smoke job makes it; return the recorded sha ("" for none). + + The checkout is one of: "merge", the PR's merge commit of its head on base0; "head", the PR + head (one parent); "pr_merge", a PR head that is itself a merge, whose first parent is a PR + commit that is not on main. Once the checkout exists, main moves on to base1 when `main_moved`. + `stale` is a record file that the PR's own steps left behind. + """ + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + (repo / "app.txt").write_text("base0\n", encoding="utf-8") + base0 = _commit(repo, "base0") + _git(repo, "checkout", "-q", "-b", "feature", base0) + (repo / "feature.txt").write_text("feature\n", encoding="utf-8") + head = _commit(repo, "feature") + if shape == "merge": + _git(repo, "checkout", "-q", "-b", "merge", base0) + _git(repo, "merge", "-q", "--no-ff", "-m", "merge", "feature") + checkout = _git(repo, "rev-parse", "HEAD") + elif shape == "pr_merge": + _git(repo, "checkout", "-q", "-b", "other", base0) + (repo / "other.txt").write_text("other\n", encoding="utf-8") + other = _commit(repo, "other") + _git(repo, "checkout", "-q", "-b", "pr", "feature") + _git(repo, "merge", "-q", "--no-ff", "-m", "merge", other) + checkout = _git(repo, "rev-parse", "HEAD") + else: + checkout = head + _git(repo, "checkout", "-q", "-b", "tip", base0) + (repo / "main.txt").write_text("base1\n", encoding="utf-8") + base1 = _commit(repo, "base1") if main_moved else base0 + _git(repo, "update-ref", "refs/remotes/origin/main", base1) + _git(repo, "checkout", "-q", "--detach", checkout) + out = tmp_path / "logs" / "base.sha" + out.parent.mkdir() + if stale is not None: + out.write_text(stale, encoding="utf-8") + step = _record_step() + env = { + **os.environ, + **{k: str(v) for k, v in step.get("env", {}).items()}, + "BASE_REF": "main", + "RECORD": str(out), + } + proc = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-c", step["run"]], + cwd=repo, + env=env, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + return out.read_text(encoding="utf-8").strip() if out.exists() else "" + + +def test_the_recorded_base_is_the_one_the_merge_was_built_on_after_main_moved(tmp_path): + # The merge commit was built on base0; main reached base1 before the checkout. The record must + # say base0, or a later docs-only push compares base1 with base1 and skips the merge's base changes. + recorded = _record_base(tmp_path, "merge", main_moved=True) + assert recorded == _git(tmp_path / "repo", "rev-parse", "HEAD^1") + + +def test_a_checkout_of_the_pr_head_records_no_base_even_with_a_stale_record(tmp_path): + # A PR that does not merge is checked out as its head: nothing is recorded, so a later docs-only + # push runs the code again. A record the PR's own steps left behind must not survive either. + assert _record_base(tmp_path, "head", main_moved=False, stale="planted") == "" + + +def test_a_merge_of_a_pr_commit_that_is_not_on_main_records_no_base(tmp_path): + # A PR head that is itself a merge has two parents, but its first parent is a PR commit, not a + # base tip: recording it would make the next docs-only push compare a PR commit with origin/main. + assert _record_base(tmp_path, "pr_merge", main_moved=False) == "" + + +def test_the_record_step_runs_only_for_a_code_run_of_a_pull_request(): + # Its artifact is read by a later push only for a passed run; the guard keeps it to runs that did code. + assert ( + _record_step()["if"] + == "steps.changes.outputs.code == 'true' && github.event_name == 'pull_request'" + ) + + +def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): + job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] + assert job["permissions"]["actions"] == "read" + + +def test_the_changes_step_runs_under_the_shell_flags_the_harness_uses(): + # shell: bash runs the step as `bash --noprofile --norc -eo pipefail {0}`, the flags _run_changes uses. + assert _step()["shell"] == "bash"