From 54b6efbd35d74ae954ed6ed02e740fcd2a53dd9e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 00:11:29 -0700 Subject: [PATCH 01/11] ci: skip agent-quality's code steps on a docs-only push The changes step diffed the whole PR (base...HEAD), so one code commit anywhere in a PR kept every later docs-only push running the Docker smoke test. A new push (synchronize) now decides from the push itself (before..after). Every other PR run still decides from the whole PR, and a push that cannot be diffed (force push, missing commits, empty diff) runs everything. Renames are diffed with --no-renames so a code file renamed to Markdown still counts as code. Documentation is Markdown in the root guides, dokimi/*.md, dokimi/docs/, dokimi/changelog.d/ and .github/. dokimi/data/ (read at run time), the tests and .agent-eval/ count as code. Thread: PRR_kwDOVBg7g88AAAABU5mMWA.b Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 48 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index f98f513a6..c6e70e2b0 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -64,24 +64,52 @@ jobs: with: persist-credentials: false fetch-depth: 0 - # Whether a PR changes anything besides Markdown. A docs-only PR skips the - # steps below (the Docker smoke test cannot be affected by it); the benchmark - # job measures the docs, so it always runs. The job itself still takes a - # runner, as its steps are skipped, not the job. + # Whether this run changes code (anything but documentation). Code-free runs skip + # the steps below (the Docker smoke test cannot be affected by a documentation + # edit); the benchmark job measures the docs, so it always runs. The job itself + # still takes a runner, as its steps are skipped, not the job, and it reports + # success. + # Scope: a new push to a PR (`synchronize`) decides from what that push changed + # (before..after), so a docs-only push after a code push skips. Every other PR + # run decides from the whole PR, base...HEAD. A push that cannot be diffed + # (no before commit, or before is not an ancestor of after: a force push) runs + # everything, as does an empty diff. + # Documentation: the Markdown the repo keeps as docs. Root guides (AGENTS.md, + # README.md, ...), dokimi/*.md, dokimi/docs/, dokimi/changelog.d/ and .github/. + # Not dokimi/data/ (skill and profile Markdown the code reads at run time), + # dokimi/tests/ (fixtures) or .agent-eval/ (eval tasks): those count as code. - id: changes working-directory: . env: EVENT: ${{ github.event_name }} + ACTION: ${{ github.event.action }} BASE_REF: ${{ github.event.pull_request.base.ref }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + DOCS: '^([^/]+|dokimi/[^/]+|dokimi/docs/(.+/)?[^/]+|dokimi/changelog\.d/[^/]+|\.github/(.+/)?[^/]+)\.md$' run: | - if [ "$EVENT" != pull_request ]; then echo code=true >> "$GITHUB_OUTPUT"; exit 0; fi - # The base branch's current tip (fetched by the checkout), not base.sha - # (the base when the PR was opened). - if git diff --name-only "origin/$BASE_REF"...HEAD | grep -qv '\.md$'; then - echo code=true >> "$GITHUB_OUTPUT" + set_code() { echo "code=$1" >> "$GITHUB_OUTPUT"; echo "changes: code=$1 ($2)"; } + if [ "$EVENT" != pull_request ]; then set_code true "not a pull request"; exit 0; fi + # --no-renames: a rename from code to Markdown must count as the code it removes. + if [ "$ACTION" = synchronize ]; then + if ! { [ -n "$BEFORE" ] && [ -n "$AFTER" ] \ + && git cat-file -e "$BEFORE^{commit}" && git cat-file -e "$AFTER^{commit}" \ + && git merge-base --is-ancestor "$BEFORE" "$AFTER"; } 2>/dev/null; then + set_code true "force push, or the commits are not fetched"; exit 0 + fi + scope=push + files=$(git diff --no-renames --name-only "$BEFORE" "$AFTER") \ + || { set_code true "push diff failed"; exit 0; } else - echo code=false >> "$GITHUB_OUTPUT" + # The base branch's current tip (fetched by the checkout), not base.sha + # (the base when the PR was opened). + scope=pr + files=$(git diff --no-renames --name-only "origin/$BASE_REF"...HEAD) \ + || { set_code true "PR diff failed"; exit 0; } fi + if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi + if printf '%s\n' "$files" | grep -qvE "$DOCS"; then set_code true "$scope changes code"; exit 0; fi + set_code false "$scope changes only documentation" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 if: steps.changes.outputs.code == 'true' with: From 6c1ed98da612e360e3775b4d2f3f62b3290f0cf8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 18:03:13 -0700 Subject: [PATCH 02/11] ci(agent-quality): skip docs-only pushes only after verified code The changes step decided a synchronize run from before..after alone. A docs-only push B cancels an in-flight code run A (same PR concurrency group, cancel-in-progress), so A's smoke test never finished and B skipped every code step: the head went green with A's code unverified, and a red A was replaced by a green B. Diff from the newest commit whose smoke run passed its code steps instead (Verify pass rate succeeded, read with the Actions API; the smoke job gets actions: read). A cancelled or failed run does not count, so its code runs again. With no passed run, or an unreadable run list, everything runs. tests/test_agent_quality_changes_scope.py runs the changes step as the workflow has it, with gh stubbed. Red before this change: the cancelled-run, failed-run, no-passed-run and unreadable-API cases return code=false, and the permission check fails. Green after: all six scope cases and the permission check pass. Review thread on PR 261: .github/workflows/agent-quality.yml:94. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 40 +++- .../tests/test_agent_quality_changes_scope.py | 209 ++++++++++++++++++ 2 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 dokimi/tests/test_agent_quality_changes_scope.py diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index c6e70e2b0..95ee73f8e 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -51,6 +51,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + # The changes step reads this workflow's earlier runs (gh api). + actions: read timeout-minutes: 15 env: # Pinned, not `main` -- an upstream rename/restructure of @@ -69,11 +71,14 @@ jobs: # edit); the benchmark job measures the docs, so it always runs. The job itself # still takes a runner, as its steps are skipped, not the job, and it reports # success. - # Scope: a new push to a PR (`synchronize`) decides from what that push changed - # (before..after), so a docs-only push after a code push skips. Every other PR - # run decides from the whole PR, base...HEAD. A push that cannot be diffed - # (no before commit, or before is not an ancestor of after: a force push) runs - # everything, as does an empty diff. + # Scope: a new push to a PR (`synchronize`) decides from everything since the last + # commit whose smoke run passed its code steps (the newest one of this PR's runs, + # reachable from the push, with Verify pass rate successful), so a docs-only push + # skips only when all the code before it was verified. A cancelled or failed run + # does not count, so the code it carried runs again. No such run, or an unreadable + # run list, runs everything. Every other PR run decides from the whole PR, + # base...HEAD. A push that cannot be diffed (no before commit, or before is not an + # ancestor of after: a force push) runs everything, as does an empty diff. # Documentation: the Markdown the repo keeps as docs. Root guides (AGENTS.md, # README.md, ...), dokimi/*.md, dokimi/docs/, dokimi/changelog.d/ and .github/. # Not dokimi/data/ (skill and profile Markdown the code reads at run time), @@ -84,11 +89,28 @@ jobs: EVENT: ${{ github.event_name }} ACTION: ${{ github.event.action }} BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REF: ${{ github.head_ref }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} BEFORE: ${{ github.event.before }} AFTER: ${{ github.event.after }} DOCS: '^([^/]+|dokimi/[^/]+|dokimi/docs/(.+/)?[^/]+|dokimi/changelog\.d/[^/]+|\.github/(.+/)?[^/]+)\.md$' run: | set_code() { echo "code=$1" >> "$GITHUB_OUTPUT"; echo "changes: code=$1 ($2)"; } + # The newest commit reachable from AFTER whose smoke run passed its code steps (its + # Verify pass rate step succeeded). Prints nothing when there is none or the run + # list cannot be read; the caller then runs everything. + last_verified() { + runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs?branch=$HEAD_REF&event=pull_request&status=completed&per_page=20" \ + --jq '.workflow_runs[] | "\(.id) \(.head_sha)"') || return 0 + while read -r id sha; do + [ -n "$id" ] || continue + git cat-file -e "$sha^{commit}" 2>/dev/null && git merge-base --is-ancestor "$sha" "$AFTER" 2>/dev/null || continue + verdict=$(gh api "repos/$REPO/actions/runs/$id/jobs" \ + --jq '.jobs[] | select(.name | startswith("External catalog smoke test")) | .steps[] | select(.name == "Verify pass rate") | .conclusion') || continue + if [ "$verdict" = success ]; then echo "$sha"; return 0; fi + done <<< "$runs" + } if [ "$EVENT" != pull_request ]; then set_code true "not a pull request"; exit 0; fi # --no-renames: a rename from code to Markdown must count as the code it removes. if [ "$ACTION" = synchronize ]; then @@ -97,8 +119,12 @@ jobs: && git merge-base --is-ancestor "$BEFORE" "$AFTER"; } 2>/dev/null; then set_code true "force push, or the commits are not fetched"; exit 0 fi - scope=push - files=$(git diff --no-renames --name-only "$BEFORE" "$AFTER") \ + # Diffed from the last passed run's commit, not from BEFORE: a run this push + # cancelled or that failed verified nothing, so its code is diffed again. + from=$(last_verified) + if [ -z "$from" ]; then set_code true "push: no passed smoke run to diff from"; exit 0; fi + scope="push since ${from:0:7}" + files=$(git diff --no-renames --name-only "$from" "$AFTER") \ || { set_code true "push diff failed"; exit 0; } else # The base branch's current tip (fetched by the checkout), not base.sha diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py new file mode 100644 index 000000000..723636a59 --- /dev/null +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -0,0 +1,209 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""agent-quality's `changes` step decides whether a push runs the smoke test. + +A docs-only push may skip the code steps only when everything since the last commit +whose smoke run passed its code steps is documentation. A cancelled or failed run does +not count as passed, so the code it carried is diffed again. The step is run here as +the workflow runs it (its script, read from the YAML), with `gh` replaced by a stub +that serves run and job fixtures. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/agent-quality.yml" +SMOKE_JOB = "external-catalog-smoke" +SMOKE_NAME = "External catalog smoke test (uxlfoundation/skills)" + +# The stub `gh api`: serves runs.json and jobs-.json from $FAKE_GH_DIR, applies --jq +# with jq -r as gh does, and fails for a missing fixture (the API being unreadable). +GH_STUB = r"""#!/bin/sh +[ "$1" = api ] || { echo "unexpected: $*" >&2; exit 2; } +path=$2; shift 2 +filter="" +while [ $# -gt 0 ]; do + case "$1" in --jq) filter=$2; shift 2 ;; *) shift ;; esac +done +case "$path" in + repos/*/actions/workflows/agent-quality.yml/runs\?*) file="$FAKE_GH_DIR/runs.json" ;; + repos/*/actions/runs/*/jobs) + id=${path#*/actions/runs/}; id=${id%/jobs}; file="$FAKE_GH_DIR/jobs-$id.json" ;; + *) echo "no fixture for $path" >&2; exit 1 ;; +esac +[ -f "$file" ] || { echo "no fixture file $file" >&2; exit 1; } +if [ -n "$filter" ]; then jq -r "$filter" "$file"; else cat "$file"; fi +""" + + +def _step() -> dict: + wf = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = wf["jobs"][SMOKE_JOB]["steps"] + return next(s for s in steps if s.get("id") == "changes") + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@example.com", *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: + """A linear history on main: `base` (code and docs), then each (name, "code" | "docs").""" + _git(repo, "init", "-q", "-b", "main") + shas: dict[str, str] = {} + (repo / "src").mkdir() + (repo / "src/app.py").write_text("base\n", encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "base") + shas["base"] = _git(repo, "rev-parse", "HEAD") + for name, kind in names: + path = "src/app.py" if kind == "code" else "README.md" + (repo / path).write_text(f"{name}\n", encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", name) + shas[name] = _git(repo, "rev-parse", "HEAD") + return shas + + +def _job(verify: str) -> dict: + return { + "jobs": [ + { + "name": SMOKE_NAME, + "steps": [{"name": "Verify pass rate", "conclusion": verify}], + }, + {"name": "benchmark", "steps": []}, + ] + } + + +def _run_changes(tmp_path: Path, history, runs, before: str, after: str) -> str: + """Run the changes step's script for a synchronize push; return the code= output.""" + repo = tmp_path / "repo" + repo.mkdir() + shas = _history(repo, history) + fake = tmp_path / "fake" + fake.mkdir() + gh = fake / "gh" + gh.write_text(GH_STUB, encoding="utf-8") + gh.chmod(0o755) + if runs is not None: + listing = [] + for run_id, (name, verify) in enumerate(runs, start=1): + listing.append({"id": run_id, "head_sha": shas[name]}) + (fake / f"jobs-{run_id}.json").write_text(json.dumps(_job(verify)), encoding="utf-8") + (fake / "runs.json").write_text(json.dumps({"workflow_runs": listing}), encoding="utf-8") + out = tmp_path / "github_output" + step = _step() + env = { + **os.environ, + **{k: str(v) for k, v in step["env"].items()}, + "EVENT": "pull_request", + "ACTION": "synchronize", + "BASE_REF": "main", + "HEAD_REF": "feature", + "REPO": "intel/dokimi", + "GH_TOKEN": "test-token", + "BEFORE": shas[before], + "AFTER": shas[after], + "GITHUB_OUTPUT": str(out), + "FAKE_GH_DIR": str(fake), + "PATH": f"{fake}{os.pathsep}{os.environ['PATH']}", + } + proc = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step["run"]], + cwd=repo, + env=env, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + lines = [line for line in out.read_text(encoding="utf-8").splitlines() if line] + assert len(lines) == 1, lines + return lines[0] + + +# (history after base, runs newest first as (commit, Verify pass rate conclusion) or None +# for an unreadable API, before, after, the code output expected) +SCENARIOS = { + # Push B (docs) cancels push A's run (code), so A's smoke run never passed: A's code + # must still run. The only passed run is V's, so the diff starts there. + "docs_push_after_cancelled_code_run": ( + [("V", "code"), ("A", "code"), ("B", "docs")], + [("A", "cancelled"), ("V", "success")], + "A", + "B", + "true", + ), + "docs_push_after_failed_code_run": ( + [("V", "code"), ("A", "code"), ("B", "docs")], + [("A", "failure"), ("V", "success")], + "A", + "B", + "true", + ), + "no_passed_run_on_the_pr": ( + [("V", "code"), ("B", "docs")], + [("V", "cancelled")], + "V", + "B", + "true", + ), + "api_unreadable": ( + [("V", "code"), ("B", "docs")], + None, + "V", + "B", + "true", + ), + # Everything since the last passed run (V) is documentation: skip. + "docs_push_after_passed_code_run": ( + [("V", "code"), ("B", "docs")], + [("V", "success")], + "V", + "B", + "false", + ), + # A docs-only run in between skipped its code steps (Verify pass rate skipped), so it + # does not count, and the docs it carries are still docs: skip. + "docs_push_after_skipped_docs_run": ( + [("V", "code"), ("D", "docs"), ("B", "docs")], + [("D", "skipped"), ("V", "success")], + "D", + "B", + "false", + ), + "code_push_after_passed_code_run": ( + [("V", "code"), ("A", "code")], + [("V", "success")], + "V", + "A", + "true", + ), +} + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +@pytest.mark.parametrize("name", sorted(SCENARIOS)) +def test_docs_only_push_skips_only_when_all_since_the_last_passed_run_is_docs(tmp_path, name): + history, runs, before, after, expect = SCENARIOS[name] + assert _run_changes(tmp_path, history, runs, before, after) == f"code={expect}" + + +def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): + job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] + assert job["permissions"]["actions"] == "read" From dbe9658d7ae708684eb41f8f4f7beb45dd62b5ce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:27:38 -0700 Subject: [PATCH 03/11] fix(ci): read a long code diff as code in agent-quality's changes step Under bash's pipefail, `printf | grep -qvE` fails when grep exits on the first code path before printf has written the rest of a long list, so a code push with many changed paths was read as documentation and skipped the smoke test. Feed the list to grep as a here-string instead. A new scope test pins a push with 2000 changed paths (red before the fix). Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 4 +++- .../tests/test_agent_quality_changes_scope.py | 22 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index 95ee73f8e..e694a681d 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -134,7 +134,9 @@ jobs: || { set_code true "PR diff failed"; exit 0; } fi if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi - if printf '%s\n' "$files" | grep -qvE "$DOCS"; then set_code true "$scope changes code"; exit 0; fi + # A here-string, not a pipe: under pipefail, grep -q exiting early on a long list + # would fail the printf upstream and read a code change as documentation. + if grep -qvE "$DOCS" <<< "$files"; then set_code true "$scope changes code"; exit 0; fi set_code false "$scope changes only documentation" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 if: steps.changes.outputs.code == 'true' diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index 723636a59..d67e4124f 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -24,6 +24,7 @@ WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/agent-quality.yml" SMOKE_JOB = "external-catalog-smoke" SMOKE_NAME = "External catalog smoke test (uxlfoundation/skills)" +BULK_FILES = 2000 # The stub `gh api`: serves runs.json and jobs-.json from $FAKE_GH_DIR, applies --jq # with jq -r as gh does, and fails for a missing fixture (the API being unreadable). @@ -71,8 +72,17 @@ def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: _git(repo, "commit", "-q", "-m", "base") shas["base"] = _git(repo, "rev-parse", "HEAD") for name, kind in names: - path = "src/app.py" if kind == "code" else "README.md" - (repo / path).write_text(f"{name}\n", encoding="utf-8") + if kind == "bulk": + # Enough changed paths to overflow a pipe buffer: the step must still read a + # code change as code (a `printf | grep -q` pipeline fails under pipefail here). + (repo / "src/generated").mkdir() + for i in range(BULK_FILES): + (repo / f"src/generated/module_{i:05d}_padding_to_make_the_path_long.py").write_text( + "x\n", encoding="utf-8" + ) + else: + path = "src/app.py" if kind == "code" else "README.md" + (repo / path).write_text(f"{name}\n", encoding="utf-8") _git(repo, "add", "-A") _git(repo, "commit", "-q", "-m", name) shas[name] = _git(repo, "rev-parse", "HEAD") @@ -194,6 +204,14 @@ def _run_changes(tmp_path: Path, history, runs, before: str, after: str) -> str: "A", "true", ), + # Many changed paths, the first one code: still code, not a docs-only push. + "code_push_with_many_paths_after_passed_run": ( + [("V", "code"), ("A", "bulk")], + [("V", "success")], + "V", + "A", + "true", + ), } From 9c88ee413be1325233c7b1431815b412fa8390a7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:56:39 -0700 Subject: [PATCH 04/11] docs: keep the comment blocks this PR adds to two lines Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index e694a681d..f57a6a0a7 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -66,23 +66,8 @@ jobs: with: persist-credentials: false fetch-depth: 0 - # Whether this run changes code (anything but documentation). Code-free runs skip - # the steps below (the Docker smoke test cannot be affected by a documentation - # edit); the benchmark job measures the docs, so it always runs. The job itself - # still takes a runner, as its steps are skipped, not the job, and it reports - # success. - # Scope: a new push to a PR (`synchronize`) decides from everything since the last - # commit whose smoke run passed its code steps (the newest one of this PR's runs, - # reachable from the push, with Verify pass rate successful), so a docs-only push - # skips only when all the code before it was verified. A cancelled or failed run - # does not count, so the code it carried runs again. No such run, or an unreadable - # run list, runs everything. Every other PR run decides from the whole PR, - # base...HEAD. A push that cannot be diffed (no before commit, or before is not an - # ancestor of after: a force push) runs everything, as does an empty diff. - # Documentation: the Markdown the repo keeps as docs. Root guides (AGENTS.md, - # README.md, ...), dokimi/*.md, dokimi/docs/, dokimi/changelog.d/ and .github/. - # Not dokimi/data/ (skill and profile Markdown the code reads at run time), - # dokimi/tests/ (fixtures) or .agent-eval/ (eval tasks): those count as code. + # Docs-only runs skip the code steps; the benchmark still runs. A push diffs from the last verified smoke run, else all. + # Docs are the repo's Markdown (DOCS below); dokimi/data/, tests and .agent-eval count as code. Force pushes run all. - id: changes working-directory: . env: @@ -97,9 +82,8 @@ jobs: DOCS: '^([^/]+|dokimi/[^/]+|dokimi/docs/(.+/)?[^/]+|dokimi/changelog\.d/[^/]+|\.github/(.+/)?[^/]+)\.md$' run: | set_code() { echo "code=$1" >> "$GITHUB_OUTPUT"; echo "changes: code=$1 ($2)"; } - # The newest commit reachable from AFTER whose smoke run passed its code steps (its - # Verify pass rate step succeeded). Prints nothing when there is none or the run - # list cannot be read; the caller then runs everything. + # Newest commit reachable from AFTER whose smoke run passed (Verify pass rate succeeded); prints nothing if none. + # An unreadable run list also prints nothing; the caller then runs everything. last_verified() { runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs?branch=$HEAD_REF&event=pull_request&status=completed&per_page=20" \ --jq '.workflow_runs[] | "\(.id) \(.head_sha)"') || return 0 From 5d22ccb85d5504a12824355bdf824c51fcbb50fd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 22:42:30 -0700 Subject: [PATCH 05/11] fix(ci): send the branch to agent-quality's runs query as a GET field The changes step put HEAD_REF into the URL query unencoded, so a branch name with & # or + changed or cut off the query and listed the wrong runs. Pass the fields with -f and --method GET: gh turns -f fields into a POST unless the method is GET. The test fails on the old URL and passes now. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 3 +- .../tests/test_agent_quality_changes_scope.py | 39 +++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index f57a6a0a7..2825ba7cf 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -85,7 +85,8 @@ jobs: # Newest commit reachable from AFTER whose smoke run passed (Verify pass rate succeeded); prints nothing if none. # An unreadable run list also prints nothing; the caller then runs everything. last_verified() { - runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs?branch=$HEAD_REF&event=pull_request&status=completed&per_page=20" \ + runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs" --method GET \ + -f branch="$HEAD_REF" -f event=pull_request -f status=completed -f per_page=20 \ --jq '.workflow_runs[] | "\(.id) \(.head_sha)"') || return 0 while read -r id sha; do [ -n "$id" ] || continue diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index d67e4124f..2ec0045d9 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -17,6 +17,7 @@ import shutil import subprocess from pathlib import Path +from urllib.parse import parse_qs, urlsplit import pytest import yaml @@ -30,13 +31,16 @@ # with jq -r as gh does, and fails for a missing fixture (the API being unreadable). GH_STUB = r"""#!/bin/sh [ "$1" = api ] || { echo "unexpected: $*" >&2; exit 2; } -path=$2; shift 2 +path=$2 +# The runs request's arguments, one per line, for the test to read back. +case "$path" in repos/*/actions/workflows/agent-quality.yml/runs*) printf '%s\n' "$@" > "$FAKE_GH_DIR/runs-request" ;; esac +shift 2 filter="" while [ $# -gt 0 ]; do case "$1" in --jq) filter=$2; shift 2 ;; *) shift ;; esac done case "$path" in - repos/*/actions/workflows/agent-quality.yml/runs\?*) file="$FAKE_GH_DIR/runs.json" ;; + repos/*/actions/workflows/agent-quality.yml/runs*) file="$FAKE_GH_DIR/runs.json" ;; repos/*/actions/runs/*/jobs) id=${path#*/actions/runs/}; id=${id%/jobs}; file="$FAKE_GH_DIR/jobs-$id.json" ;; *) echo "no fixture for $path" >&2; exit 1 ;; @@ -101,7 +105,7 @@ def _job(verify: str) -> dict: } -def _run_changes(tmp_path: Path, history, runs, before: str, after: str) -> str: +def _run_changes(tmp_path: Path, history, runs, before: str, after: str, head_ref: str = "feature") -> str: """Run the changes step's script for a synchronize push; return the code= output.""" repo = tmp_path / "repo" repo.mkdir() @@ -125,7 +129,7 @@ def _run_changes(tmp_path: Path, history, runs, before: str, after: str) -> str: "EVENT": "pull_request", "ACTION": "synchronize", "BASE_REF": "main", - "HEAD_REF": "feature", + "HEAD_REF": head_ref, "REPO": "intel/dokimi", "GH_TOKEN": "test-token", "BEFORE": shas[before], @@ -222,6 +226,33 @@ def test_docs_only_push_skips_only_when_all_since_the_last_passed_run_is_docs(tm assert _run_changes(tmp_path, history, runs, before, after) == f"code={expect}" +def _runs_request(fake: Path) -> tuple[str, dict[str, list[str]]]: + """The method and the query of the runs request: the URL's own query, plus the -f fields.""" + args = (fake / "runs-request").read_text(encoding="utf-8").splitlines() + method = args[args.index("--method") + 1] if "--method" in args else "GET" + query = parse_qs(urlsplit(args[1]).query, keep_blank_values=True) + for flag, field in zip(args, args[1:]): + if flag in ("-f", "-F", "--raw-field", "--field"): + key, _, value = field.partition("=") + query.setdefault(key, []).append(value) + return method, query + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_a_branch_name_with_query_characters_reaches_the_runs_query_whole(tmp_path): + # & # and + are legal in a branch name. In the URL they would end the branch value; as + # GET fields gh encodes them. gh sends fields as POST unless --method GET is given. + branch = "a&per_page=1#b+c" + code = _run_changes( + tmp_path, [("V", "code"), ("B", "docs")], [("V", "success")], "V", "B", head_ref=branch + ) + assert code == "code=false" + method, query = _runs_request(tmp_path / "fake") + assert method == "GET" + assert query["branch"] == [branch] + assert query["event"] == ["pull_request"] + + def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] assert job["permissions"]["actions"] == "read" From 384ea5eebc919d06662ceb5c6bca8e82be8d320d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 22:43:31 -0700 Subject: [PATCH 06/11] fix(ci): read non-ASCII doc names as docs in agent-quality's changes step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git quotes a non-ASCII path in --name-only output unless core.quotepath is off. A quoted "docs/é.md" does not match the DOCS pattern, so a docs-only change was read as code. It failed safe, but ran the smoke test for nothing. Both diffs now run with core.quotepath=false. The tests fail on the old diffs and pass now. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 4 +- .../tests/test_agent_quality_changes_scope.py | 38 ++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index 2825ba7cf..beff1279d 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -109,13 +109,13 @@ jobs: from=$(last_verified) if [ -z "$from" ]; then set_code true "push: no passed smoke run to diff from"; exit 0; fi scope="push since ${from:0:7}" - files=$(git diff --no-renames --name-only "$from" "$AFTER") \ + files=$(git -c core.quotepath=false diff --no-renames --name-only "$from" "$AFTER") \ || { set_code true "push diff failed"; exit 0; } else # The base branch's current tip (fetched by the checkout), not base.sha # (the base when the PR was opened). scope=pr - files=$(git diff --no-renames --name-only "origin/$BASE_REF"...HEAD) \ + files=$(git -c core.quotepath=false diff --no-renames --name-only "origin/$BASE_REF"...HEAD) \ || { set_code true "PR diff failed"; exit 0; } fi if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index 2ec0045d9..0419ef15c 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -84,6 +84,11 @@ def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: (repo / f"src/generated/module_{i:05d}_padding_to_make_the_path_long.py").write_text( "x\n", encoding="utf-8" ) + elif kind == "docs_non_ascii": + # git quotes a non-ASCII path in --name-only output unless core.quotepath is off, + # and a quoted name does not match the DOCS pattern. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/é.md").write_text(f"{name}\n", encoding="utf-8") else: path = "src/app.py" if kind == "code" else "README.md" (repo / path).write_text(f"{name}\n", encoding="utf-8") @@ -105,11 +110,21 @@ def _job(verify: str) -> dict: } -def _run_changes(tmp_path: Path, history, runs, before: str, after: str, head_ref: str = "feature") -> str: - """Run the changes step's script for a synchronize push; return the code= output.""" +def _run_changes( + tmp_path: Path, + history, + runs, + before: str, + after: str, + head_ref: str = "feature", + action: str = "synchronize", +) -> str: + """Run the changes step's script for a push (or an opened PR); return the code= output.""" repo = tmp_path / "repo" repo.mkdir() shas = _history(repo, history) + # The base branch's tip, as the checkout fetches it: the diff base of an opened PR. + _git(repo, "update-ref", "refs/remotes/origin/main", shas["base"]) fake = tmp_path / "fake" fake.mkdir() gh = fake / "gh" @@ -127,7 +142,7 @@ def _run_changes(tmp_path: Path, history, runs, before: str, after: str, head_re **os.environ, **{k: str(v) for k, v in step["env"].items()}, "EVENT": "pull_request", - "ACTION": "synchronize", + "ACTION": action, "BASE_REF": "main", "HEAD_REF": head_ref, "REPO": "intel/dokimi", @@ -216,6 +231,13 @@ def _run_changes(tmp_path: Path, history, runs, before: str, after: str, head_re "A", "true", ), + "docs_push_with_non_ascii_doc_name_after_passed_run": ( + [("V", "code"), ("B", "docs_non_ascii")], + [("V", "success")], + "V", + "B", + "false", + ), } @@ -231,9 +253,9 @@ def _runs_request(fake: Path) -> tuple[str, dict[str, list[str]]]: args = (fake / "runs-request").read_text(encoding="utf-8").splitlines() method = args[args.index("--method") + 1] if "--method" in args else "GET" query = parse_qs(urlsplit(args[1]).query, keep_blank_values=True) - for flag, field in zip(args, args[1:]): + for i, flag in enumerate(args[:-1]): if flag in ("-f", "-F", "--raw-field", "--field"): - key, _, value = field.partition("=") + key, _, value = args[i + 1].partition("=") query.setdefault(key, []).append(value) return method, query @@ -253,6 +275,12 @@ def test_a_branch_name_with_query_characters_reaches_the_runs_query_whole(tmp_pa assert query["event"] == ["pull_request"] +def test_an_opened_pr_with_only_a_non_ascii_doc_name_is_docs_only(tmp_path): + # The PR diff (origin/main...HEAD) takes the same quoting path as the push diff. + history = [("B", "docs_non_ascii")] + assert _run_changes(tmp_path, history, None, "base", "B", action="opened") == "code=false" + + def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] assert job["permissions"]["actions"] == "read" From f5e352a60967d02e274829e883dafeb36ef63d97 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 22:43:43 -0700 Subject: [PATCH 07/11] docs(ci): state the real reason for the here-string in the changes step The step has no shell key, so it runs bash -e without pipefail. The comment gave a pipefail reason that does not apply to it. The here-string is still harmless, so only the comment changes. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index beff1279d..7e948837c 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -119,8 +119,8 @@ jobs: || { set_code true "PR diff failed"; exit 0; } fi if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi - # A here-string, not a pipe: under pipefail, grep -q exiting early on a long list - # would fail the printf upstream and read a code change as documentation. + # A here-string, not a pipe: grep -q may exit early, and the pipe's printf would get SIGPIPE. + # This step runs without pipefail, so that is harmless; the here-string just avoids it. if grep -qvE "$DOCS" <<< "$files"; then set_code true "$scope changes code"; exit 0; fi set_code false "$scope changes only documentation" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 From 7d654b300b76c43809d86d8aac3b05a4df7dc0a1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:32:05 -0700 Subject: [PATCH 08/11] test(ci): pin the docs set, renames and the ancestor check in the changes step The scope tests used README.md as the only docs path and a linear history with no renames, so loosening the DOCS pattern, dropping --no-renames or the ancestor check in last_verified left them green. Add scenarios for each: a non-Markdown file under docs/, Markdown under data/ and tests/, docs/ at depth, a changelog.d entry one level down, a code-to-docs rename, and a passed run on a commit outside the push's history. Each of the six mutants fails at least one test; the workflow is unchanged. Thread: PRRT_kwDOVBg7g86x68oj Signed-off-by: Nikolay Petrov --- .../tests/test_agent_quality_changes_scope.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index 0419ef15c..7243b3a7c 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -89,12 +89,42 @@ def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: # and a quoted name does not match the DOCS pattern. (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) (repo / "dokimi/docs/é.md").write_text(f"{name}\n", encoding="utf-8") + elif kind == "docs_txt": + # Only Markdown is docs: a .txt under dokimi/docs/ is code. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/x.txt").write_text(f"{name}\n", encoding="utf-8") + elif kind == "markdown_outside_docs": + # Markdown under data/ and tests/ is not the docs set: the smoke test reads it. + for d in ("dokimi/data", "dokimi/tests"): + (repo / d).mkdir(parents=True, exist_ok=True) + (repo / d / "x.md").write_text(f"{name}\n", encoding="utf-8") + elif kind == "nested_docs": + # dokimi/docs/ is docs at any depth; changelog.d/ is docs one level deep. + (repo / "dokimi/docs/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/sub/a.md").write_text(f"{name}\n", encoding="utf-8") + (repo / "dokimi/changelog.d").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/1.md").write_text(f"{name}\n", encoding="utf-8") + elif kind == "nested_changelog": + (repo / "dokimi/changelog.d/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/sub/1.md").write_text(f"{name}\n", encoding="utf-8") + elif kind == "rename_code_to_md": + # A pure rename of code to a docs-set path: only the new name is listed unless + # renames are off. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + _git(repo, "mv", "src/app.py", "dokimi/docs/app.md") + elif kind == "offbranch": + # A commit on a side branch from the last commit of main (only docs on top of it), so + # that it is not an ancestor of the push but the diff from it reads as docs. + _git(repo, "checkout", "-q", "-b", f"side-{name}", list(shas.values())[-1]) + (repo / "README.md").write_text(f"{name}\n", encoding="utf-8") else: path = "src/app.py" if kind == "code" else "README.md" (repo / path).write_text(f"{name}\n", encoding="utf-8") _git(repo, "add", "-A") _git(repo, "commit", "-q", "-m", name) shas[name] = _git(repo, "rev-parse", "HEAD") + if kind == "offbranch": + _git(repo, "checkout", "-q", "main") return shas @@ -231,6 +261,52 @@ def _run_changes( "A", "true", ), + # Only the Markdown in the docs set is docs: a .txt under docs/, or Markdown under data/ and tests/, is code. + "non_markdown_under_docs_after_passed_run_is_code": ( + [("V", "code"), ("A", "docs_txt")], + [("V", "success")], + "V", + "A", + "true", + ), + "markdown_under_data_and_tests_after_passed_run_is_code": ( + [("V", "code"), ("A", "markdown_outside_docs")], + [("V", "success")], + "V", + "A", + "true", + ), + # docs/ is docs at any depth; changelog.d/ is docs only one level deep. + "nested_docs_and_flat_changelog_after_passed_run_are_docs": ( + [("V", "code"), ("B", "nested_docs")], + [("V", "success")], + "V", + "B", + "false", + ), + "nested_changelog_entry_after_passed_run_is_code": ( + [("V", "code"), ("A", "nested_changelog")], + [("V", "success")], + "V", + "A", + "true", + ), + # A rename from code to Markdown counts as the code it removes (--no-renames). + "rename_from_code_to_markdown_after_passed_run_is_code": ( + [("V", "code"), ("A", "rename_code_to_md")], + [("V", "success")], + "V", + "A", + "true", + ), + # A passed run on a commit outside the push's history is not a diff base. + "passed_run_off_the_history_is_not_a_base": ( + [("V", "code"), ("X", "offbranch"), ("B", "docs")], + [("X", "success")], + "V", + "B", + "true", + ), "docs_push_with_non_ascii_doc_name_after_passed_run": ( [("V", "code"), ("B", "docs_non_ascii")], [("V", "success")], From 25d39535105464b9a15070d823ad9b26e7e3e44b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 11:52:25 -0700 Subject: [PATCH 09/11] test(agent-quality): split _history into one writer per change kind _history had one if/elif chain over the change kinds, complexity 13 over the 10 cap with no baseline entry. Each kind is now a writer function in a dispatch table, the plain edit is _write_change, and the commit step is _commit. Same files, same order, same commits; the 18 tests are unchanged. Signed-off-by: Nikolay Petrov --- .../tests/test_agent_quality_changes_scope.py | 131 +++++++++++------- 1 file changed, 83 insertions(+), 48 deletions(-) diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index 7243b3a7c..c11ab65be 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -16,6 +16,7 @@ import os import shutil import subprocess +from collections.abc import Callable from pathlib import Path from urllib.parse import parse_qs, urlsplit @@ -65,64 +66,98 @@ def _git(repo: Path, *args: str) -> str: ).stdout.strip() +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +def _write_bulk(repo: Path, name: str) -> None: + # Enough changed paths to overflow a pipe buffer: the step must still read a + # code change as code (a `printf | grep -q` pipeline fails under pipefail here). + (repo / "src/generated").mkdir() + for i in range(BULK_FILES): + (repo / f"src/generated/module_{i:05d}_padding_to_make_the_path_long.py").write_text( + "x\n", encoding="utf-8" + ) + + +def _write_docs_non_ascii(repo: Path, name: str) -> None: + # git quotes a non-ASCII path in --name-only output unless core.quotepath is off, + # and a quoted name does not match the DOCS pattern. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/é.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_docs_txt(repo: Path, name: str) -> None: + # Only Markdown is docs: a .txt under dokimi/docs/ is code. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/x.txt").write_text(f"{name}\n", encoding="utf-8") + + +def _write_markdown_outside_docs(repo: Path, name: str) -> None: + # Markdown under data/ and tests/ is not the docs set: the smoke test reads it. + for d in ("dokimi/data", "dokimi/tests"): + (repo / d).mkdir(parents=True, exist_ok=True) + (repo / d / "x.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_nested_docs(repo: Path, name: str) -> None: + # dokimi/docs/ is docs at any depth; changelog.d/ is docs one level deep. + (repo / "dokimi/docs/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/docs/sub/a.md").write_text(f"{name}\n", encoding="utf-8") + (repo / "dokimi/changelog.d").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/1.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_nested_changelog(repo: Path, name: str) -> None: + (repo / "dokimi/changelog.d/sub").mkdir(parents=True, exist_ok=True) + (repo / "dokimi/changelog.d/sub/1.md").write_text(f"{name}\n", encoding="utf-8") + + +def _write_rename_code_to_md(repo: Path, name: str) -> None: + # A pure rename of code to a docs-set path: only the new name is listed unless + # renames are off. + (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) + _git(repo, "mv", "src/app.py", "dokimi/docs/app.md") + + +# The kinds that write something other than the plain edit in _write_change. +_WRITERS: dict[str, Callable[[Path, str], None]] = { + "bulk": _write_bulk, + "docs_non_ascii": _write_docs_non_ascii, + "docs_txt": _write_docs_txt, + "markdown_outside_docs": _write_markdown_outside_docs, + "nested_docs": _write_nested_docs, + "nested_changelog": _write_nested_changelog, + "rename_code_to_md": _write_rename_code_to_md, +} + + +def _write_change(repo: Path, name: str, kind: str) -> None: + writer = _WRITERS.get(kind) + if writer is not None: + writer(repo, name) + return + # "code" edits src/app.py; "docs" and "offbranch" edit README.md. + path = "src/app.py" if kind == "code" else "README.md" + (repo / path).write_text(f"{name}\n", encoding="utf-8") + + def _history(repo: Path, names: list[tuple[str, str]]) -> dict[str, str]: """A linear history on main: `base` (code and docs), then each (name, "code" | "docs").""" _git(repo, "init", "-q", "-b", "main") - shas: dict[str, str] = {} (repo / "src").mkdir() (repo / "src/app.py").write_text("base\n", encoding="utf-8") (repo / "README.md").write_text("base\n", encoding="utf-8") - _git(repo, "add", "-A") - _git(repo, "commit", "-q", "-m", "base") - shas["base"] = _git(repo, "rev-parse", "HEAD") + shas: dict[str, str] = {"base": _commit(repo, "base")} for name, kind in names: - if kind == "bulk": - # Enough changed paths to overflow a pipe buffer: the step must still read a - # code change as code (a `printf | grep -q` pipeline fails under pipefail here). - (repo / "src/generated").mkdir() - for i in range(BULK_FILES): - (repo / f"src/generated/module_{i:05d}_padding_to_make_the_path_long.py").write_text( - "x\n", encoding="utf-8" - ) - elif kind == "docs_non_ascii": - # git quotes a non-ASCII path in --name-only output unless core.quotepath is off, - # and a quoted name does not match the DOCS pattern. - (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) - (repo / "dokimi/docs/é.md").write_text(f"{name}\n", encoding="utf-8") - elif kind == "docs_txt": - # Only Markdown is docs: a .txt under dokimi/docs/ is code. - (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) - (repo / "dokimi/docs/x.txt").write_text(f"{name}\n", encoding="utf-8") - elif kind == "markdown_outside_docs": - # Markdown under data/ and tests/ is not the docs set: the smoke test reads it. - for d in ("dokimi/data", "dokimi/tests"): - (repo / d).mkdir(parents=True, exist_ok=True) - (repo / d / "x.md").write_text(f"{name}\n", encoding="utf-8") - elif kind == "nested_docs": - # dokimi/docs/ is docs at any depth; changelog.d/ is docs one level deep. - (repo / "dokimi/docs/sub").mkdir(parents=True, exist_ok=True) - (repo / "dokimi/docs/sub/a.md").write_text(f"{name}\n", encoding="utf-8") - (repo / "dokimi/changelog.d").mkdir(parents=True, exist_ok=True) - (repo / "dokimi/changelog.d/1.md").write_text(f"{name}\n", encoding="utf-8") - elif kind == "nested_changelog": - (repo / "dokimi/changelog.d/sub").mkdir(parents=True, exist_ok=True) - (repo / "dokimi/changelog.d/sub/1.md").write_text(f"{name}\n", encoding="utf-8") - elif kind == "rename_code_to_md": - # A pure rename of code to a docs-set path: only the new name is listed unless - # renames are off. - (repo / "dokimi/docs").mkdir(parents=True, exist_ok=True) - _git(repo, "mv", "src/app.py", "dokimi/docs/app.md") - elif kind == "offbranch": + if kind == "offbranch": # A commit on a side branch from the last commit of main (only docs on top of it), so # that it is not an ancestor of the push but the diff from it reads as docs. _git(repo, "checkout", "-q", "-b", f"side-{name}", list(shas.values())[-1]) - (repo / "README.md").write_text(f"{name}\n", encoding="utf-8") - else: - path = "src/app.py" if kind == "code" else "README.md" - (repo / path).write_text(f"{name}\n", encoding="utf-8") - _git(repo, "add", "-A") - _git(repo, "commit", "-q", "-m", name) - shas[name] = _git(repo, "rev-parse", "HEAD") + _write_change(repo, name, kind) + shas[name] = _commit(repo, name) if kind == "offbranch": _git(repo, "checkout", "-q", "main") return shas From 548027b611478f7f28c126adc7f89d2eae560b55 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 13:56:18 -0700 Subject: [PATCH 10/11] fix(agent-quality): run the changes step under bash; skip only while the base is unchanged The changes step declares shell: bash, so GitHub runs it with the same -eo pipefail the tests use. A docs-only push now skips the smoke run only if the base-branch tip is still the one the passed run recorded: the base can move after that run, and the push's merge would then hold changes no run tested. The smoke job records the base it fetched; a run with no record runs the code again. The gh stub infers POST when a request carries fields, so the test catches a removed --method GET. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 28 ++++++++--- .../tests/test_agent_quality_changes_scope.py | 48 +++++++++++++++++-- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index 2e70c81af..ddad9f888 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -70,6 +70,8 @@ jobs: # Docs are the repo's Markdown (DOCS below); dokimi/data/, tests and .agent-eval count as code. Force pushes run all. - id: changes working-directory: . + # Explicit, so the step runs as `bash --noprofile --norc -eo pipefail`, the flags the test harness uses. + shell: bash env: EVENT: ${{ github.event_name }} ACTION: ${{ github.event.action }} @@ -82,8 +84,8 @@ jobs: DOCS: '^([^/]+|dokimi/[^/]+|dokimi/docs/(.+/)?[^/]+|dokimi/changelog\.d/[^/]+|\.github/(.+/)?[^/]+)\.md$' run: | set_code() { echo "code=$1" >> "$GITHUB_OUTPUT"; echo "changes: code=$1 ($2)"; } - # Newest commit reachable from AFTER whose smoke run passed (Verify pass rate succeeded); prints nothing if none. - # An unreadable run list also prints nothing; the caller then runs everything. + # Newest commit reachable from AFTER whose smoke run passed (Verify pass rate succeeded): prints " ", + # or nothing if none. An unreadable run list also prints nothing; the caller then runs everything. last_verified() { runs=$(gh api "repos/$REPO/actions/workflows/agent-quality.yml/runs" --method GET \ -f branch="$HEAD_REF" -f event=pull_request -f status=completed -f per_page=20 \ @@ -93,7 +95,7 @@ jobs: git cat-file -e "$sha^{commit}" 2>/dev/null && git merge-base --is-ancestor "$sha" "$AFTER" 2>/dev/null || continue verdict=$(gh api "repos/$REPO/actions/runs/$id/jobs" \ --jq '.jobs[] | select(.name | startswith("External catalog smoke test")) | .steps[] | select(.name == "Verify pass rate") | .conclusion') || continue - if [ "$verdict" = success ]; then echo "$sha"; return 0; fi + if [ "$verdict" = success ]; then echo "$sha $id"; return 0; fi done <<< "$runs" } if [ "$EVENT" != pull_request ]; then set_code true "not a pull request"; exit 0; fi @@ -106,8 +108,16 @@ jobs: fi # Diffed from the last passed run's commit, not from BEFORE: a run this push # cancelled or that failed verified nothing, so its code is diffed again. - from=$(last_verified) + read -r from verified_run <<< "$(last_verified)" || true if [ -z "$from" ]; then set_code true "push: no passed smoke run to diff from"; exit 0; fi + # The diff above does not show base changes: the run tested its merge with the base as it was then. + # Skip only if the base is still that one (the smoke job records it; no record means run everything). + dl=$(mktemp -d) + tested_base=$(gh run download "$verified_run" -R "$REPO" -n external-catalog-smoke-runs -D "$dl" >/dev/null 2>&1 && cat "$dl/base.sha") || tested_base= + base=$(git rev-parse --verify -q "origin/$BASE_REF") || base= + if [ -z "$tested_base" ] || [ "$tested_base" != "$base" ]; then + set_code true "push: the base moved since ${from:0:7}, or that run did not record it"; exit 0 + fi scope="push since ${from:0:7}" files=$(git -c core.quotepath=false diff --no-renames --name-only "$from" "$AFTER") \ || { set_code true "push diff failed"; exit 0; } @@ -119,8 +129,7 @@ jobs: || { set_code true "PR diff failed"; exit 0; } fi if [ -z "$files" ]; then set_code true "$scope diff is empty"; exit 0; fi - # A here-string, not a pipe: grep -q may exit early, and the pipe's printf would get SIGPIPE. - # This step runs without pipefail, so that is harmless; the here-string just avoids it. + # A here-string, not a pipe: grep -q may exit early, and under pipefail the pipe's printf would get SIGPIPE. if grep -qvE "$DOCS" <<< "$files"; then set_code true "$scope changes code"; exit 0; fi set_code false "$scope changes only documentation" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -162,6 +171,13 @@ jobs: --tasks-root "${{ runner.temp }}/uxl-skills/evaluation/harbor/tasks" \ --harnesses docker-oracle,docker-probe \ --out-json "${{ github.workspace }}/logs/external-catalog-smoke/runs.json" + # The base-branch tip this run's checkout fetched, which its merge commit was built on. The + # changes step of a later docs-only push skips only while the base is still this one. + - name: Record the base this run tested + if: steps.changes.outputs.code == 'true' && github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: git rev-parse --verify "origin/$BASE_REF" > "${{ github.workspace }}/logs/external-catalog-smoke/base.sha" # Every cell passed and none errored (an errored cell leaves pass_rate's # denominator, so pass_rate alone would read 1.0), and the probe's fetch # was denied by the egress proxy; see scripts/check_smoke_runs.py. diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index c11ab65be..abc331a2b 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -28,9 +28,17 @@ SMOKE_NAME = "External catalog smoke test (uxlfoundation/skills)" BULK_FILES = 2000 -# The stub `gh api`: serves runs.json and jobs-.json from $FAKE_GH_DIR, applies --jq +# The stub `gh`: `gh api` serves runs.json and jobs-.json from $FAKE_GH_DIR, applies --jq # with jq -r as gh does, and fails for a missing fixture (the API being unreadable). +# `gh run download -D ` copies artifact-/ from $FAKE_GH_DIR into . GH_STUB = r"""#!/bin/sh +if [ "$1" = run ] && [ "$2" = download ]; then + id=$3; shift 3 + while [ $# -gt 0 ]; do case "$1" in -D) dir=$2; shift 2 ;; *) shift ;; esac; done + [ -d "$FAKE_GH_DIR/artifact-$id" ] || { echo "no artifact for run $id" >&2; exit 1; } + mkdir -p "$dir" && cp -R "$FAKE_GH_DIR/artifact-$id/." "$dir/" + exit $? +fi [ "$1" = api ] || { echo "unexpected: $*" >&2; exit 2; } path=$2 # The runs request's arguments, one per line, for the test to read back. @@ -183,8 +191,12 @@ def _run_changes( after: str, head_ref: str = "feature", action: str = "synchronize", + tested_base: str | None = "base", ) -> str: - """Run the changes step's script for a push (or an opened PR); return the code= output.""" + """Run the changes step's script for a push (or an opened PR); return the code= output. + + tested_base names the commit that a passed run recorded as the base it tested (None: it recorded none). + """ repo = tmp_path / "repo" repo.mkdir() shas = _history(repo, history) @@ -200,6 +212,10 @@ def _run_changes( for run_id, (name, verify) in enumerate(runs, start=1): listing.append({"id": run_id, "head_sha": shas[name]}) (fake / f"jobs-{run_id}.json").write_text(json.dumps(_job(verify)), encoding="utf-8") + if verify == "success" and tested_base is not None: + artifact = fake / f"artifact-{run_id}" + artifact.mkdir() + (artifact / "base.sha").write_text(shas[tested_base] + "\n", encoding="utf-8") (fake / "runs.json").write_text(json.dumps({"workflow_runs": listing}), encoding="utf-8") out = tmp_path / "github_output" step = _step() @@ -362,10 +378,13 @@ def test_docs_only_push_skips_only_when_all_since_the_last_passed_run_is_docs(tm def _runs_request(fake: Path) -> tuple[str, dict[str, list[str]]]: """The method and the query of the runs request: the URL's own query, plus the -f fields.""" args = (fake / "runs-request").read_text(encoding="utf-8").splitlines() - method = args[args.index("--method") + 1] if "--method" in args else "GET" + # gh api sends a request with fields as POST unless --method says otherwise. + field_flags = ("-f", "-F", "--raw-field", "--field") + default_method = "POST" if any(flag in field_flags for flag in args) else "GET" + method = args[args.index("--method") + 1] if "--method" in args else default_method query = parse_qs(urlsplit(args[1]).query, keep_blank_values=True) for i, flag in enumerate(args[:-1]): - if flag in ("-f", "-F", "--raw-field", "--field"): + if flag in field_flags: key, _, value = args[i + 1].partition("=") query.setdefault(key, []).append(value) return method, query @@ -392,6 +411,27 @@ def test_an_opened_pr_with_only_a_non_ascii_doc_name_is_docs_only(tmp_path): assert _run_changes(tmp_path, history, None, "base", "B", action="opened") == "code=false" +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_docs_push_runs_the_code_again_when_the_base_moved_since_the_passed_run(tmp_path): + # The passed run tested its merge with another base ("V" stands for it): main has moved on + # since, so the docs-only push's merge holds base changes that no run tested. + history = [("V", "code"), ("B", "docs")] + code = _run_changes(tmp_path, history, [("V", "success")], "V", "B", tested_base="V") + assert code == "code=true" + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq for the gh stub's --jq") +def test_docs_push_runs_the_code_again_when_the_passed_run_recorded_no_base(tmp_path): + history = [("V", "code"), ("B", "docs")] + code = _run_changes(tmp_path, history, [("V", "success")], "V", "B", tested_base=None) + assert code == "code=true" + + def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] assert job["permissions"]["actions"] == "read" + + +def test_the_changes_step_runs_under_the_shell_flags_the_harness_uses(): + # shell: bash runs the step as `bash --noprofile --norc -eo pipefail {0}`, the flags _run_changes uses. + assert _step()["shell"] == "bash" From af61af96c525af3839b5b5567bdb39954e7d3e45 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:26:25 -0700 Subject: [PATCH 11/11] fix(agent-quality): record the base the merge was built on The smoke checkout is the PR's merge commit, built on the base tip of its time. Recording origin/ after the checkout can hold a newer tip than the merge used, so a later docs-only push would skip while base changes that no run tested were in the merge. The record is now the merge's first parent, and only when that parent is on origin/. A checkout that is not such a merge (a PR that does not merge, or a PR head that is itself a merge) records nothing, and a stale record is removed first. The record step's and the changes step's comments describe that behaviour. The tests run the record step on a merge built on base0 with main moved to base1 (it recorded base1 before), on a PR head with a stale record, and on a PR head that merges a PR commit. Signed-off-by: Nikolay Petrov --- .github/workflows/agent-quality.yml | 25 +++-- .../tests/test_agent_quality_changes_scope.py | 102 ++++++++++++++++-- 2 files changed, 113 insertions(+), 14 deletions(-) diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index ddad9f888..6f42eb7c4 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -66,8 +66,8 @@ jobs: with: persist-credentials: false fetch-depth: 0 - # Docs-only runs skip the code steps; the benchmark still runs. A push diffs from the last verified smoke run, else all. - # Docs are the repo's Markdown (DOCS below); dokimi/data/, tests and .agent-eval count as code. Force pushes run all. + # Docs-only runs skip the code steps (the benchmark still runs). A push diffs from the last verified smoke run, else all; docs are the Markdown in DOCS below. + # dokimi/data/, tests and .agent-eval count as code, and a base that moved since that run's merge also runs everything. Force pushes run all. - id: changes working-directory: . # Explicit, so the step runs as `bash --noprofile --norc -eo pipefail`, the flags the test harness uses. @@ -110,8 +110,8 @@ jobs: # cancelled or that failed verified nothing, so its code is diffed again. read -r from verified_run <<< "$(last_verified)" || true if [ -z "$from" ]; then set_code true "push: no passed smoke run to diff from"; exit 0; fi - # The diff above does not show base changes: the run tested its merge with the base as it was then. - # Skip only if the base is still that one (the smoke job records it; no record means run everything). + # The diff above does not show base changes: the run tested its merge with the base it was built on. + # Skip only if origin/$BASE_REF is still that base (the smoke job records it; no record means run everything). dl=$(mktemp -d) tested_base=$(gh run download "$verified_run" -R "$REPO" -n external-catalog-smoke-runs -D "$dl" >/dev/null 2>&1 && cat "$dl/base.sha") || tested_base= base=$(git rev-parse --verify -q "origin/$BASE_REF") || base= @@ -122,7 +122,7 @@ jobs: files=$(git -c core.quotepath=false diff --no-renames --name-only "$from" "$AFTER") \ || { set_code true "push diff failed"; exit 0; } else - # The base branch's current tip (fetched by the checkout), not base.sha + # The base branch's current tip (fetched by the checkout), not the event's pull_request.base.sha # (the base when the PR was opened). scope=pr files=$(git -c core.quotepath=false diff --no-renames --name-only "origin/$BASE_REF"...HEAD) \ @@ -171,13 +171,22 @@ jobs: --tasks-root "${{ runner.temp }}/uxl-skills/evaluation/harbor/tasks" \ --harnesses docker-oracle,docker-probe \ --out-json "${{ github.workspace }}/logs/external-catalog-smoke/runs.json" - # The base-branch tip this run's checkout fetched, which its merge commit was built on. The - # changes step of a later docs-only push skips only while the base is still this one. + # The base the merge commit (refs/pull/N/merge) was built on, its first parent; not origin/$BASE_REF, which may have moved since. + # A PR that does not merge records nothing. A later docs-only push skips only while origin/$BASE_REF is still this one. - name: Record the base this run tested if: steps.changes.outputs.code == 'true' && github.event_name == 'pull_request' env: BASE_REF: ${{ github.event.pull_request.base.ref }} - run: git rev-parse --verify "origin/$BASE_REF" > "${{ github.workspace }}/logs/external-catalog-smoke/base.sha" + RECORD: ${{ github.workspace }}/logs/external-catalog-smoke/base.sha + run: | + # The checkout is the PR's merge commit; its first parent is the base tip the merge was built on. + # Recorded only if that parent is on origin/; else nothing, so the next push runs everything. + rm -f "$RECORD" + if git rev-parse --verify -q HEAD^2 > /dev/null \ + && parent=$(git rev-parse --verify HEAD^1) \ + && git merge-base --is-ancestor "$parent" "origin/$BASE_REF"; then + echo "$parent" > "$RECORD" + fi # Every cell passed and none errored (an errored cell leaves pass_rate's # denominator, so pass_rate alone would read 1.0), and the probe's fetch # was denied by the egress proxy; see scripts/check_smoke_runs.py. diff --git a/dokimi/tests/test_agent_quality_changes_scope.py b/dokimi/tests/test_agent_quality_changes_scope.py index abc331a2b..511d98b30 100644 --- a/dokimi/tests/test_agent_quality_changes_scope.py +++ b/dokimi/tests/test_agent_quality_changes_scope.py @@ -4,10 +4,12 @@ """agent-quality's `changes` step decides whether a push runs the smoke test. A docs-only push may skip the code steps only when everything since the last commit -whose smoke run passed its code steps is documentation. A cancelled or failed run does -not count as passed, so the code it carried is diffed again. The step is run here as -the workflow runs it (its script, read from the YAML), with `gh` replaced by a stub -that serves run and job fixtures. +whose smoke run passed its code steps is documentation, and the base branch is still the +one that run's merge was built on (the smoke job records it). A cancelled or failed run +does not count as passed, so the code it carried is diffed again. The changes step and +the record step are run here as the workflow runs them (their scripts, read from the +YAML); the record step runs on a merge the test builds. `gh` is replaced by a stub that +serves run and job fixtures. """ from __future__ import annotations @@ -28,8 +30,7 @@ SMOKE_NAME = "External catalog smoke test (uxlfoundation/skills)" BULK_FILES = 2000 -# The stub `gh`: `gh api` serves runs.json and jobs-.json from $FAKE_GH_DIR, applies --jq -# with jq -r as gh does, and fails for a missing fixture (the API being unreadable). +# The stub `gh`: `gh api` serves runs.json and jobs-.json from $FAKE_GH_DIR (--jq as jq -r; a missing fixture fails). # `gh run download -D ` copies artifact-/ from $FAKE_GH_DIR into . GH_STUB = r"""#!/bin/sh if [ "$1" = run ] && [ "$2" = download ]; then @@ -427,6 +428,95 @@ def test_docs_push_runs_the_code_again_when_the_passed_run_recorded_no_base(tmp_ assert code == "code=true" +def _record_step() -> dict: + wf = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = wf["jobs"][SMOKE_JOB]["steps"] + return next(s for s in steps if s.get("name") == "Record the base this run tested") + + +def _record_base(tmp_path: Path, shape: str, main_moved: bool, stale: str | None = None) -> str: + """Run the record step on a checkout as the smoke job makes it; return the recorded sha ("" for none). + + The checkout is one of: "merge", the PR's merge commit of its head on base0; "head", the PR + head (one parent); "pr_merge", a PR head that is itself a merge, whose first parent is a PR + commit that is not on main. Once the checkout exists, main moves on to base1 when `main_moved`. + `stale` is a record file that the PR's own steps left behind. + """ + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + (repo / "app.txt").write_text("base0\n", encoding="utf-8") + base0 = _commit(repo, "base0") + _git(repo, "checkout", "-q", "-b", "feature", base0) + (repo / "feature.txt").write_text("feature\n", encoding="utf-8") + head = _commit(repo, "feature") + if shape == "merge": + _git(repo, "checkout", "-q", "-b", "merge", base0) + _git(repo, "merge", "-q", "--no-ff", "-m", "merge", "feature") + checkout = _git(repo, "rev-parse", "HEAD") + elif shape == "pr_merge": + _git(repo, "checkout", "-q", "-b", "other", base0) + (repo / "other.txt").write_text("other\n", encoding="utf-8") + other = _commit(repo, "other") + _git(repo, "checkout", "-q", "-b", "pr", "feature") + _git(repo, "merge", "-q", "--no-ff", "-m", "merge", other) + checkout = _git(repo, "rev-parse", "HEAD") + else: + checkout = head + _git(repo, "checkout", "-q", "-b", "tip", base0) + (repo / "main.txt").write_text("base1\n", encoding="utf-8") + base1 = _commit(repo, "base1") if main_moved else base0 + _git(repo, "update-ref", "refs/remotes/origin/main", base1) + _git(repo, "checkout", "-q", "--detach", checkout) + out = tmp_path / "logs" / "base.sha" + out.parent.mkdir() + if stale is not None: + out.write_text(stale, encoding="utf-8") + step = _record_step() + env = { + **os.environ, + **{k: str(v) for k, v in step.get("env", {}).items()}, + "BASE_REF": "main", + "RECORD": str(out), + } + proc = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-c", step["run"]], + cwd=repo, + env=env, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + return out.read_text(encoding="utf-8").strip() if out.exists() else "" + + +def test_the_recorded_base_is_the_one_the_merge_was_built_on_after_main_moved(tmp_path): + # The merge commit was built on base0; main reached base1 before the checkout. The record must + # say base0, or a later docs-only push compares base1 with base1 and skips the merge's base changes. + recorded = _record_base(tmp_path, "merge", main_moved=True) + assert recorded == _git(tmp_path / "repo", "rev-parse", "HEAD^1") + + +def test_a_checkout_of_the_pr_head_records_no_base_even_with_a_stale_record(tmp_path): + # A PR that does not merge is checked out as its head: nothing is recorded, so a later docs-only + # push runs the code again. A record the PR's own steps left behind must not survive either. + assert _record_base(tmp_path, "head", main_moved=False, stale="planted") == "" + + +def test_a_merge_of_a_pr_commit_that_is_not_on_main_records_no_base(tmp_path): + # A PR head that is itself a merge has two parents, but its first parent is a PR commit, not a + # base tip: recording it would make the next docs-only push compare a PR commit with origin/main. + assert _record_base(tmp_path, "pr_merge", main_moved=False) == "" + + +def test_the_record_step_runs_only_for_a_code_run_of_a_pull_request(): + # Its artifact is read by a later push only for a passed run; the guard keeps it to runs that did code. + assert ( + _record_step()["if"] + == "steps.changes.outputs.code == 'true' && github.event_name == 'pull_request'" + ) + + def test_the_smoke_job_may_read_the_runs_it_is_scoped_by(): job = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"][SMOKE_JOB] assert job["permissions"]["actions"] == "read"