From 5cf4a57723976f546527c77db779264ea3d7724a Mon Sep 17 00:00:00 2001 From: Viet Le Date: Fri, 30 Oct 2026 11:54:37 +0000 Subject: [PATCH 1/3] Update Beam.py --- src/bhom_schema/FixtureAlpha_oM/Beam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bhom_schema/FixtureAlpha_oM/Beam.py b/src/bhom_schema/FixtureAlpha_oM/Beam.py index 32a0793..41c7048 100644 --- a/src/bhom_schema/FixtureAlpha_oM/Beam.py +++ b/src/bhom_schema/FixtureAlpha_oM/Beam.py @@ -31,4 +31,4 @@ class Beam(BHoMObject): Start: PointLike | None = None Count: Annotated[int, ClrType('long')] = 3 Small: Annotated[float, ClrType('float')] = 0.5 - Material: str = Field(default='Steel', description='Name of the material.') + Material: str = Field(default='Concrete', description='Name of the material.') From 12b6e47de38bdd9cd7d750f5e6589f6395391ddf Mon Sep 17 00:00:00 2001 From: Viet Le Date: Fri, 30 Oct 2026 12:31:07 +0000 Subject: [PATCH 2/3] Make the C# PR alignment check more visible --- .../python/test_py2cs/test_workflows.py | 26 +++- .github/actions/bhom-assemblies/action.yml | 60 +++++++++ .github/workflows/pr-checks.yml | 122 ++++++++++-------- docs/CODING-PLAN.md | 2 +- docs/Testing-On-GitHub.md | 5 +- docs/Workflows.md | 2 +- 6 files changed, 152 insertions(+), 65 deletions(-) create mode 100644 .github/actions/bhom-assemblies/action.yml diff --git a/.ci/unit-tests/python/test_py2cs/test_workflows.py b/.ci/unit-tests/python/test_py2cs/test_workflows.py index f6695ee..6e9f261 100644 --- a/.ci/unit-tests/python/test_py2cs/test_workflows.py +++ b/.ci/unit-tests/python/test_py2cs/test_workflows.py @@ -33,7 +33,7 @@ def test_all_workflows_are_valid_and_have_the_expected_jobs(): Protects the generation, PR checks and linked C# PR workflows from broken edits. """ assert list(load("python-schema-generation.yml")["jobs"]) == ["generate"] - assert list(load("pr-checks.yml")["jobs"]) == ["checks"] + assert list(load("pr-checks.yml")["jobs"]) == ["checks", "alignment"] assert list(load("linked-csharp-pr.yml")["jobs"]) == ["prepare", "repos", "raise", "close", "status", "sweep"] def test_the_weekly_generation_commit_cannot_start_the_linked_pr_workflow(): @@ -133,8 +133,8 @@ def test_the_generation_and_the_checks_build_the_same_repositories(): Protects align, which compares the PR with what the generator would write. """ - for name in ("python-schema-generation.yml", "pr-checks.yml"): - text = (WORKFLOWS / name).read_text(encoding="utf-8") + for path in (WORKFLOWS / "python-schema-generation.yml", WORKFLOWS.parent / "actions" / "bhom-assemblies" / "action.yml"): + text = path.read_text(encoding="utf-8") assert "Clone-Repositories.ps1 -List Included-repositories.txt" in text and "Build-Repositories.ps1 -List Included-repositories.txt" in text def test_the_status_jobs_write_statuses_and_nothing_else(): @@ -202,11 +202,12 @@ def test_the_release_workflow_stores_no_secret_and_nothing_is_published_in_a_dry def test_the_pr_checks_build_only_when_the_cache_of_assemblies_misses(): - """Checks that pr-checks caches the built BHoM assemblies under a key made from the current commits, builds only on a cache miss, and saves the cache right after the build. + """Checks that the shared assemblies action caches the built BHoM assemblies under a key made from the current commits, builds only on a cache miss, and saves the cache right after the build. Protects the speed of the PR checks from a change that rebuilds every time, and the cache from becoming stale. """ - steps = load("pr-checks.yml")["jobs"]["checks"]["steps"] + action = yaml.safe_load((WORKFLOWS.parent / "actions" / "bhom-assemblies" / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] by_name = {s.get("name", s.get("uses", "")): s for s in steps} restore = by_name["Restore the built BHoM assemblies"] assert restore["with"]["key"] == "${{ steps.commits.outputs.key }}" @@ -214,4 +215,17 @@ def test_the_pr_checks_build_only_when_the_cache_of_assemblies_misses(): assert by_name[name]["if"] == "steps.assemblies.outputs.cache-hit != 'true'" assert "git ls-remote" in by_name["Find the current commit of each repository"]["run"] order = [s.get("name", "") for s in steps] - assert order.index("Build the repositories in dependency order") < order.index("Save the built BHoM assemblies") < order.index("Build and test") + assert order.index("Build the repositories in dependency order") < order.index("Save the built BHoM assemblies") + + +def test_the_alignment_check_is_its_own_job_with_a_name_that_says_what_is_pending(): + """Checks that the C# alignment runs as its own job of pr-checks, named after the linked C# pull request, using the shared assemblies action and explaining a failure in the job summary, and that the main job no longer runs align. + + Protects the pull request page from showing an unexplained failure while the C# pull request is still open. + """ + jobs = load("pr-checks.yml")["jobs"] + assert "linked C# pull request" in jobs["alignment"]["name"] + assert any(s.get("uses") == "./.github/actions/bhom-assemblies" for s in jobs["alignment"]["steps"]) + assert any(s.get("name") == "Explain the failure" and "GITHUB_STEP_SUMMARY" in s["run"] for s in jobs["alignment"]["steps"]) + assert not any("align" in s.get("run", "") for s in jobs["checks"]["steps"]) + assert any("py2cs check" in s.get("run", "") for s in jobs["checks"]["steps"]) diff --git a/.github/actions/bhom-assemblies/action.yml b/.github/actions/bhom-assemblies/action.yml new file mode 100644 index 0000000..ca11dff --- /dev/null +++ b/.github/actions/bhom-assemblies/action.yml @@ -0,0 +1,60 @@ +name: Built BHoM assemblies +description: > + Installs the built assemblies of the repositories in Included-repositories.txt into C:\ProgramData\BHoM\Assemblies, from a cache when none of them + has a new commit, otherwise by cloning and building them (and saving the cache). + +runs: + using: composite + steps: + # The built assemblies only change when one of the cloned repositories does, so they are cached under the current commit of each + # repository (the same branch choice as Clone-Repositories.ps1). A new commit anywhere gives a new key, so the cache is never stale. + - name: Find the current commit of each repository + id: commits + shell: pwsh + run: | + $lines = foreach ($repository in (Get-Content Included-repositories.txt | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" -and -not $_.StartsWith("#") })) { + $commit = "" + foreach ($branch in "develop", "main", "master") { + $found = git ls-remote "https://github.com/$repository.git" "refs/heads/$branch" + if ($found) { $commit = ($found -split "\s+")[0]; break } + } + + if (-not $commit) { throw "Could not find a develop, main or master branch of $repository" } + "$repository $commit" + } + + $text = $lines -join "`n" + Write-Host $text + $hash = [System.BitConverter]::ToString([System.Security.Cryptography.SHA256]::Create().ComputeHash([System.Text.Encoding]::UTF8.GetBytes($text))).Replace("-", "").ToLower() + "key=bhom-assemblies-v1-$hash" >> $env:GITHUB_OUTPUT + + - name: Restore the built BHoM assemblies + id: assemblies + uses: actions/cache/restore@v4 + with: + path: C:\ProgramData\BHoM\Assemblies + key: ${{ steps.commits.outputs.key }} + + - uses: microsoft/setup-msbuild@v2 + if: steps.assemblies.outputs.cache-hit != 'true' + + - uses: NuGet/setup-nuget@v2 + if: steps.assemblies.outputs.cache-hit != 'true' + + - name: Clone the repositories + if: steps.assemblies.outputs.cache-hit != 'true' + shell: pwsh + run: ./.github/scripts/Clone-Repositories.ps1 -List Included-repositories.txt + + - name: Build the repositories in dependency order + if: steps.assemblies.outputs.cache-hit != 'true' + shell: pwsh + run: ./.github/scripts/Build-Repositories.ps1 -List Included-repositories.txt + + # Saved right after the build, so a failure later in the job does not lose it + - name: Save the built BHoM assemblies + if: steps.assemblies.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: C:\ProgramData\BHoM\Assemblies + key: ${{ steps.commits.outputs.key }} diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5980159..9762066 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -32,58 +32,8 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - # The built assemblies only change when one of the cloned repositories does, so they are cached under the current commit of each - # repository (the same branch choice as Clone-Repositories.ps1). A new commit anywhere gives a new key, so the cache is never stale. - - name: Find the current commit of each repository - id: commits - shell: pwsh - run: | - $lines = foreach ($repository in (Get-Content Included-repositories.txt | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" -and -not $_.StartsWith("#") })) { - $commit = "" - foreach ($branch in "develop", "main", "master") { - $found = git ls-remote "https://github.com/$repository.git" "refs/heads/$branch" - if ($found) { $commit = ($found -split "\s+")[0]; break } - } - - if (-not $commit) { throw "Could not find a develop, main or master branch of $repository" } - "$repository $commit" - } - - $text = $lines -join "`n" - Write-Host $text - $hash = [System.BitConverter]::ToString([System.Security.Cryptography.SHA256]::Create().ComputeHash([System.Text.Encoding]::UTF8.GetBytes($text))).Replace("-", "").ToLower() - "key=bhom-assemblies-v1-$hash" >> $env:GITHUB_OUTPUT - - - name: Restore the built BHoM assemblies - id: assemblies - uses: actions/cache/restore@v4 - with: - path: C:\ProgramData\BHoM\Assemblies - key: ${{ steps.commits.outputs.key }} - - - uses: microsoft/setup-msbuild@v2 - if: steps.assemblies.outputs.cache-hit != 'true' - - - uses: NuGet/setup-nuget@v2 - if: steps.assemblies.outputs.cache-hit != 'true' - - - name: Clone the repositories - if: steps.assemblies.outputs.cache-hit != 'true' - shell: pwsh - run: ./.github/scripts/Clone-Repositories.ps1 -List Included-repositories.txt - - - name: Build the repositories in dependency order - if: steps.assemblies.outputs.cache-hit != 'true' - shell: pwsh - run: ./.github/scripts/Build-Repositories.ps1 -List Included-repositories.txt - - # Saved right after the build, so a failure later in the job does not lose it - - name: Save the built BHoM assemblies - if: steps.assemblies.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: C:\ProgramData\BHoM\Assemblies - key: ${{ steps.commits.outputs.key }} + - name: Install the built BHoM assemblies + uses: ./.github/actions/bhom-assemblies # The published JSON schemas are the test oracle for the JSON that Python writes. Read only; the generator does not use them. - name: Clone BHoM_JSONSchema (test oracle) @@ -120,9 +70,56 @@ jobs: shell: pwsh run: .venv\Scripts\python -m py2cs check --root src/bhom_schema --base-root "$env:RUNNER_TEMP/base/src/bhom_schema" - # The Python PR is a proposal: it only passes once the linked C# PR is merged, because the clones above are the C# develop branches. - # After the C# PR is merged, re-run this workflow. - - name: py2cs align (fails until the linked C# PR is merged) + # Its own job, so that the pull request page names what is pending. The Python PR is a proposal: this only passes once the linked C# pull + # request is merged, because the assemblies built here come from the C# develop branches. After that merge, re-run this job. + alignment: + name: C# alignment (fails until the linked C# pull request is merged) + runs-on: windows-latest + env: + PYTHONPATH: ${{ github.workspace }}/.ci/generation + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: pyproject.toml + + - name: Does the Python PR change the classes? + id: changed + shell: pwsh + if: github.event_name == 'pull_request' + run: | + $base = "${{ github.event.pull_request.base.sha }}" + $changed = git diff --name-only $base HEAD -- src/bhom_schema + "classes=$([bool]$changed)".ToLower() | Out-File $env:GITHUB_OUTPUT -Append + + - name: Install the built BHoM assemblies + if: steps.changed.outputs.classes == 'true' + uses: ./.github/actions/bhom-assemblies + + - name: Build the generator + if: steps.changed.outputs.classes == 'true' + run: dotnet build .ci/generation/PythonSchemaGeneration/PythonSchemaGeneration.csproj -c Release --nologo + + - name: Create the virtual environment + if: steps.changed.outputs.classes == 'true' + shell: pwsh + run: | + python -m venv .venv + .venv\Scripts\python -m pip install -e . + + - name: Get the classes before the change + if: steps.changed.outputs.classes == 'true' + shell: pwsh + run: git worktree add "$env:RUNNER_TEMP/base" ${{ github.event.pull_request.base.sha }} + + - name: py2cs align + id: align if: steps.changed.outputs.classes == 'true' shell: pwsh run: | @@ -130,3 +127,18 @@ jobs: --base-root "$env:RUNNER_TEMP/base/src/bhom_schema" ` --head-root src/bhom_schema ` --assemblies "$env:ProgramData\BHoM\Assemblies" + + - name: Explain the failure + if: failure() && steps.align.conclusion == 'failure' + shell: pwsh + run: | + @' + ## C# alignment failed + + The classes changed in this pull request are compared with the Python that the generator writes from the **merged C#**. They differ, which usually means the linked C# pull request is not merged yet. + + 1. Open the linked draft pull request (the commit status `linked C# pull request` links it) and merge it in the C# repository. + 2. Re-run this job. It should then pass. + + If the C# is already merged and it still fails, the differences are listed in the log of the step `py2cs align`: either the C# review changed the class (copy that change into the Python), or the generator and py2cs disagree (a bug to fix in this repository). + '@ | Out-File $env:GITHUB_STEP_SUMMARY -Append diff --git a/docs/CODING-PLAN.md b/docs/CODING-PLAN.md index 00f7c9a..62da0dd 100644 --- a/docs/CODING-PLAN.md +++ b/docs/CODING-PLAN.md @@ -9,7 +9,7 @@ taken so far (section 9). ## Implementation status (phases 0 to 6 done, 7 and 8 not started) -Run `.ci/unit-tests/run-all.ps1`; it builds, generates, drafts, round-trips, runs NUnit (92 tests), pytest (198 tests) and Pyright. +Run `.ci/unit-tests/run-all.ps1`; it builds, generates, drafts, round-trips, runs NUnit (92 tests), pytest (199 tests) and Pyright. Where the implementation differs from the plan above: diff --git a/docs/Testing-On-GitHub.md b/docs/Testing-On-GitHub.md index 00da068..b8988d7 100644 --- a/docs/Testing-On-GitHub.md +++ b/docs/Testing-On-GitHub.md @@ -129,7 +129,8 @@ Goal: a class that nothing else uses, written from scratch. | Where | What you should see | |---|---| -| **pr-checks** on the Python PR | Build, NUnit, pytest and Pyright pass. `py2cs check` passes. `py2cs align` **fails until the linked C# PR is merged**: it compares the Python with what the C# `develop` generates. That failure is the proof that the two are not in step yet | +| **PR checks / checks** on the Python PR | Build, NUnit, pytest and Pyright pass. `py2cs check` passes | +| **PR checks / C# alignment** on the Python PR | **Fails until the linked C# PR is merged**, with an explanation in the job summary: it compares the Python with what the C# `develop` generates. That failure is the proof that the two are not in step yet | | **linked-csharp-pr**, `prepare`, `repos`, `raise` | Green. The log of `repos` names `BHoM/CSharpSchema_Test`; `raise` pushes `python-schema/pr-` and opens a **draft** PR | | The C# draft PR | The files and lines in the table at the top of the exercise, a link back to the Python PR, labels, and nothing else changed. Open the diff and read it as a C# reviewer would. Check that it builds | | Commit status on the Python PR | `linked C# pull request`: pending while the draft is open | @@ -142,7 +143,7 @@ prints what it would do and proves the App token; unticked it does it. 1. In CSharpSchema_Test, review the draft, fix anything wrong **in C#** (the C# is the source of truth), mark it ready and merge it. 2. Back on the Python PR: within the hour the status becomes `success` (or start *Linked C# pull request* by hand to refresh it now). -3. Re-run **pr-checks**. `py2cs align` must now pass: the Python generated from the merged C# equals the Python in the PR. +3. Re-run **C# alignment**. It must now pass: the Python generated from the merged C# equals the Python in the PR. If it still fails, the message lists what differs; either the C# review changed something (copy the change into the Python) or the generator and `py2cs` disagree (a bug to fix in this repository). The NUnit tests and their golden files use their own test classes (`.ci/unit-tests/csharp/Fixtures`), so a merged diff --git a/docs/Workflows.md b/docs/Workflows.md index 91c9811..31b91fc 100644 --- a/docs/Workflows.md +++ b/docs/Workflows.md @@ -7,7 +7,7 @@ written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built | Workflow | When | What it does | |---|---|---| | [python-schema-generation.yml](../.github/workflows/python-schema-generation.yml) | Every Saturday at 4 AM UTC, and by hand | Clones and builds the C# repositories, generates `src/bhom_schema` from the C#, checks it, and commits it straight to `develop`. Not a pull request. If a check fails, nothing is committed | -| [pr-checks.yml](../.github/workflows/pr-checks.yml) | Every pull request to `develop` | Builds the same repositories (`Included-repositories.txt`; the built assemblies are cached under the current commit of each repository, so unchanged repositories are not rebuilt), clones BHoM_JSONSchema read only as the test oracle for the JSON, then runs `run-all.ps1` (build, NUnit, pytest, Pyright). For a PR that changes `src/bhom_schema` it also runs `py2cs check` and `py2cs align` | +| [pr-checks.yml](../.github/workflows/pr-checks.yml) | Every pull request to `develop` | Two jobs. **checks**: installs the built assemblies of the repositories in `Included-repositories.txt` (the shared action `.github/actions/bhom-assemblies` caches them under the current commit of each repository, so unchanged repositories are not rebuilt), clones BHoM_JSONSchema read only as the test oracle for the JSON, runs `run-all.ps1` (build, NUnit, pytest, Pyright) and, for a PR that changes `src/bhom_schema`, `py2cs check`. **C# alignment (fails until the linked C# pull request is merged)**: for such a PR, builds the generator and runs `py2cs align`; when it fails it explains in the job summary that the linked C# pull request has to be merged first, then the job re-run | | [linked-csharp-pr.yml](../.github/workflows/linked-csharp-pr.yml) | A pull request that changes `src/bhom_schema`, every hour, and by hand | Raises, updates or closes a linked **draft** pull request in the C# repository, and sets the commit status `linked C# pull request` on the Python PR | | [release.yml](../.github/workflows/release.yml) | By hand only (dry run by default) | Regenerates with `--release X.Y`, checks, builds the wheel and the release notes; when not a dry run, pushes branch `vX.Y` and tag `vX.Y.0`, creates the GitHub Release and publishes to TestPyPI. See "Making a release" | From 40ed81ffc3d656d9b8fafc6100f8d71d84975887 Mon Sep 17 00:00:00 2001 From: Viet Le Date: Fri, 30 Oct 2026 14:48:53 +0000 Subject: [PATCH 3/3] Simplify list of checks --- .../python/test_py2cs/test_workflows.py | 134 ++++---- .github/actions/linked-pr-context/action.yml | 116 +++++++ .github/workflows/linked-csharp-pr-close.yml | 71 +++++ .github/workflows/linked-csharp-pr-sweep.yml | 42 +++ .github/workflows/linked-csharp-pr.yml | 290 ++++-------------- docs/Private-Repositories.md | 6 +- docs/Testing-On-GitHub.md | 2 +- docs/Workflows.md | 21 +- 8 files changed, 385 insertions(+), 297 deletions(-) create mode 100644 .github/actions/linked-pr-context/action.yml create mode 100644 .github/workflows/linked-csharp-pr-close.yml create mode 100644 .github/workflows/linked-csharp-pr-sweep.yml diff --git a/.ci/unit-tests/python/test_py2cs/test_workflows.py b/.ci/unit-tests/python/test_py2cs/test_workflows.py index 6e9f261..63a9726 100644 --- a/.ci/unit-tests/python/test_py2cs/test_workflows.py +++ b/.ci/unit-tests/python/test_py2cs/test_workflows.py @@ -28,22 +28,34 @@ def lines(name: str) -> list[str]: return [line.strip() for line in (REPO / name).read_text(encoding="utf-8").splitlines() if line.strip() and not line.strip().startswith("#")] def test_all_workflows_are_valid_and_have_the_expected_jobs(): - """Checks that the three workflows are valid YAML with the expected jobs. + """Checks that the workflows are valid YAML with the expected jobs. Protects the generation, PR checks and linked C# PR workflows from broken edits. """ assert list(load("python-schema-generation.yml")["jobs"]) == ["generate"] assert list(load("pr-checks.yml")["jobs"]) == ["checks", "alignment"] - assert list(load("linked-csharp-pr.yml")["jobs"]) == ["prepare", "repos", "raise", "close", "status", "sweep"] + assert list(load("linked-csharp-pr.yml")["jobs"]) == ["linked"], "one job, so the pull request page shows one line" + assert list(load("linked-csharp-pr-close.yml")["jobs"]) == ["close"] + assert list(load("linked-csharp-pr-sweep.yml")["jobs"]) == ["sweep"] def test_the_weekly_generation_commit_cannot_start_the_linked_pr_workflow(): - """Checks that linked-csharp-pr.yml has no push trigger and runs for pull requests only on src/bhom_schema paths. + """Checks that none of the linked-PR workflows has a push trigger, that they run for pull requests only on src/bhom_schema paths, and that only the close workflow reacts to a closed pull request. - Protects against the weekly generation commit, which is a push, starting a linked C# PR. + Protects against the weekly generation commit, which is a push, starting a linked C# PR, and the open pull request page from showing jobs that only matter on closing. """ - triggers = load("linked-csharp-pr.yml")["on"] - assert set(triggers) == {"pull_request", "schedule", "workflow_dispatch"}, "never a push trigger: the weekly commit is a push" - assert triggers["pull_request"]["paths"] == ["src/bhom_schema/**"] + raised = load("linked-csharp-pr.yml")["on"] + assert set(raised) == {"pull_request", "workflow_dispatch"}, "never a push trigger: the weekly commit is a push" + assert raised["pull_request"]["paths"] == ["src/bhom_schema/**"] + assert "closed" not in raised["pull_request"]["types"] + + closed = load("linked-csharp-pr-close.yml")["on"] + assert set(closed) == {"pull_request", "workflow_dispatch"} + assert closed["pull_request"]["types"] == ["closed"] and closed["pull_request"]["paths"] == ["src/bhom_schema/**"] + + swept = load("linked-csharp-pr-sweep.yml")["on"] + assert set(swept) == {"schedule", "workflow_dispatch"} + assert swept["schedule"] == [{"cron": "17 * * * *"}] + def test_the_weekly_generation_commits_directly_and_is_not_a_pull_request(): """Checks that the generation workflow runs on schedule or on demand and pushes straight to develop without opening a PR. @@ -61,55 +73,72 @@ def test_workflows_start_with_read_only_permissions_and_raise_them_per_job(): Protects the token permissions of the workflows. """ - assert load("linked-csharp-pr.yml")["permissions"] == {"contents": "read"} - assert load("pr-checks.yml")["permissions"] == {"contents": "read"} - assert load("linked-csharp-pr.yml")["jobs"]["raise"]["permissions"] == {"contents": "read", "pull-requests": "write"} - assert "permissions" not in load("linked-csharp-pr.yml")["jobs"]["close"] + for name in ("linked-csharp-pr.yml", "linked-csharp-pr-close.yml", "linked-csharp-pr-sweep.yml", "pr-checks.yml"): + assert load(name)["permissions"] == {"contents": "read"}, name + + assert load("linked-csharp-pr.yml")["jobs"]["linked"]["permissions"] == {"contents": "read", "pull-requests": "write", "statuses": "write"} + assert "permissions" not in load("linked-csharp-pr-close.yml")["jobs"]["close"] -def test_the_app_token_is_made_for_one_repository_of_the_matrix_only(): - """Checks that the raise and close jobs make the GitHub App token for the single repository of the matrix and the owner. + +def test_the_app_token_is_made_only_for_the_repositories_the_change_touches(): + """Checks that the raise and close jobs make the GitHub App token only for the repositories that the change touches (a subset of the allowed list) and the owner. Protects the C# repositories from a token with wider reach. """ - jobs = load("linked-csharp-pr.yml")["jobs"] - for name in ("raise", "close"): - step = next(s for s in jobs[name]["steps"] if str(s.get("uses", "")).startswith("actions/create-github-app-token")) - assert step["with"]["repositories"] == "${{ matrix.name }}" + for name, job in (("linked-csharp-pr.yml", "linked"), ("linked-csharp-pr-close.yml", "close")): + step = next(s for s in load(name)["jobs"][job]["steps"] if str(s.get("uses", "")).startswith("actions/create-github-app-token")) + assert step["with"]["repositories"] == "${{ steps.ctx.outputs.names }}" assert step["with"]["owner"] == "${{ github.repository_owner }}" + action = yaml.safe_load((WORKFLOWS.parent / "actions" / "linked-pr-context" / "action.yml").read_text(encoding="utf-8")) + repos_step = next(s for s in action["runs"]["steps"] if s.get("id") == "repos") + assert "--allowed-repos base/Converted-repositories.txt" in repos_step["run"], "the touched repositories are always checked against the allowed list" + + def test_tokens_only_reach_the_steps_that_need_them(): """Checks that the app token reaches only steps that create it or need it, and not the planning step. Protects the token from the steps that run code from the PR. """ - jobs = load("linked-csharp-pr.yml")["jobs"] - for job in jobs.values(): - for step in job["steps"]: - uses_token = "steps.token.outputs.token" in str(step) - assert not uses_token or step.get("id") == "token" or "PY2CS_CSHARP_TOKEN" in step.get("env", {}) or step.get("with", {}).get("token"), step.get("name") + for name in ("linked-csharp-pr.yml", "linked-csharp-pr-close.yml"): + for job in load(name)["jobs"].values(): + for step in job["steps"]: + uses_token = "steps.token.outputs.token" in str(step) + assert not uses_token or step.get("id") == "token" or "PY2CS_CSHARP_TOKEN" in step.get("env", {}) or step.get("with", {}).get("token"), step.get("name") - plan_step = next(s for s in jobs["raise"]["steps"] if s.get("name") == "Plan the C# changes") + plan_step = next(s for s in load("linked-csharp-pr.yml")["jobs"]["linked"]["steps"] if s.get("name") == "Plan the C# changes") assert "PY2CS_CSHARP_TOKEN" not in plan_step.get("env", {}), "planning does not need the token" + def test_the_tooling_comes_from_the_pull_request_and_the_allowed_repositories_from_the_base(): - """Checks that the linked-PR workflow runs py2cs and the patcher from the PR head checkout, and reads the list of repositories a C# pull request may be raised in from the base. + """Checks that the linked-PR workflows run py2cs and the patcher from the PR head checkout, and read the list of repositories a C# pull request may be raised in from the base. Protects the choice that a fix to the tooling works on its own pull request, and the list of where the App may write from being changed by that pull request. """ - text = (WORKFLOWS / "linked-csharp-pr.yml").read_text(encoding="utf-8") + action = (WORKFLOWS.parent / "actions" / "linked-pr-context" / "action.yml").read_text(encoding="utf-8") + text = "".join((WORKFLOWS / name).read_text(encoding="utf-8") for name in ("linked-csharp-pr.yml", "linked-csharp-pr-close.yml")) + action assert "PYTHONPATH: head/.ci/generation" in text and "PYTHONPATH: base/.ci/generation" not in text assert "PY2CS_PATCHER: head/.ci/generation/" in text assert "base/Converted-repositories.txt" in text and "head/Converted-repositories.txt" not in text + def test_fork_pull_requests_get_no_linked_pull_request(): - """Checks that the linked-PR workflow marks pull requests from forks as not eligible and that the repos job requires eligibility. + """Checks that the shared action marks pull requests from forks as not eligible, and that every step of the raise and close jobs that uses a secret or writes waits for the gate of that action. - Protects the secrets of the workflow from fork PRs. + Protects the secrets of the workflows from fork PRs. """ - text = (WORKFLOWS / "linked-csharp-pr.yml").read_text(encoding="utf-8") - assert ".head.repo.full_name" in text and "eligible=false" in text - jobs = load("linked-csharp-pr.yml")["jobs"] - assert "eligible == 'true'" in jobs["repos"]["if"] + action = (WORKFLOWS.parent / "actions" / "linked-pr-context" / "action.yml").read_text(encoding="utf-8") + assert ".head.repo.full_name" in action and "eligible=false" in action + + for name, job, gate in (("linked-csharp-pr.yml", "linked", "raise"), ("linked-csharp-pr-close.yml", "close", "close")): + for step in load(name)["jobs"][job]["steps"]: + if step.get("id") == "ctx" or str(step.get("uses", "")).startswith("actions/checkout") or step.get("name") == "Set the commit status": + continue + assert f"steps.ctx.outputs.{gate} == 'true'" in step["if"], step.get("name") + + status = next(s for s in load("linked-csharp-pr.yml")["jobs"]["linked"]["steps"] if s.get("name") == "Set the commit status") + assert status["if"] == "steps.ctx.outputs.open == 'true'" + def test_the_main_bhom_repository_is_never_written_to(): """Checks that the list of converted repositories holds only the test repository and that BHoM is in the build list. @@ -138,38 +167,39 @@ def test_the_generation_and_the_checks_build_the_same_repositories(): assert "Clone-Repositories.ps1 -List Included-repositories.txt" in text and "Build-Repositories.ps1 -List Included-repositories.txt" in text def test_the_status_jobs_write_statuses_and_nothing_else(): - """Checks that the status and sweep jobs get statuses write permission, no other job does and the status jobs make no app token. + """Checks that the linked job and the sweep job get statuses write permission, the close job does not, and that no App token is made for the sweep. - Protects the permissions of the status part of the linked-PR workflow. + Protects the permissions of the status part of the linked-PR workflows. """ - jobs = load("linked-csharp-pr.yml")["jobs"] - assert jobs["status"]["permissions"] == {"contents": "read", "statuses": "write"} - assert jobs["sweep"]["permissions"] == {"contents": "read", "pull-requests": "read", "statuses": "write"} - for name in ("prepare", "repos", "raise", "close"): - assert "statuses" not in jobs[name].get("permissions", {}) + assert load("linked-csharp-pr.yml")["jobs"]["linked"]["permissions"]["statuses"] == "write" + sweep = load("linked-csharp-pr-sweep.yml")["jobs"]["sweep"] + assert sweep["permissions"] == {"contents": "read", "pull-requests": "read", "statuses": "write"} + assert "statuses" not in load("linked-csharp-pr-close.yml")["jobs"]["close"].get("permissions", {}) - # They read the C# repositories with the workflow's own token, so no GitHub App token is made for them - for name in ("status", "sweep"): - assert not any(str(s.get("uses", "")).startswith("actions/create-github-app-token") for s in jobs[name]["steps"]) + # The sweep reads the C# repositories with the workflow's own token, so no GitHub App token is made for it + assert not any(str(s.get("uses", "")).startswith("actions/create-github-app-token") for s in sweep["steps"]) -def test_the_hourly_sweep_runs_only_on_the_schedule_and_the_other_jobs_never_do(): - """Checks that the sweep job runs only on the hourly schedule, the prepare job never does and the cron is 17 * * * *. - Protects the hourly refresh of statuses from running the raise or close jobs. +def test_the_hourly_sweep_is_its_own_workflow_and_no_other_workflow_runs_on_a_schedule_for_it(): + """Checks that the hourly sweep lives in its own workflow, which has the cron 17 * * * * and no pull_request trigger, and that the other linked-PR workflows have no schedule. + + Protects the pull request page from showing the sweep, and the hourly refresh from running the raise or close steps. """ - jobs = load("linked-csharp-pr.yml")["jobs"] - assert jobs["sweep"]["if"] == "github.event_name == 'schedule'" - assert jobs["prepare"]["if"] == "github.event_name != 'schedule'" - assert load("linked-csharp-pr.yml")["on"]["schedule"] == [{"cron": "17 * * * *"}] + assert load("linked-csharp-pr-sweep.yml")["on"]["schedule"] == [{"cron": "17 * * * *"}] + assert "pull_request" not in load("linked-csharp-pr-sweep.yml")["on"] + assert "schedule" not in load("linked-csharp-pr.yml")["on"] and "schedule" not in load("linked-csharp-pr-close.yml")["on"] + def test_a_python_pr_with_no_csharp_change_still_gets_a_status(): - """Checks that the status job also runs when no repository is affected and passes --no-csharp-change. + """Checks that the linked job sets the status for every open pull request of this repository, also when no repository is affected, and passes --no-csharp-change then. Protects the required status from staying pending on a Python PR that needs no C# change. """ - status = load("linked-csharp-pr.yml")["jobs"]["status"] - assert "needs.repos.outputs.any == 'false'" in status["if"] - assert "--no-csharp-change" in str(status["steps"]) + steps = load("linked-csharp-pr.yml")["jobs"]["linked"]["steps"] + status = next(s for s in steps if s.get("name") == "Set the commit status") + assert status["if"] == "steps.ctx.outputs.open == 'true'" + assert "--no-csharp-change" in status["run"] + def test_the_release_workflow_is_manual_and_only_the_publish_job_gets_an_identity_token(): diff --git a/.github/actions/linked-pr-context/action.yml b/.github/actions/linked-pr-context/action.yml new file mode 100644 index 0000000..6be8375 --- /dev/null +++ b/.github/actions/linked-pr-context/action.yml @@ -0,0 +1,116 @@ +name: Linked pull request context +description: > + Finds out about the Python pull request (its number, commits, whether it is closed, and whether it comes from a fork), checks out its base and head, + and lists the C# repositories the change touches. Used by the linked C# pull request workflows. + +inputs: + pr-number: + description: Number of the Python pull request for a manual run; leave empty for a pull_request event + required: false + default: '' + +outputs: + number: + description: The pull request number + value: ${{ steps.pr.outputs.number }} + url: + description: The pull request address + value: ${{ steps.pr.outputs.url }} + merged: + description: true when the pull request was merged + value: ${{ steps.pr.outputs.merged }} + open: + description: true for a pull request of this repository (not a fork) that is not closed + value: ${{ steps.gate.outputs.open }} + raise: + description: true when it is open and the change touches a C# repository + value: ${{ steps.gate.outputs.raise }} + close: + description: true when it is closed and the change touches a C# repository + value: ${{ steps.gate.outputs.close }} + repos: + description: The C# repositories touched, as a JSON list of Owner/Name + value: ${{ steps.repos.outputs.repos }} + names: + description: The same repositories as a comma-separated list of names, for the App token + value: ${{ steps.repos.outputs.names }} + +runs: + using: composite + steps: + - name: Find out about the pull request + id: pr + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + INPUT_NUMBER: ${{ inputs.pr-number }} + run: | + if [ "$EVENT" = "workflow_dispatch" ]; then + data=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$INPUT_NUMBER") + else + data=$(gh api "repos/$GITHUB_REPOSITORY/pulls/${{ github.event.pull_request.number }}") + fi + + # Forks never receive the App secrets, so a pull request from a fork gets no linked C# pull request + if [ "$(echo "$data" | jq -r '.head.repo.full_name')" != "$GITHUB_REPOSITORY" ]; then + echo "eligible=false" >> "$GITHUB_OUTPUT" + echo "The pull request is from a fork: no linked C# pull request." + exit 0 + fi + + { + echo "eligible=true" + echo "number=$(echo "$data" | jq -r '.number')" + echo "url=$(echo "$data" | jq -r '.html_url')" + echo "base_sha=$(echo "$data" | jq -r '.base.sha')" + echo "head_sha=$(echo "$data" | jq -r '.head.sha')" + echo "closed=$([ "$(echo "$data" | jq -r '.state')" = closed ] && echo true || echo false)" + echo "merged=$(echo "$data" | jq -r '.merged')" + } >> "$GITHUB_OUTPUT" + + # The tooling (py2cs, the patcher) comes from the pull request, so a fix to it works on the same pull request. The Python before the change + # and the list of repositories a C# pull request may be raised in come from the base branch. + - uses: actions/checkout@v4 + if: steps.pr.outputs.eligible == 'true' + with: + ref: ${{ steps.pr.outputs.base_sha }} + path: base + + - uses: actions/checkout@v4 + if: steps.pr.outputs.eligible == 'true' + with: + ref: ${{ steps.pr.outputs.head_sha }} + path: head + + - uses: actions/setup-python@v5 + if: steps.pr.outputs.eligible == 'true' + with: + python-version: '3.12' + + - name: Which repositories does the change touch? + id: repos + if: steps.pr.outputs.eligible == 'true' + shell: bash + env: + PYTHONPATH: head/.ci/generation + run: | + repos=$(python -m py2cs repos --base-root base/src/bhom_schema --head-root head/src/bhom_schema --allowed-repos base/Converted-repositories.txt) + echo "Repositories: $repos" + echo "repos=$repos" >> "$GITHUB_OUTPUT" + echo "names=$(echo "$repos" | jq -r 'map(split("/")[1]) | join(",")')" >> "$GITHUB_OUTPUT" + echo "any=$([ "$repos" != "[]" ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + + - name: What is to be done? + id: gate + shell: bash + env: + ELIGIBLE: ${{ steps.pr.outputs.eligible }} + CLOSED: ${{ steps.pr.outputs.closed }} + ANY: ${{ steps.repos.outputs.any }} + run: | + { + echo "open=$([ "$ELIGIBLE" = true ] && [ "$CLOSED" = false ] && echo true || echo false)" + echo "raise=$([ "$ELIGIBLE" = true ] && [ "$CLOSED" = false ] && [ "$ANY" = true ] && echo true || echo false)" + echo "close=$([ "$ELIGIBLE" = true ] && [ "$CLOSED" = true ] && [ "$ANY" = true ] && echo true || echo false)" + } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/linked-csharp-pr-close.yml b/.github/workflows/linked-csharp-pr-close.yml new file mode 100644 index 0000000..abe0bd2 --- /dev/null +++ b/.github/workflows/linked-csharp-pr-close.yml @@ -0,0 +1,71 @@ +name: Linked C# pull request (close) + +# When the Python pull request is closed: unmerged closes the linked C# draft with a comment, merged only comments on it. +# A separate workflow from linked-csharp-pr.yml, so that nothing of it shows on a pull request that is still open. + +on: + pull_request: + types: [closed] + paths: + - 'src/bhom_schema/**' + workflow_dispatch: + inputs: + pr_number: + description: Number of the Python pull request (it must be closed) + required: true + type: string + dry_run: + description: List what would be done, change nothing (the App token is still made, which proves the installation) + type: boolean + default: true + +permissions: + contents: read + +# The same group as linked-csharp-pr.yml: closing the pull request cancels a raise that is still running +concurrency: + group: linked-csharp-pr-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + close: + name: close the C# draft + runs-on: ubuntu-latest + steps: + # The repository itself, only to reach the shared action below + - uses: actions/checkout@v4 + + - name: Find the pull request and the C# repositories it touches + id: ctx + uses: ./.github/actions/linked-pr-context + with: + pr-number: ${{ inputs.pr_number }} + + - name: Make a token for the C# repositories touched + id: token + if: steps.ctx.outputs.close == 'true' + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.BHOM_APP_ID }} + private-key: ${{ secrets.BHOM_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ steps.ctx.outputs.names }} + + # Unmerged: the linked draft is closed with a comment. Merged: only a comment. + - name: Close or comment on the linked pull request + if: steps.ctx.outputs.close == 'true' + shell: bash + env: + PYTHONPATH: head/.ci/generation + PY2CS_CSHARP_TOKEN: ${{ steps.token.outputs.token }} + DRY_RUN: ${{ inputs.dry_run }} + MERGED: ${{ steps.ctx.outputs.merged }} + REPOS: ${{ steps.ctx.outputs.repos }} + run: | + flags="" + if [ "$DRY_RUN" = "true" ]; then flags="$flags --dry-run"; fi + if [ "$MERGED" = "true" ]; then flags="$flags --merged"; fi + for repo in $(echo "$REPOS" | jq -r '.[]'); do + python -m py2cs close --repo "$repo" --python-repo "${{ github.repository }}" \ + --python-pr "${{ steps.ctx.outputs.number }}" $flags + done diff --git a/.github/workflows/linked-csharp-pr-sweep.yml b/.github/workflows/linked-csharp-pr-sweep.yml new file mode 100644 index 0000000..e581776 --- /dev/null +++ b/.github/workflows/linked-csharp-pr-sweep.yml @@ -0,0 +1,42 @@ +name: Linked C# pull request (hourly status) + +# Every hour: has the C# pull request of an open Python PR been merged (or closed) since? Nothing tells this repository when that happens. +# This only reads the C# repositories, which must be public, and writes the commit status "linked C# pull request" on this repository. +# A schedule only, so nothing of it shows on a pull request. + +on: + schedule: + - cron: '17 * * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + sweep: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + statuses: write + steps: + # The default branch, so the tooling is the trusted one + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Update the status of every open Python PR that changes the classes + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PYTHONPATH: .ci/generation + PY2CS_PYTHON_TOKEN: ${{ github.token }} + run: | + gh pr list --state open --limit 100 --json number,headRefOid,headRepositoryOwner,files \ + --jq '.[] | select(.headRepositoryOwner.login == "${{ github.repository_owner }}") | select(any(.files[]; .path | startswith("src/bhom_schema/"))) | "\(.number) \(.headRefOid)"' | + while read -r number sha; do + echo "Python PR #$number" + python -m py2cs status --python-repo "${{ github.repository }}" --python-pr "$number" --sha "$sha" --repos-file Converted-repositories.txt + done diff --git a/.github/workflows/linked-csharp-pr.yml b/.github/workflows/linked-csharp-pr.yml index 5fed3c0..74db51b 100644 --- a/.github/workflows/linked-csharp-pr.yml +++ b/.github/workflows/linked-csharp-pr.yml @@ -4,19 +4,18 @@ name: Linked C# pull request # It never raises anything for a push, so the weekly generation (a push to develop) cannot start it. # # It also sets a commit status on the Python PR, "linked C# pull request": pending while the C# draft is open, success once it is -# merged, failure if it was closed without merging. Nothing tells this repository when a C# PR is merged, so an hourly run looks at -# every open Python PR and updates the status. Make the status a required check on develop to stop a Python PR being merged first. +# merged, failure if it was closed without merging. Nothing tells this repository when a C# PR is merged, so an hourly run +# (linked-csharp-pr-sweep.yml) looks at every open Python PR and updates the status. Make the status a required check on develop to stop a +# Python PR being merged first. Closing the Python PR is handled by linked-csharp-pr-close.yml. # -# Where C# pull requests may be raised is listed in Converted-repositories.txt. The GitHub App token is made for one repository at a -# time, and only that repository is written to. +# Where C# pull requests may be raised is listed in Converted-repositories.txt. The GitHub App token is made only for the repositories the +# change touches, which are always listed there. on: pull_request: - types: [opened, synchronize, reopened, ready_for_review, closed] + types: [opened, synchronize, reopened, ready_for_review] paths: - 'src/bhom_schema/**' - schedule: - - cron: '17 * * * *' # hourly: refresh the status of open Python PRs workflow_dispatch: inputs: pr_number: @@ -31,283 +30,110 @@ on: permissions: contents: read -# A newer push cancels the older run for the same pull request +# A newer push cancels the older run for the same pull request; closing the pull request cancels a run in progress concurrency: group: linked-csharp-pr-${{ github.event.pull_request.number || inputs.pr_number }} cancel-in-progress: true jobs: - prepare: - if: github.event_name != 'schedule' - runs-on: ubuntu-latest - outputs: - eligible: ${{ steps.pr.outputs.eligible }} - number: ${{ steps.pr.outputs.number }} - url: ${{ steps.pr.outputs.url }} - base_sha: ${{ steps.pr.outputs.base_sha }} - head_sha: ${{ steps.pr.outputs.head_sha }} - closed: ${{ steps.pr.outputs.closed }} - merged: ${{ steps.pr.outputs.merged }} - dry_run: ${{ steps.pr.outputs.dry_run }} - steps: - - name: Find out about the pull request - id: pr - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EVENT: ${{ github.event_name }} - INPUT_NUMBER: ${{ inputs.pr_number }} - INPUT_DRY_RUN: ${{ inputs.dry_run }} - run: | - if [ "$EVENT" = "workflow_dispatch" ]; then - data=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$INPUT_NUMBER") - dry_run=$INPUT_DRY_RUN - else - data=$(gh api "repos/$GITHUB_REPOSITORY/pulls/${{ github.event.pull_request.number }}") - dry_run=false - fi - - # Forks never receive the App secrets, so a pull request from a fork gets no linked C# pull request - if [ "$(echo "$data" | jq -r '.head.repo.full_name')" != "$GITHUB_REPOSITORY" ]; then - echo "eligible=false" >> "$GITHUB_OUTPUT" - echo "The pull request is from a fork: no linked C# pull request." - exit 0 - fi - - { - echo "eligible=true" - echo "number=$(echo "$data" | jq -r '.number')" - echo "url=$(echo "$data" | jq -r '.html_url')" - echo "base_sha=$(echo "$data" | jq -r '.base.sha')" - echo "head_sha=$(echo "$data" | jq -r '.head.sha')" - echo "closed=$([ "$(echo "$data" | jq -r '.state')" = closed ] && echo true || echo false)" - echo "merged=$(echo "$data" | jq -r '.merged')" - echo "dry_run=$dry_run" - } >> "$GITHUB_OUTPUT" - - repos: - needs: prepare - if: needs.prepare.outputs.eligible == 'true' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.repos.outputs.matrix }} - any: ${{ steps.repos.outputs.any }} - steps: - # The tooling (py2cs, the patcher) comes from the pull request, so a fix to it works on the same pull request. The Python before the change - # and the list of repositories a C# pull request may be raised in come from the base branch. - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.base_sha }} - path: base - - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.head_sha }} - path: head - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Which repositories does the change touch? - id: repos - shell: bash - env: - PYTHONPATH: head/.ci/generation - run: | - repos=$(python -m py2cs repos --base-root base/src/bhom_schema --head-root head/src/bhom_schema --allowed-repos base/Converted-repositories.txt) - echo "Repositories: $repos" - echo "matrix=$(echo "$repos" | jq -c '{include: map({repo: ., name: (split("/")[1])})}')" >> "$GITHUB_OUTPUT" - echo "any=$([ "$repos" != "[]" ] && echo true || echo false)" >> "$GITHUB_OUTPUT" - - raise: - needs: [prepare, repos] - if: needs.prepare.outputs.closed == 'false' && needs.repos.outputs.any == 'true' + linked: + name: raise or update the C# draft runs-on: ubuntu-latest permissions: contents: read pull-requests: write - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.repos.outputs.matrix) }} + statuses: write steps: + # The repository itself, only to reach the shared action below - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.base_sha }} - path: base - - uses: actions/checkout@v4 + - name: Find the pull request and the C# repositories it touches + id: ctx + uses: ./.github/actions/linked-pr-context with: - ref: ${{ needs.prepare.outputs.head_sha }} - path: head + pr-number: ${{ inputs.pr_number }} - # A token for this one repository, valid for about an hour. It is made on a dry run too, which proves the App is installed. - - name: Make a token for ${{ matrix.repo }} only + # A token for the repositories the change touches only, valid for about an hour. It is made on a dry run too, which proves the App is installed. + - name: Make a token for the C# repositories touched id: token + if: steps.ctx.outputs.raise == 'true' uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.BHOM_APP_ID }} private-key: ${{ secrets.BHOM_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - repositories: ${{ matrix.name }} + repositories: ${{ steps.ctx.outputs.names }} - - name: Checkout the C# repository - uses: actions/checkout@v4 - with: - repository: ${{ matrix.repo }} - token: ${{ steps.token.outputs.token }} - path: csharp/${{ matrix.name }} - persist-credentials: false + - name: Clone the C# repositories + if: steps.ctx.outputs.raise == 'true' + shell: bash + env: + PY2CS_CSHARP_TOKEN: ${{ steps.token.outputs.token }} + REPOS: ${{ steps.ctx.outputs.repos }} + run: | + header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$PY2CS_CSHARP_TOKEN" | base64 -w0)" + echo "::add-mask::$header" + for repo in $(echo "$REPOS" | jq -r '.[]'); do + git -c "http.extraheader=$header" clone --depth 1 --quiet "https://github.com/$repo.git" "csharp/${repo#*/}" + done - uses: actions/setup-dotnet@v4 + if: steps.ctx.outputs.raise == 'true' with: dotnet-version: '8.0.x' - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Build the C# patcher + if: steps.ctx.outputs.raise == 'true' run: dotnet build head/.ci/generation/PythonSchemaGeneration.Patcher/PythonSchemaGeneration.Patcher.csproj -c Release --nologo - name: Plan the C# changes + if: steps.ctx.outputs.raise == 'true' shell: bash env: PYTHONPATH: head/.ci/generation PY2CS_PATCHER: head/.ci/generation/PythonSchemaGeneration.Patcher/bin/Release/net8.0/PythonSchemaGeneration.Patcher.dll + REPOS: ${{ steps.ctx.outputs.repos }} run: | - python -m py2cs plan \ - --base-root base/src/bhom_schema --head-root head/src/bhom_schema \ - --clones-parent csharp --allowed-repos base/Converted-repositories.txt \ - --only-repo "${{ matrix.repo }}" \ - --python-pr "${{ github.repository }}#${{ needs.prepare.outputs.number }}" \ - --out plan + for repo in $(echo "$REPOS" | jq -r '.[]'); do + python -m py2cs plan \ + --base-root base/src/bhom_schema --head-root head/src/bhom_schema \ + --clones-parent csharp --allowed-repos base/Converted-repositories.txt \ + --only-repo "$repo" \ + --python-pr "${{ github.repository }}#${{ steps.ctx.outputs.number }}" \ + --out "plan/${repo#*/}" + done # The tokens are passed to this step only, through the environment, and are never printed - name: Push the branch and open or update the draft pull request + if: steps.ctx.outputs.raise == 'true' shell: bash env: PYTHONPATH: head/.ci/generation PY2CS_CSHARP_TOKEN: ${{ steps.token.outputs.token }} PY2CS_PYTHON_TOKEN: ${{ github.token }} - DRY_RUN: ${{ needs.prepare.outputs.dry_run }} + DRY_RUN: ${{ inputs.dry_run }} + REPOS: ${{ steps.ctx.outputs.repos }} run: | flags="" if [ "$DRY_RUN" = "true" ]; then flags="--dry-run"; fi - python -m py2cs pr --plan-dir plan --repo "${{ matrix.repo }}" \ - --python-repo "${{ github.repository }}" --python-pr "${{ needs.prepare.outputs.number }}" \ - --python-url "${{ needs.prepare.outputs.url }}" $flags - - close: - needs: [prepare, repos] - if: needs.prepare.outputs.closed == 'true' && needs.repos.outputs.any == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.repos.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.base_sha }} - path: base - - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.head_sha }} - path: head - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Make a token for ${{ matrix.repo }} only - id: token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ secrets.BHOM_APP_ID }} - private-key: ${{ secrets.BHOM_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: ${{ matrix.name }} - - # Unmerged: the linked draft is closed with a comment. Merged: only a comment. - - name: Close or comment on the linked pull request - shell: bash - env: - PYTHONPATH: head/.ci/generation - PY2CS_CSHARP_TOKEN: ${{ steps.token.outputs.token }} - DRY_RUN: ${{ needs.prepare.outputs.dry_run }} - MERGED: ${{ needs.prepare.outputs.merged }} - run: | - flags="" - if [ "$DRY_RUN" = "true" ]; then flags="$flags --dry-run"; fi - if [ "$MERGED" = "true" ]; then flags="$flags --merged"; fi - python -m py2cs close --repo "${{ matrix.repo }}" --python-repo "${{ github.repository }}" \ - --python-pr "${{ needs.prepare.outputs.number }}" $flags - - # After the draft is raised or updated: the status of the Python PR is pending until the C# pull request is merged - status: - needs: [prepare, repos, raise] - # After a raise, or when the change needs no C# change at all - if: needs.raise.result == 'success' || (needs.repos.result == 'success' && needs.repos.outputs.any == 'false' && needs.prepare.outputs.closed == 'false') - runs-on: ubuntu-latest - permissions: - contents: read - statuses: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.base_sha }} - path: base - - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.head_sha }} - path: head - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + for repo in $(echo "$REPOS" | jq -r '.[]'); do + python -m py2cs pr --plan-dir "plan/${repo#*/}" --repo "$repo" \ + --python-repo "${{ github.repository }}" --python-pr "${{ steps.ctx.outputs.number }}" \ + --python-url "${{ steps.ctx.outputs.url }}" $flags + done + # After the draft is raised or updated, or when the change needs no C# change at all: pending until the C# pull request is merged - name: Set the commit status + if: steps.ctx.outputs.open == 'true' shell: bash env: PYTHONPATH: head/.ci/generation PY2CS_PYTHON_TOKEN: ${{ github.token }} - DRY_RUN: ${{ needs.prepare.outputs.dry_run }} + DRY_RUN: ${{ inputs.dry_run }} + RAISED: ${{ steps.ctx.outputs.raise }} run: | flags="" if [ "$DRY_RUN" = "true" ]; then flags="--dry-run"; fi - if [ "${{ needs.repos.outputs.any }}" = "false" ]; then flags="$flags --no-csharp-change"; fi - python -m py2cs status --python-repo "${{ github.repository }}" --python-pr "${{ needs.prepare.outputs.number }}" \ - --sha "${{ needs.prepare.outputs.head_sha }}" --repos-file base/Converted-repositories.txt $flags - - # Every hour: has the C# pull request of an open Python PR been merged (or closed) since? - # This only reads the C# repositories, which must be public, and writes statuses on this repository. - sweep: - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - statuses: write - steps: - # The default branch, so the tooling is the trusted one - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Update the status of every open Python PR that changes the classes - shell: bash - env: - GH_TOKEN: ${{ github.token }} - PYTHONPATH: .ci/generation - PY2CS_PYTHON_TOKEN: ${{ github.token }} - run: | - gh pr list --state open --limit 100 --json number,headRefOid,headRepositoryOwner,files \ - --jq '.[] | select(.headRepositoryOwner.login == "${{ github.repository_owner }}") | select(any(.files[]; .path | startswith("src/bhom_schema/"))) | "\(.number) \(.headRefOid)"' | - while read -r number sha; do - echo "Python PR #$number" - python -m py2cs status --python-repo "${{ github.repository }}" --python-pr "$number" --sha "$sha" --repos-file Converted-repositories.txt - done + if [ "$RAISED" != "true" ]; then flags="$flags --no-csharp-change"; fi + python -m py2cs status --python-repo "${{ github.repository }}" --python-pr "${{ steps.ctx.outputs.number }}" \ + --sha "$(git -C head rev-parse HEAD)" --repos-file base/Converted-repositories.txt $flags diff --git a/docs/Private-Repositories.md b/docs/Private-Repositories.md index cb79cb1..11c33e6 100644 --- a/docs/Private-Repositories.md +++ b/docs/Private-Repositories.md @@ -8,9 +8,9 @@ repositories, for example the ones in the BuroHappoldEngineering organisation, a | Part | Why it fails | |---|---| -| Hourly sweep in `linked-csharp-pr.yml` | It reads the C# pull requests with this repository's `GITHUB_TOKEN`, which can only read this repository and public repositories | +| Hourly sweep in `linked-csharp-pr-sweep.yml` | It reads the C# pull requests with this repository's `GITHUB_TOKEN`, which can only read this repository and public repositories | | `.github/scripts/Clone-Repositories.ps1` | It clones anonymously over HTTPS | -| The token step in `linked-csharp-pr.yml` | It passes `owner: ${{ github.repository_owner }}`, so it only works for C# repositories in the same organisation as this one. Repositories in another organisation fail even without the sweep | +| The token steps in `linked-csharp-pr.yml` and `linked-csharp-pr-close.yml` | It passes `owner: ${{ github.repository_owner }}`, so it only works for C# repositories in the same organisation as this one. Repositories in another organisation fail even without the sweep | | Public logs and artifacts | This repository is public, so its Actions logs and uploaded artifacts are public. The weekly generation uploads a report and a manifest that list class names, warnings and build output. Private code built or generated here would be exposed | | `src/bhom_schema` | Classes generated from a private repository and committed here would be public | @@ -69,7 +69,7 @@ BHoM/BHoM. These places assume public repositories in one organisation, and would change: -- `.github/workflows/linked-csharp-pr.yml`: `owner: ${{ github.repository_owner }}` in both token steps, and the `sweep` job. +- `.github/workflows/linked-csharp-pr.yml` and `linked-csharp-pr-close.yml`: `owner: ${{ github.repository_owner }}` in the token steps, and `linked-csharp-pr-sweep.yml`. - `.github/scripts/Clone-Repositories.ps1`: anonymous `git clone`. - `.github/workflows/python-schema-generation.yml`: uploads the report and manifest as a public artifact. - `Converted-repositories.txt` and `Included-repositories.txt`: one list each, for one deployment. diff --git a/docs/Testing-On-GitHub.md b/docs/Testing-On-GitHub.md index b8988d7..b0a73d7 100644 --- a/docs/Testing-On-GitHub.md +++ b/docs/Testing-On-GitHub.md @@ -131,7 +131,7 @@ Goal: a class that nothing else uses, written from scratch. |---|---| | **PR checks / checks** on the Python PR | Build, NUnit, pytest and Pyright pass. `py2cs check` passes | | **PR checks / C# alignment** on the Python PR | **Fails until the linked C# PR is merged**, with an explanation in the job summary: it compares the Python with what the C# `develop` generates. That failure is the proof that the two are not in step yet | -| **linked-csharp-pr**, `prepare`, `repos`, `raise` | Green. The log of `repos` names `BHoM/CSharpSchema_Test`; `raise` pushes `python-schema/pr-` and opens a **draft** PR | +| **Linked C# pull request / raise or update the C# draft** | Green. The step *Which repositories does the change touch?* names `BHoM/CSharpSchema_Test`; *Push the branch and open or update the draft pull request* pushes `python-schema/pr-` and opens a **draft** PR | | The C# draft PR | The files and lines in the table at the top of the exercise, a link back to the Python PR, labels, and nothing else changed. Open the diff and read it as a C# reviewer would. Check that it builds | | Commit status on the Python PR | `linked C# pull request`: pending while the draft is open | | The Python PR after a push | Pushing again updates the same branch and the same draft; no second PR | diff --git a/docs/Workflows.md b/docs/Workflows.md index 31b91fc..18e2033 100644 --- a/docs/Workflows.md +++ b/docs/Workflows.md @@ -1,6 +1,6 @@ # GitHub workflows -Four workflows run the loop between this repository and the repository that owns the C# classes, and make releases. For now that repository is +These workflows run the loop between this repository and the repository that owns the C# classes, and make releases. For now that repository is [BHoM/CSharpSchema_Test](https://github.com/BHoM/CSharpSchema_Test), a repository made for testing. **No other repository is ever written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built (read only) because the generator needs them. @@ -8,7 +8,9 @@ written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built |---|---|---| | [python-schema-generation.yml](../.github/workflows/python-schema-generation.yml) | Every Saturday at 4 AM UTC, and by hand | Clones and builds the C# repositories, generates `src/bhom_schema` from the C#, checks it, and commits it straight to `develop`. Not a pull request. If a check fails, nothing is committed | | [pr-checks.yml](../.github/workflows/pr-checks.yml) | Every pull request to `develop` | Two jobs. **checks**: installs the built assemblies of the repositories in `Included-repositories.txt` (the shared action `.github/actions/bhom-assemblies` caches them under the current commit of each repository, so unchanged repositories are not rebuilt), clones BHoM_JSONSchema read only as the test oracle for the JSON, runs `run-all.ps1` (build, NUnit, pytest, Pyright) and, for a PR that changes `src/bhom_schema`, `py2cs check`. **C# alignment (fails until the linked C# pull request is merged)**: for such a PR, builds the generator and runs `py2cs align`; when it fails it explains in the job summary that the linked C# pull request has to be merged first, then the job re-run | -| [linked-csharp-pr.yml](../.github/workflows/linked-csharp-pr.yml) | A pull request that changes `src/bhom_schema`, every hour, and by hand | Raises, updates or closes a linked **draft** pull request in the C# repository, and sets the commit status `linked C# pull request` on the Python PR | +| [linked-csharp-pr.yml](../.github/workflows/linked-csharp-pr.yml) | A pull request that changes `src/bhom_schema`, and by hand | One job, **raise or update the C# draft**: finds the C# repositories the change touches, raises or updates a linked **draft** pull request in each, and sets the commit status `linked C# pull request` on the Python PR. The shared action `.github/actions/linked-pr-context` does the finding | +| [linked-csharp-pr-close.yml](../.github/workflows/linked-csharp-pr-close.yml) | A Python pull request that changes `src/bhom_schema` is closed, and by hand | Closes the linked C# draft with a comment when the Python PR was closed unmerged, or only comments when it was merged. A separate workflow so that nothing of it shows on an open pull request | +| [linked-csharp-pr-sweep.yml](../.github/workflows/linked-csharp-pr-sweep.yml) | Every hour, and by hand | Refreshes the commit status `linked C# pull request` of every open Python PR that changes the classes. A schedule only, so nothing of it shows on a pull request | | [release.yml](../.github/workflows/release.yml) | By hand only (dry run by default) | Regenerates with `--release X.Y`, checks, builds the wheel and the release notes; when not a dry run, pushes branch `vX.Y` and tag `vX.Y.0`, creates the GitHub Release and publishes to TestPyPI. See "Making a release" | ## How they fit together @@ -33,12 +35,13 @@ written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built │ pr-checks.yml │ │ linked-csharp-pr.yml │ │ on every PR to develop │ │ on a PR that changes src/bhom_schema │ │ build, NUnit, pytest, │ │ │ -│ Pyright; py2cs check + align │ │ prepare ─► repos ─► raise ─► status │ -│ (writes nothing) │ │ (draft C# PR, and the commit │ -└──────────────────────────────┘ │ status on the Python PR) │ - │ PR closed: prepare ─► repos ─► close │ - │ every hour: sweep (refreshes the status of │ - │ every open Python PR) │ +│ Pyright; py2cs check; │ │ one job: raise or update the C# draft, and set │ +│ C# alignment (own job) │ │ the commit status on the Python PR │ +│ (writes nothing) │ │ │ +└──────────────────────────────┘ │ PR closed: linked-csharp-pr-close.yml │ + │ every hour: linked-csharp-pr-sweep.yml │ + │ (refreshes the status of every │ + │ open Python PR) │ └─────────────────────┬───────────────────────────┘ │ the C# draft is reviewed and merged in C# ▼ @@ -127,7 +130,7 @@ Real PyPI is added the same way when BHoM/BHoM is in scope. The branch `v10.0` i ## The linked pull request status Nothing tells this repository when a C# pull request is merged, so `linked-csharp-pr.yml` also sets a commit status on the Python PR -and refreshes it every hour: +and `linked-csharp-pr-sweep.yml` refreshes it every hour: | Status | Meaning | |---|---|