From aff1d763f00a8992664303b5080de3b3bd7514a2 Mon Sep 17 00:00:00 2001 From: Duncan Leo Date: Sat, 14 Nov 2026 10:55:08 -0800 Subject: [PATCH 1/2] feat: open review PRs for new GTFS snapshots Co-Authored-By: Codex --- .github/workflows/gtfs-capture.yml | 86 +++++++++++++++++++++++++++++- docs/gtfs-public-handoff.md | 21 +++++--- 2 files changed, 97 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gtfs-capture.yml b/.github/workflows/gtfs-capture.yml index f0d01c1c..de94ffd4 100644 --- a/.github/workflows/gtfs-capture.yml +++ b/.github/workflows/gtfs-capture.yml @@ -1,4 +1,4 @@ -name: Inspect public GTFS schedule capture +name: Inspect and open review for public GTFS schedule capture on: repository_dispatch: @@ -16,8 +16,16 @@ permissions: jobs: inspect: runs-on: ubuntu-latest + outputs: + sha256: ${{ steps.acquire.outputs.sha256 }} + retrieved-at: ${{ steps.acquire.outputs.retrieved-at }} + independent-outcome: ${{ steps.validate.outcome }} + inspection-outcome: ${{ steps.report.outcome }} + artifact-url: ${{ steps.artifact.outputs.artifact-url }} steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: actions/setup-node@v6 with: node-version: 24 @@ -30,7 +38,10 @@ jobs: GTFS_SNAPSHOT_BASE_URL: ${{ vars.GTFS_SNAPSHOT_BASE_URL }} GTFS_CAPTURE_KEY: ${{ github.event.client_payload.capture_key || inputs.capture_key }} GTFS_INPUT_DIR: ${{ runner.temp }}/gtfs-input - run: node scripts/acquire-gtfs-capture.mjs + run: | + node scripts/acquire-gtfs-capture.mjs + jq -r '"sha256=\(.sha256)", "retrieved-at=\(.retrieved_at)"' \ + "$GTFS_INPUT_DIR/capture.json" >> "$GITHUB_OUTPUT" - uses: actions/setup-java@v5 with: distribution: temurin @@ -40,6 +51,7 @@ jobs: url=$(node -p "JSON.parse(require('fs').readFileSync('scripts/gtfs-validator-lock.json')).url") curl --fail --location --max-time 120 "$url" -o "$RUNNER_TEMP/gtfs-validator.jar" - name: Independently validate verified snapshot + id: validate if: ${{ !cancelled() && steps.acquire.outcome == 'success' }} run: >- node scripts/validate-gtfs-independent.mjs @@ -47,6 +59,7 @@ jobs: "$RUNNER_TEMP/gtfs-input/schedule.zip" "$RUNNER_TEMP/gtfs-independent" - name: Inspect verified local snapshot + id: report if: ${{ !cancelled() && steps.acquire.outcome == 'success' }} run: >- node packages/cli/dist/index.js gtfs inspect @@ -54,6 +67,7 @@ jobs: --capture "$RUNNER_TEMP/gtfs-input/capture.json" --write "$RUNNER_TEMP/gtfs-report.json" - uses: actions/upload-artifact@v7 + id: artifact if: always() with: name: gtfs-inspection-${{ github.run_id }} @@ -61,3 +75,71 @@ jobs: ${{ runner.temp }}/gtfs-report.json ${{ runner.temp }}/gtfs-independent/ if-no-files-found: ignore + + review: + needs: inspect + if: >- + ${{ !cancelled() && needs.inspect.outputs.sha256 != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + concurrency: + group: gtfs-review-${{ needs.inspect.outputs.sha256 }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.repository.default_branch }} + - name: Open snapshot review + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ github.event.repository.default_branch }} + SNAPSHOT_ORIGIN: ${{ vars.GTFS_SNAPSHOT_BASE_URL }} + SHA256: ${{ needs.inspect.outputs.sha256 }} + RETRIEVED_AT: ${{ needs.inspect.outputs.retrieved-at }} + INDEPENDENT_OUTCOME: ${{ needs.inspect.outputs.independent-outcome }} + INSPECTION_OUTCOME: ${{ needs.inspect.outputs.inspection-outcome }} + ARTIFACT_URL: ${{ needs.inspect.outputs.artifact-url }} + run: | + branch="gtfs-review/$SHA256" + ref="refs/heads/$branch" + existing_pr=$(gh api --method GET "repos/$GITHUB_REPOSITORY/pulls" \ + -f state=all -f head="$GITHUB_REPOSITORY_OWNER:$branch" \ + --jq '.[0].html_url // empty') + if [[ -n "$existing_pr" ]]; then + echo "Snapshot review already exists at $existing_pr" + exit 0 + fi + + title="chore(gtfs): review schedule snapshot ${SHA256:0:12}" + remote_ref=$(git ls-remote --refs origin "$ref") + if [[ -z "$remote_ref" ]]; then + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + origin="${SNAPSHOT_ORIGIN%/}" + file="docs/gtfs-captures/$SHA256.md" + mkdir -p docs/gtfs-captures + cat > "$file" <> "$file" + fi + git add -- "$file" + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -m "$title" -m 'Co-Authored-By: Codex ' + git push origin "HEAD:$ref" + fi + + gh pr create --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" \ + --head "$branch" --title "$title" \ + --body "Review the new LTA train GTFS capture. The added record links the source ZIP and inspection results. Check validation findings, service-date coverage, and mapping changes before preparing a timetable release." diff --git a/docs/gtfs-public-handoff.md b/docs/gtfs-public-handoff.md index f00ee0cf..d8524860 100644 --- a/docs/gtfs-public-handoff.md +++ b/docs/gtfs-public-handoff.md @@ -48,8 +48,8 @@ Only schedule hashes trigger GitHub `repository_dispatch` event `gtfs_capture`: The crawler records notification state only in private Tigris. Failed notifications remain pending and are retried on subsequent enabled DataMall polling runs. A bounded batch of 20 avoids unbounded catch-up. Successful hashes are not -notified again. A crash after dispatch can cause duplicate delivery; the receiver -only inspects and uploads a report, so replay performs no canonical writes. +notified again. A crash after dispatch can cause duplicate delivery. The +receiver may repeat inspection, but opens at most one review PR per ZIP hash. Published alerts/trip updates do not dispatch to the schedule workflow. ## Provisioning @@ -75,12 +75,17 @@ then downloads the ZIP without following redirects. Transfers have byte limits and request timeouts. Both files are hash/size verified before materialization in a fresh temporary directory. -Acquisition is in the CLI layer, separate from the offline `@mrtdown/fs` -inspector. The workflow invokes that inspector and retains its JSON report as -a GitHub Actions artifact, including reports with structural errors. It has -read-only repository permissions. It does not commit, merge, run models or -publish a scheduled-arrivals artifact. Trusted delivery does not replace -independent GTFS validation or reviewed canonical mappings. +The read-only inspection job runs the independent validator and the offline +`@mrtdown/fs` inspector and uploads their available reports as a GitHub Actions +artifact, including findings with errors. + +After the receipt and ZIP are verified, a separate job on the default branch +opens a review PR for that ZIP hash even if validation reports errors. The PR +adds `docs/gtfs-captures/.md` with the capture time, source links, +validation outcomes, and links to the run and available reports. Existing PRs, +including closed ones, are reused on replay. The job uses the repository's +GitHub Actions permission to create PRs. Canonical mapping review and timetable +release approval remain separate steps. For a local smoke test after publication: From 6812354e3a2eb2f969446057464cccb0ecfcb35c Mon Sep 17 00:00:00 2001 From: Duncan Leo Date: Sat, 14 Nov 2026 11:11:55 -0800 Subject: [PATCH 2/2] fix: preserve GTFS review artifacts across reruns Co-Authored-By: Codex --- .github/workflows/gtfs-capture.yml | 42 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/gtfs-capture.yml b/.github/workflows/gtfs-capture.yml index de94ffd4..b9bd2144 100644 --- a/.github/workflows/gtfs-capture.yml +++ b/.github/workflows/gtfs-capture.yml @@ -1,4 +1,4 @@ -name: Inspect and open review for public GTFS schedule capture +name: Inspect public GTFS schedule capture on: repository_dispatch: @@ -70,7 +70,8 @@ jobs: id: artifact if: always() with: - name: gtfs-inspection-${{ github.run_id }} + # Reruns keep their own reports and preserve links from earlier attempts. + name: gtfs-inspection-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ runner.temp }}/gtfs-report.json ${{ runner.temp }}/gtfs-independent/ @@ -97,49 +98,50 @@ jobs: GH_TOKEN: ${{ github.token }} BASE_BRANCH: ${{ github.event.repository.default_branch }} SNAPSHOT_ORIGIN: ${{ vars.GTFS_SNAPSHOT_BASE_URL }} - SHA256: ${{ needs.inspect.outputs.sha256 }} + SNAPSHOT_SHA256: ${{ needs.inspect.outputs.sha256 }} RETRIEVED_AT: ${{ needs.inspect.outputs.retrieved-at }} INDEPENDENT_OUTCOME: ${{ needs.inspect.outputs.independent-outcome }} INSPECTION_OUTCOME: ${{ needs.inspect.outputs.inspection-outcome }} ARTIFACT_URL: ${{ needs.inspect.outputs.artifact-url }} run: | - branch="gtfs-review/$SHA256" - ref="refs/heads/$branch" + review_branch="gtfs-review/$SNAPSHOT_SHA256" + review_ref="refs/heads/$review_branch" existing_pr=$(gh api --method GET "repos/$GITHUB_REPOSITORY/pulls" \ - -f state=all -f head="$GITHUB_REPOSITORY_OWNER:$branch" \ + -f state=all -f head="$GITHUB_REPOSITORY_OWNER:$review_branch" \ --jq '.[0].html_url // empty') if [[ -n "$existing_pr" ]]; then echo "Snapshot review already exists at $existing_pr" exit 0 fi - title="chore(gtfs): review schedule snapshot ${SHA256:0:12}" - remote_ref=$(git ls-remote --refs origin "$ref") - if [[ -z "$remote_ref" ]]; then + title="chore(gtfs): review schedule snapshot ${SNAPSHOT_SHA256:0:12}" + # A previous attempt may have pushed the branch before PR creation failed. + existing_ref=$(git ls-remote --refs origin "$review_ref") + if [[ -z "$existing_ref" ]]; then run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" origin="${SNAPSHOT_ORIGIN%/}" - file="docs/gtfs-captures/$SHA256.md" + record_path="docs/gtfs-captures/$SNAPSHOT_SHA256.md" mkdir -p docs/gtfs-captures - cat > "$file" < "$record_path" <> "$file" + printf '%s\n' "- [Download available reports]($ARTIFACT_URL)" >> "$record_path" fi - git add -- "$file" + git add -- "$record_path" git -c user.name='github-actions[bot]' \ -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ commit -m "$title" -m 'Co-Authored-By: Codex ' - git push origin "HEAD:$ref" + git push --force-with-lease="$review_ref:" origin "HEAD:$review_ref" fi gh pr create --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" \ - --head "$branch" --title "$title" \ + --head "$review_branch" --title "$title" \ --body "Review the new LTA train GTFS capture. The added record links the source ZIP and inspection results. Check validation findings, service-date coverage, and mapping changes before preparing a timetable release."