From 8700ef5a9782474aa04733a25bf119956036f209 Mon Sep 17 00:00:00 2001 From: liveforpresent Date: Tue, 27 Oct 2026 23:48:18 +0900 Subject: [PATCH] feat(ci): link snapshot PRs to candidate verification evidence --- .github/workflows/api-smoke.yml | 27 +- .github/workflows/contract-evidence.yml | 92 ++++++ .github/workflows/contract-propagation.yml | 62 ++-- .github/workflows/contract-sync.yml | 28 +- docs/CONTRACT_PROPAGATION.md | 162 +++++++++- docs/CONTRACT_PROPAGATION_SETTINGS.md | 54 ++++ scripts/contract_evidence.py | 335 +++++++++++++++++++++ scripts/publish_contract_sync.py | 97 ++++-- scripts/test_contract_evidence.py | 333 ++++++++++++++++++++ scripts/test_contract_workflows.py | 54 ++++ scripts/test_publish_contract_sync.py | 175 +++++++++++ 11 files changed, 1342 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/contract-evidence.yml create mode 100644 docs/CONTRACT_PROPAGATION_SETTINGS.md create mode 100644 scripts/contract_evidence.py create mode 100644 scripts/test_contract_evidence.py create mode 100644 scripts/test_contract_workflows.py create mode 100644 scripts/test_publish_contract_sync.py diff --git a/.github/workflows/api-smoke.yml b/.github/workflows/api-smoke.yml index 0591ee7..429dbed 100644 --- a/.github/workflows/api-smoke.yml +++ b/.github/workflows/api-smoke.yml @@ -1,6 +1,26 @@ name: API smoke on: + workflow_call: + inputs: + context_ref: + required: true + type: string + core_ref: + required: true + type: string + compute_ref: + required: true + type: string + web_ref: + required: true + type: string + integration_ref: + required: true + type: string + secrets: + REFINVEST_CI_APP_PRIVATE_KEY: + required: true pull_request: push: branches: @@ -27,7 +47,7 @@ permissions: contents: read concurrency: - group: api-smoke-${{ github.workflow }}-${{ inputs.context_ref || github.ref }} + group: api-smoke-${{ github.workflow }}-${{ github.event_name }}-${{ inputs.context_ref && github.run_id || github.ref }} cancel-in-progress: true jobs: @@ -35,6 +55,7 @@ jobs: name: Core to Compute API smoke and Web browser acceptance if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false runs-on: ubuntu-latest + timeout-minutes: 45 services: postgres: image: postgres:17-alpine @@ -173,7 +194,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: api-smoke-and-web-acceptance-compatibility-evidence + name: api-smoke-and-web-acceptance-compatibility-evidence-${{ github.run_attempt }} path: integration/.integration-data/evidence/compatibility.json if-no-files-found: warn @@ -181,6 +202,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: api-smoke-and-web-acceptance-logs + name: api-smoke-and-web-acceptance-logs-${{ github.run_attempt }} path: integration/.integration-data/logs if-no-files-found: warn diff --git a/.github/workflows/contract-evidence.yml b/.github/workflows/contract-evidence.yml new file mode 100644 index 0000000..f314d01 --- /dev/null +++ b/.github/workflows/contract-evidence.yml @@ -0,0 +1,92 @@ +name: Record candidate contract evidence + +on: + workflow_run: + workflows: [Propagate Context snapshots] + types: [completed] + +permissions: + contents: read + actions: read + +concurrency: + group: contract-evidence-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + record: + if: >- + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.repository.full_name == github.repository && + (github.event.workflow_run.event == 'workflow_dispatch' || + github.event.workflow_run.event == 'repository_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # Never check out the candidate or execute files from downloaded artifacts. + - name: Checkout trusted default branch + uses: actions/checkout@v4 + with: + ref: refs/heads/main + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Create read-only Context token + id: context-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.REFINVEST_CI_APP_CLIENT_ID }} + private-key: ${{ secrets.REFINVEST_CI_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: context + permission-contents: read + + - name: Create evidence PR token + id: pr-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.REFINVEST_CI_APP_CLIENT_ID }} + private-key: ${{ secrets.REFINVEST_CI_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + integration + core-api + compute-api + web + permission-contents: read + permission-pull-requests: write + + - name: Collect completed jobs and link PR evidence + env: + GH_TOKEN: ${{ steps.pr-token.outputs.token }} + GH_CONTEXT_TOKEN: ${{ steps.context-token.outputs.token }} + GH_ACTIONS_TOKEN: ${{ github.token }} + SOURCE_REPOSITORY: ${{ github.repository }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + SOURCE_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + CONTRACT_REPORT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + python -B scripts/contract_evidence.py \ + --repository "$SOURCE_REPOSITORY" --run-id "$SOURCE_RUN_ID" \ + --attempt "$SOURCE_ATTEMPT" --output "$RUNNER_TEMP/contract-evidence" + + - name: Publish evidence summary + if: always() + run: | + if [ -f "$RUNNER_TEMP/contract-evidence/contract-candidate-evidence.md" ]; then + cat "$RUNNER_TEMP/contract-evidence/contract-candidate-evidence.md" >> "$GITHUB_STEP_SUMMARY" + else + echo '후보 검증 증거를 수집하지 못했습니다. 성공으로 간주하지 말고 이 실행의 로그를 확인하세요.' >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload final candidate evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: contract-candidate-evidence-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + path: ${{ runner.temp }}/contract-evidence/ + if-no-files-found: warn diff --git a/.github/workflows/contract-propagation.yml b/.github/workflows/contract-propagation.yml index 94f04cb..cd4729d 100644 --- a/.github/workflows/contract-propagation.yml +++ b/.github/workflows/contract-propagation.yml @@ -19,7 +19,15 @@ concurrency: jobs: propagate: + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + context_ref: ${{ steps.propagate.outputs.context_ref }} + core_ref: ${{ steps.propagate.outputs.core_ref }} + compute_ref: ${{ steps.propagate.outputs.compute_ref }} + web_ref: ${{ steps.propagate.outputs.web_ref }} + integration_ref: ${{ steps.propagate.outputs.integration_ref }} steps: - name: Create read-only Context token id: context-token @@ -45,7 +53,6 @@ jobs: web permission-contents: write permission-pull-requests: write - permission-actions: write - name: Checkout integration uses: actions/checkout@v4 @@ -53,6 +60,7 @@ jobs: repository: ${{ github.repository_owner }}/integration token: ${{ steps.propagation-token.outputs.token }} path: integration + ref: ${{ github.sha }} fetch-depth: 0 - name: Checkout Context @@ -109,35 +117,37 @@ jobs: echo "compute_ref=$(jq -r '.heads.computeApi' "$RUNNER_TEMP/contract-propagation.json")" >> "$GITHUB_OUTPUT" echo "web_ref=$(jq -r '.heads.web' "$RUNNER_TEMP/contract-propagation.json")" >> "$GITHUB_OUTPUT" echo "integration_ref=$(jq -r '.heads.integration' "$RUNNER_TEMP/contract-propagation.json")" >> "$GITHUB_OUTPUT" - echo "run_api_smoke=$(jq -r '.runApiSmoke' "$RUNNER_TEMP/contract-propagation.json")" >> "$GITHUB_OUTPUT" - - - name: Dispatch candidate contract verification - env: - GH_TOKEN: ${{ steps.propagation-token.outputs.token }} - run: | - gh workflow run contract-sync.yml --repo '${{ github.repository_owner }}/integration' \ - -f context_ref='${{ steps.propagate.outputs.context_ref }}' \ - -f core_ref='${{ steps.propagate.outputs.core_ref }}' \ - -f compute_ref='${{ steps.propagate.outputs.compute_ref }}' \ - -f web_ref='${{ steps.propagate.outputs.web_ref }}' \ - -f integration_ref='${{ steps.propagate.outputs.integration_ref }}' - - - name: Dispatch candidate API smoke - if: steps.propagate.outputs.run_api_smoke == 'true' - env: - GH_TOKEN: ${{ steps.propagation-token.outputs.token }} - run: | - gh workflow run api-smoke.yml --repo '${{ github.repository_owner }}/integration' \ - -f context_ref='${{ steps.propagate.outputs.context_ref }}' \ - -f core_ref='${{ steps.propagate.outputs.core_ref }}' \ - -f compute_ref='${{ steps.propagate.outputs.compute_ref }}' \ - -f web_ref='${{ steps.propagate.outputs.web_ref }}' \ - -f integration_ref='${{ steps.propagate.outputs.integration_ref }}' - name: Upload propagation result if: always() uses: actions/upload-artifact@v4 with: - name: contract-propagation-result + name: contract-propagation-result-${{ github.run_attempt }} path: ${{ runner.temp }}/contract-propagation.json if-no-files-found: ignore + + candidate-sync: + name: candidate-sync + needs: propagate + uses: ./.github/workflows/contract-sync.yml + with: + context_ref: ${{ needs.propagate.outputs.context_ref }} + core_ref: ${{ needs.propagate.outputs.core_ref }} + compute_ref: ${{ needs.propagate.outputs.compute_ref }} + web_ref: ${{ needs.propagate.outputs.web_ref }} + integration_ref: ${{ needs.propagate.outputs.integration_ref }} + secrets: + REFINVEST_CI_APP_PRIVATE_KEY: ${{ secrets.REFINVEST_CI_APP_PRIVATE_KEY }} + + candidate-smoke: + name: candidate-smoke + needs: [propagate, candidate-sync] + uses: ./.github/workflows/api-smoke.yml + with: + context_ref: ${{ needs.propagate.outputs.context_ref }} + core_ref: ${{ needs.propagate.outputs.core_ref }} + compute_ref: ${{ needs.propagate.outputs.compute_ref }} + web_ref: ${{ needs.propagate.outputs.web_ref }} + integration_ref: ${{ needs.propagate.outputs.integration_ref }} + secrets: + REFINVEST_CI_APP_PRIVATE_KEY: ${{ secrets.REFINVEST_CI_APP_PRIVATE_KEY }} diff --git a/.github/workflows/contract-sync.yml b/.github/workflows/contract-sync.yml index 3405864..9f77549 100644 --- a/.github/workflows/contract-sync.yml +++ b/.github/workflows/contract-sync.yml @@ -1,6 +1,26 @@ name: Contract sync on: + workflow_call: + inputs: + context_ref: + required: true + type: string + core_ref: + required: true + type: string + compute_ref: + required: true + type: string + web_ref: + required: true + type: string + integration_ref: + required: true + type: string + secrets: + REFINVEST_CI_APP_PRIVATE_KEY: + required: true pull_request: push: branches: @@ -33,6 +53,7 @@ jobs: name: Verify Context/Core/Compute/Web snapshots if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Checkout integration uses: actions/checkout@v4 @@ -95,6 +116,11 @@ jobs: - name: Install Analytics contract verifier dependencies run: python -m pip install -r integration/scripts/requirements-contract-sync.txt + - name: Test propagation policy and evidence binding + run: | + python -B -m unittest discover -s integration/scripts -p 'test_contract_*.py' + python -B -m unittest discover -s integration/scripts -p 'test_publish_contract_sync.py' + - name: Verify OpenAPI snapshots id: contract-sync shell: pwsh @@ -146,6 +172,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: contract-sync-compatibility-evidence + name: contract-sync-compatibility-evidence-${{ github.run_attempt }} path: integration/.integration-data/evidence/compatibility.json if-no-files-found: warn diff --git a/docs/CONTRACT_PROPAGATION.md b/docs/CONTRACT_PROPAGATION.md index a4cdbf5..5b3fc07 100644 --- a/docs/CONTRACT_PROPAGATION.md +++ b/docs/CONTRACT_PROPAGATION.md @@ -57,18 +57,147 @@ impact plan alone as an E2E pass result. ## GitHub Actions rollout -The integration `contract-propagation.yml` workflow accepts a merged Context -SHA by `repository_dispatch` or manual dispatch. It opens or updates one -`automation/context-snapshots` PR per affected repository. It never merges a -PR or edits product code. It dispatches `contract-sync.yml` with the exact -candidate commit tuple, and also dispatches `api-smoke.yml` when OpenAPI or -protocol fixture snapshots change. The two verification workflows accept -explicit refs for all five checkouts. Their artifacts record tested commits. +integration `contract-propagation.yml`은 `main`에서만 실행하며, +`repository_dispatch` 또는 수동 dispatch로 **현재 Context main의 전체 40자리 SHA**를 +받는다. 과거 main의 조상 SHA도 현재 main과 다르면 거부한다. 정보성 파일만 바뀌어도 +새 Context SHA로 다시 검증하는 보수적인 정책이다. + +영향 저장소마다 `automation/context-snapshots` PR을 생성·갱신한다. 전체 후보 조합을 +먼저 확정한 뒤 PR 본문에 Context SHA, 해당 PR head, 변경 없는 저장소를 포함한 +Core/Compute/Web/integration SHA, 관련 PR, 변경 파일과 담당자 체크리스트를 기록한다. +제품 코드, 자동 승인, 자동 병합, 보호 규칙 우회 또는 LLM 병합 판정은 포함하지 않는다. + +검증은 별도 실행을 dispatch하고 끝내지 않고, 같은 전파 실행의 재사용 워크플로 job으로 +연결한다. `candidate-sync` 성공 후 `candidate-smoke`를 실행한다. 두 job 모두 다섯 +저장소를 **고정된 후보 SHA**로 checkout한다. 규범 문서만 변경되거나 스냅샷 PR이 +없는 경우에도 동일하게 검사한다. E2E 시작 전 계약·문서·fixture 검사는 유지한다. +일반 PR/push/schedule 및 개별 수동 검증 경로도 유지한다. + +## Evidence and outcomes + +| 결과물 | 내용 | +| --- | --- | +| 원래 전파 실행의 `contract-propagation-result-` | `contract-propagation.json`: 원본 Context, 네 후보 head, 네 base main, 실제 PR 번호/head, 변경 파일, 실행 ID/attempt, 워크플로 소스 SHA. 처음 상태는 `pending`이다. | +| 각 검증 job의 `*-compatibility-evidence-` | `compatibility.json`: 실제 checkout SHA와 worktree, 계약 hash, 검사 단계의 outcome. 단계 증거만으로 전체 job 성공을 뜻하지 않는다. | +| 완료 관찰 실행의 `contract-candidate-evidence--` | 최종 JSON과 Markdown: 전체 후보, 실제 checkout 증거, job 링크와 conclusion, 전체 outcome, 최신성 조회, PR 갱신 결과. 실행 Summary와 PR에서 연결한다. | + +`contract-evidence.yml`은 전파 실행이 종료된 뒤 `workflow_run: completed`로 별도 실행된다. +취소·시간 초과로 원래 실행의 마지막 단계가 사라져도 완료 job의 GitHub API conclusion을 +조회한다. 종료된 **동일 run ID/attempt**의 artifact만 사용한다. 기본 브랜치의 신뢰된 +코드만 실행하며, 다운로드한 ZIP에서는 정해진 JSON만 읽고 압축 파일이나 후보 코드를 +실행하지 않는다. Context 읽기, PR 본문 쓰기, Actions artifact 읽기 토큰은 분리한다. + +| Outcome | 해석 | +| --- | --- | +| `pending` | 시작·대기·실행 중. 검증 성공이 아니다. | +| `success` | 전파 실행과 두 검증 job이 모두 완료·성공했고, 두 artifact의 성공 증거와 실제 다섯 checkout SHA 및 clean worktree가 후보와 일치한다. 의미상 승인이나 병합 허가가 아니다. | +| `failure` | 실패·skipped·job 식별 실패·성공 증거 누락 또는 불일치. 각 check의 원래 conclusion과 오류를 함께 확인한다. | +| `cancelled` | 실행 또는 필수 job이 취소됨. 앞선 단계의 성공 증거로 대체하지 않는다. | +| `timed_out` | GitHub가 필수 job 또는 실행을 시간 초과로 종료함. 일반 실패/취소와 구분한다. | + +최신성은 별도 필드다. `current`는 **완료 관찰 시점**에 Context main, 각 base main, +모든 열린 PR head/base가 후보와 일치했다는 뜻이다. `stale`이면 과거 성공을 재사용하지 +않는다. API 조회 실패는 `unconfirmed`이며 성공으로 취급하지 않는다. 두 job 중 하나가 +시간 초과이고 다른 job이 취소된 경우 전체 outcome은 `timed_out`이며 개별 conclusion도 남긴다. + +manifest가 만들어지기 전 취소·실패했거나 artifact가 만료/누락되면 후보 증거는 미확인이다. +이때도 완료 job을 조회할 수 있으면 종료 결과를 최종 JSON/Markdown에 남기고 후보 SHA는 +`null`, `tupleAvailable: false`로 표시한다. 관찰 워크플로는 실패로 종료하며 PR을 갱신하지 않는다. +관찰 실행의 실패 Summary/로그를 확인하고 새 전파를 시작한다. PR의 `pending`이 계속 +남아도 통과가 아니다. 원래 실행의 초록 표시만으로 관찰 실행의 성공과 최신성까지 +확인했다고 해석하지 않는다. 관찰 시점 이후 변경을 상시 감시하거나 PR 병합을 +서버에서 차단하는 구현은 아니다. + +재검증은 현재 Context main SHA로 **새 전파 실행**을 시작하는 것이 원칙이다. +GitHub의 실패 job만 재실행하면 이전 attempt의 manifest를 재사용하지 않으므로 증거가 +미확인으로 남을 수 있다. 모든 job을 재실행하거나 새 전파를 수행한다. 늦게 끝난 관찰자가 +새 전파의 PR 본문을 덮어쓰지 않도록 run/attempt 본문 표식과 PR head를 다시 확인한다. + +## Maintainer merge procedure + +1. 관련 PR을 하나의 변경 묶음으로 모은다. Context 원본 diff, 각 스냅샷 PR과 구현 PR, + 변경 없는 저장소 SHA도 함께 검토한다. 문서만 Core에 복사되어도 Compute/Web 의미에 + 영향이 있을 수 있다. OpenAPI·규범 문서·fixture를 파일 종류만으로 자동 승인하지 않는다. +2. 담당자가 의미 변경(호환성, 요청/응답, 상태·오류, persistence/dispatch, 버전·hash, + fixture/테스트 범위)을 검토한다. 추가 구현이 필요하면 각 서비스 소유 저장소에서 + 리뷰한다. 계약·제품 규칙 변경은 Context 정본부터 다룬다. 봇 브랜치에 구현을 섞지 않는다. +3. 구현이 필요한 경우 기존/새 계약 모두와 호환되는 구현 PR을 먼저 서비스 main에 반영하고 + 다시 전파한다. 단일 안전한 병합 순서가 없는 breaking change는 병합을 중단하고 Context에 + 버전 분리·점진적 전환 방식을 먼저 마련한다. smoke 성공만으로 의미 검토를 생략하지 않는다. +4. 병합 **직전** PR API의 `head.sha`, `base.ref`, Context `main` SHA, 네 서비스/integration + `main` SHA를 최종 JSON의 `heads`, `contextRevision`, `bases`와 대조한다. 예: + + ```powershell + gh api repos/refinvest-dev/core-api/pulls/ --jq '{head: .head.sha, base: .base.ref, state: .state}' + gh api repos/refinvest-dev/context/git/ref/heads/main --jq .object.sha + gh api repos/refinvest-dev/core-api/git/ref/heads/main --jq .object.sha + # compute-api, web, integration의 main 및 변경 묶음의 나머지 PR도 같은 방식으로 조회한다. + ``` + + head, base main, Context가 달라지면 기존 성공 증거를 재사용하지 않는다. rebase, main 병합, + 충돌 해결, 새 구현 커밋 모두 새 후보 SHA다. Context는 관련 내용뿐 아니라 어떤 새 커밋이든 + 재전파한다. API를 조회할 수 없거나 artifact가 없으면 확인되지 않은 상태로 중단한다. +5. 두 필수 후보 검증의 최종 `success`, 최신성 `current`, 동일 run/attempt와 실제 checkout + SHA를 확인한다. 저장소 자체 CI·필수 리뷰·보호 규칙도 별도로 충족해야 한다. 모든 검토 + 항목과 중간 상태 호환성·복구 계획을 PR에 기록한 후 담당자가 **수동 병합**한다. + +## Multi-repository merge and recovery + +변경 묶음의 대표 PR에 아래 기록을 남긴다. PR 본문은 봇이 갱신하므로 담당자의 상세 +검토·체크리스트 완료 기록은 별도 PR 댓글 또는 연결한 문서에도 보관한다. + +| 항목 | 담당자가 기록할 내용 | +| --- | --- | +| 변경 묶음 | Context SHA, 모든 스냅샷/구현 PR, 다섯 후보 SHA, run/attempt와 최종 증거 링크 | +| 순서와 중간 상태 | 각 단계의 old/new Core·Compute·Web 조합, 배포 영향, API/규범 의미 호환성, 확인 책임자 | +| 각 병합 | 직전 검증 링크, 실제 merge commit SHA, 시각, 남은 PR과 main SHA | +| 실패·복구 | 실패 로그/DB 근거, 중단 시점, 이미 병합한 범위, revert 또는 후속 수정 PR, 복구 검증 링크 | + +기본 순서는 선행 호환 구현 → Compute/Core(의존성에 따라 순서를 명시) → Web → integration +fixture다. 구현 의존성에 따라 다른 순서를 선택할 수 있지만 **각 중간 상태의 호환성**을 먼저 +증명해야 한다. GitHub의 여러 저장소 병합은 원자적이지 않다. 전체 후보 조합 검증 성공은 +중간 main 조합이나 실제 배포 순서의 성공 증거가 아니다. + +하나를 병합하면 base/main 조합이 바뀐다. 실제 merge/squash SHA와 남은 PR head를 포함하도록 +새 전파를 시작하고 다시 후보 검증을 완료한 후 다음 PR을 수동 병합한다. 앞선 묶음의 성공을 +새 main SHA에 그대로 붙이지 않는다. 이 규칙은 일부 저장소만 변경된 경우에도 같다. + +부분 병합·push·PR 갱신·후속 CI 실패 또는 예상 밖 drift가 발생하면 남은 병합을 즉시 +중단한다. 이미 성공한 병합을 전체 성공으로 기록하지 않는다. 원래 run/attempt artifact, +서비스 로그와 integration DB 상태, 실제 각 main/PR SHA를 보존한다. 이미 병합한 변경이 +호환되면 담당자 검토를 거친 후속 수정 PR로 전진 복구하고, 그렇지 않으면 역순 revert PR을 +검토한다. 자동 revert·강제 push·보호 규칙 우회를 수행하지 않는다. 계약 의미를 바꿔야 하면 +Context에서 먼저 수정한다. 복구 후 전체 후보 검증과 최종 main 조합의 일반 CI를 다시 확인한다. +integration 이외 DB의 destructive operation은 수행하지 않는다. + +일반 integration PR/push/schedule 검사는 sibling `main`을 사용한다. Context가 먼저 변경되어 +스냅샷 PR들이 남아 있는 동안 **integration main drift 실패는 실제 실패로 유지**한다. 후보 +조합이 성공해도 main drift 실패를 성공으로 바꾸거나 숨기지 않는다. 반대로 main의 초록 +표시도 열린 PR 후보 조합을 검증한 결과가 아니다. 마지막 병합 후 main 검사까지 성공했을 때 +묶음의 완료를 기록한다. + +## Automated safeguards and human responsibility + +자동으로 검사하는 것은 Context의 현재 main/전체 SHA, 깨끗한 작업 트리, 계약 map 분류, +이전/현재 mapped 경로와 revision marker만 허용하는 봇 브랜치 diff, push 직전 파일 제한과 +main 최신성, 일반 fast-forward push, PR head/base 정합성, 같은 run/attempt의 완료 job과 +실제 checkout 증거다. merge 충돌·예상 밖 파일·동시 push는 중단하며 force push하지 않는다. + +제품 의미·구현 영향·중간 호환성·병합 직전 최신성 재조회·필수 리뷰 충족·병합/복구 결정은 +담당자가 확인한다. `current`는 조회 시점의 결과이며 조회와 PR 본문 PATCH 사이 또는 +이후의 변경을 원자적으로 잠그지 않는다. 변경 묶음의 동시 수정을 조율하고 병합 직전 +다시 조회해야 한다. 현재 구현은 서비스 PR SHA에 별도 required check/status를 게시하지 +않으므로 PR 본문은 보호 규칙을 대체하지 않는다. + +## GitHub App and rollout Install the same GitHub App on `context`, `integration`, `core-api`, `compute-api`, and `web`. The app needs Contents write and Pull requests write -on the four snapshot targets, Contents read on Context, and Actions write on -integration to start candidate checks. Store +on the four snapshot targets and Contents read on Context. Candidate checks +now use reusable workflows, so propagation no longer requests Actions write. +The completion observer uses the integration `GITHUB_TOKEN` with Actions read +for completed jobs/artifacts, a Contents read/Pull requests write target token, +and a separate Contents read Context token. Store `REFINVEST_CI_APP_CLIENT_ID` as a variable and `REFINVEST_CI_APP_PRIVATE_KEY` as a secret in both Context and integration. The Context push workflow needs a token with Contents write on integration @@ -82,9 +211,12 @@ impact plan without write access and skips fork PRs because app secrets are unavailable there. The Context workflow source is maintained in the Context repository at `.github/workflows/contract-propagation.yml`. -When several repositories have PRs, review the candidate tuple first, then -merge Core, Compute, and Web changes before the integration fixture PR. The -regular integration PR checks use sibling `main` refs; the explicit candidate -checks are the evidence for the whole proposed tuple during propagation. -An integration `main` drift check can fail while those PRs remain open, and -must not be reported as a passing E2E run. +`contract-evidence.yml`은 기본 브랜치에 있어야 완료 이벤트를 받는다. 이 변경을 integration +main에 반영한 뒤 새 전파를 실행한다. 최초에는 trigger 활성화 전 성공·실패·취소·시간 초과, +head 변경, 새 Context, 일부 저장소만 변경되는 실제 봇 PR 경로를 확인한다. 로컬 mock 및 +bare Git remote 테스트는 GitHub App 설치, Actions job 이름, artifact 전달, PR 쓰기의 운영 +검증을 대신하지 않는다. 새 워크플로 소스를 후보 integration SHA의 소스와 구분해 기록한다. + +설정 조회 결과, 권장 보호 설정과 적용 순서는 +[`CONTRACT_PROPAGATION_SETTINGS.md`](CONTRACT_PROPAGATION_SETTINGS.md)를 확인한다. +이 구현에서 GitHub 저장소 설정을 변경하지 않는다. diff --git a/docs/CONTRACT_PROPAGATION_SETTINGS.md b/docs/CONTRACT_PROPAGATION_SETTINGS.md new file mode 100644 index 0000000..f227f5c --- /dev/null +++ b/docs/CONTRACT_PROPAGATION_SETTINGS.md @@ -0,0 +1,54 @@ +# Contract propagation settings review + +## Read-only inspection + +2026-10-27에 `refinvest-dev/{context,core-api,compute-api,web,integration}`의 +repository, `branches/main/protection`, `rulesets?includes_parents=true` REST API를 +익명 GET으로 조회했다. 로컬에 `gh`와 인증 세션이 없어 관리자 인증 조회는 하지 못했다. +설정, 보호 규칙, App 설치 또는 PR을 변경하지 않았다. + +| 항목 | 관찰 결과 | 해석 | +| --- | --- | --- | +| Context repository/rulesets | 404 | 비공개 또는 접근 불가 여부를 구분할 수 없다. 미확인. | +| Core/Compute/Web/integration repository | 200, 공개 저장소, 기본 브랜치 main | 조회 가능한 기본 정보만 확인. | +| main branch protection (다섯 저장소) | 401 | 인증 필요. 리뷰/필수 체크/관리자 적용/bypass는 미확인. | +| Core/Compute/Web/integration rulesets | 공개 조회에서 빈 배열 응답을 확인했지만 전체 적용 정책은 미확인 | branch protection과 인증 범위 밖 정책의 부재를 뜻하지 않는다. 아래 운영 확인에서 인증 후 재조회한다. | +| 자동 병합 허용, App 권한·설치, CODEOWNERS 적용, required checks | 미확인 | 공개 repository 응답만으로 실제 병합 권한이나 정책을 추정하지 않는다. | + +## Recommended settings and application order + +아래는 관리자가 검토·적용할 권장 사항이며 이 작업에서 적용하지 않았다. + +1. 관리자 read 권한으로 다섯 저장소의 branch protection, 조직 포함 ruleset, bypass actor, + App 설치 범위·권한, Actions 허용·secret/variable, auto-merge 허용 여부를 조회하고 보관한다. + 기존 required check 이름과 리뷰 정책을 먼저 확인한다. 조회할 수 없는 항목은 미확인으로 남긴다. +2. Context와 네 대상 main에 PR 필수, 담당자 승인 1명 이상, CODEOWNERS 리뷰, 새 push 시 + 기존 승인 무효화, 최신 push 작성자 이외의 승인, conversation resolution을 권장한다. + 관리자·App에도 적용하고 봇을 bypass 목록에 넣지 않는다. force push와 브랜치 삭제를 막는다. + CODEOWNERS의 실제 팀/담당자는 관리자가 지정한다. 자동 승인·자동 병합은 사용하지 않는다. +3. 각 저장소의 실제 PR head에서 실행되는 기존 서비스 CI와 계약/문서 검사를 required로 설정한다. + 적용 전에 실제 check context 이름을 확인한다. 이 변경은 기존 검증 job 이름을 유지한다. + 후보 검증은 integration의 실행에 연결되며 서비스 PR head에 check-run을 게시하지 않는다. + 따라서 `candidate-sync` 같은 이름을 서비스 required check로 바로 등록하면 영구 대기가 + 될 수 있다. PR 본문과 외부 후보 증거 확인은 담당자의 수동 절차로 유지한다. +4. integration main에 이 워크플로와 관찰 코드를 반영한다. App은 Context Contents read, + 대상 Contents/Pull requests write를 사용하고, 관찰 토큰은 Contents read/Pull requests write, + integration `GITHUB_TOKEN`은 Actions read로 제한한다. Context dispatch에 필요한 integration + Contents write는 기존 Context 측 토큰에 유지한다. App의 설치 권한 자체를 줄이는 것은 + 다른 워크플로 사용처를 확인한 후 별도 변경한다. 이 구현은 토큰 요청에서 Actions write를 제거한다. +5. Context 자동 trigger를 활성화하기 전에 현재 Context main SHA로 수동 전파한다. 실제 봇 PR의 + 생성/갱신, 다섯 고정 checkout, run/attempt artifact, 재사용 workflow job 이름, 종료 관찰, + 한국어 본문, 체크리스트와 담당자 리뷰를 확인한다. 실패·취소·시간 초과도 실제로 확인한다. + `workflow_run` 관찰자가 기본 브랜치에 배포되어 있는지와 artifact 보존 기간을 확인한다. +6. PR head 변경, 새 Context, 대상 main 변경, 부분 병합 후 재전파로 기존 성공을 재사용하지 + 않는지 확인한다. 한 저장소만 변경되는 경우와 늦은 관찰자의 덮어쓰기 방지도 점검한다. + 보호 규칙이 봇의 직접 main push와 리뷰 없는 병합을 막는지 관리자 확인을 남긴다. +7. 위 확인을 완료한 뒤 Context 자동 dispatch를 활성화한다. 이후 main drift와 후보 검증을 + 구분해 운영하고, 실패 시 나머지 병합을 중단하고 변경 묶음 단위로 기록·복구한다. + +## Remaining operational verification + +실제 GitHub App 토큰의 checkout/push/PR 쓰기, 재사용 워크플로 호출, 완료 관찰자의 권한, +API job 이름과 timeout conclusion, artifact 다운로드, 보호 규칙 적용은 로컬 테스트로 +확인되지 않았다. 실제 봇 PR 경로를 실행하기 전에는 **운영 검증 완료로 표시하지 않는다**. +보고서가 오래되면 권한·보호 설정을 다시 조회한다. diff --git a/scripts/contract_evidence.py b/scripts/contract_evidence.py new file mode 100644 index 0000000..6797d18 --- /dev/null +++ b/scripts/contract_evidence.py @@ -0,0 +1,335 @@ +"""Link completed candidate checks to their exact snapshot PR tuple. + +The completion workflow runs trusted default-branch code. Downloaded artifacts +are JSON data only; they are never extracted or executed. This module never +approves, merges, or changes repository settings. +""" + +from __future__ import annotations + +import argparse +import io +import json +import os +import re +import subprocess +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +from contract_sync import ContractSyncError, REPOSITORIES + +BRANCH = "automation/context-snapshots" +REMOTE_NAMES = {"coreApi": "core-api", "computeApi": "compute-api", + "web": "web", "integration": "integration"} +CHECKS = { + "candidate-sync": ("contract-sync-compatibility-evidence", "snapshot-sync"), + "candidate-smoke": ("api-smoke-and-web-acceptance-compatibility-evidence", + "api-smoke-and-web-acceptance"), +} + + +def gh_json(endpoint: str, *, payload: dict | None = None): + args = ["gh", "api", endpoint] + if payload is not None: + args += ["--method", "PATCH", "--input", "-"] + environment = os.environ.copy() + token_name = ("GH_ACTIONS_TOKEN" if "/actions/" in endpoint else + "GH_CONTEXT_TOKEN" if "/context/" in endpoint else "GH_TOKEN") + if environment.get(token_name): + environment["GH_TOKEN"] = environment[token_name] + result = subprocess.run( + args, input=json.dumps(payload) if payload is not None else None, + capture_output=True, text=True, encoding="utf-8", check=False, env=environment, + ) + if result.returncode: + raise ContractSyncError(f"GitHub API failed for {endpoint}: {result.stderr.strip()}") + return json.loads(result.stdout) + + +def pages(endpoint: str, key: str | None = None) -> list: + output = [] + for page in range(1, 101): + data = gh_json(f"{endpoint}{'&' if '?' in endpoint else '?'}per_page=100&page={page}") + entries = data[key] if key else data + output.extend(entries) + if len(entries) < 100: + return output + raise ContractSyncError("GitHub pagination limit exceeded") + + +def artifact_json(repository: str, artifacts: list, name: str, filename: str) -> dict | None: + matches = [item for item in artifacts if item["name"] == name and not item["expired"]] + if not matches: + return None + if len(matches) != 1 or matches[0]["size_in_bytes"] > 2_000_000: + raise ContractSyncError(f"Ambiguous or oversized artifact: {name}") + archive = subprocess.run( + ["gh", "api", f"repos/{repository}/actions/artifacts/{matches[0]['id']}/zip"], + capture_output=True, check=False, + env={**os.environ, "GH_TOKEN": os.environ.get("GH_ACTIONS_TOKEN", os.environ.get("GH_TOKEN", ""))}, + ) + if archive.returncode: + raise ContractSyncError(f"Unable to download artifact: {name}") + with zipfile.ZipFile(io.BytesIO(archive.stdout)) as bundle: + info = bundle.getinfo(filename) + if info.file_size > 2_000_000: + raise ContractSyncError(f"Oversized JSON: {name}") + return json.loads(bundle.read(info)) + + +def marker(manifest: dict) -> str: + return f"" + + +def validate_manifest(manifest: dict, repository: str, run_id: int, attempt: int) -> None: + if (manifest.get("schemaVersion") != 2 or manifest.get("repository") != repository + or manifest.get("runId") != run_id or manifest.get("runAttempt") != attempt + or set(manifest.get("heads", {})) != set(REPOSITORIES) + or set(manifest.get("bases", {})) != set(REPOSITORIES)): + raise ContractSyncError("Propagation manifest identity/tuple mismatch") + for sha in [manifest.get("contextRevision"), manifest.get("workflowSha"), + *manifest["heads"].values(), *manifest["bases"].values()]: + if not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ContractSyncError("Expected immutable full commit SHAs") + if not set(manifest.get("pullRequests", {})) <= set(REPOSITORIES): + raise ContractSyncError("Unexpected PR repository") + for key, pr in manifest.get("pullRequests", {}).items(): + if (type(pr.get("number")) is not int or pr["number"] < 1 + or pr.get("head") != manifest["heads"][key] + or pr.get("base") != manifest["bases"][key]): + raise ContractSyncError("PR identity/tuple mismatch") + + +def evaluate(manifest: dict, run: dict, jobs: list, evidence: dict) -> dict: + """Only completed jobs AND matching checkout evidence can produce success.""" + expected = {"context": manifest["contextRevision"], **manifest["heads"]} + checks = {} + for prefix, (_, verification_name) in CHECKS.items(): + matches = [job for job in jobs if job["name"].startswith(prefix + " / ")] + item = {"outcome": "not-recorded", "url": run["html_url"], "errors": []} + if len(matches) == 1: + job = matches[0] + item["url"] = job["html_url"] + item["outcome"] = (job.get("conclusion") or "not-recorded" + if job["status"] == "completed" else "pending") + else: + item["errors"].append("검증 job을 유일하게 식별하지 못했습니다.") + data = evidence.get(prefix) + item["checkedOutRevisions"] = { + key: entry.get("revision") for key, entry in (data or {}).get("repositories", {}).items() + } + item["compatibilityEvidence"] = data + if item["outcome"] == "success": + if (not data or data.get("verification", {}).get("name") != verification_name + or data.get("verification", {}).get("outcome") != "success" + or data.get("validation", {}).get("outcome") != "success"): + item["errors"].append("성공한 checkout 검증 증거가 없습니다.") + else: + actual = data.get("repositories", {}) + for key, sha in expected.items(): + entry = actual.get(key, {}) + if (not entry.get("available") or entry.get("revision") != sha + or entry.get("worktree") != "clean"): + item["errors"].append(f"후보 checkout 불일치: {key}") + if item["errors"]: + item["outcome"] = "evidence-mismatch" + checks[prefix] = item + outcomes = [item["outcome"] for item in checks.values()] + if run["status"] != "completed": + outcome = "pending" + elif (any(job.get("conclusion") == "timed_out" for job in jobs) + or run.get("conclusion") == "timed_out"): + outcome = "timed_out" + elif run.get("conclusion") == "cancelled" or "cancelled" in outcomes: + outcome = "cancelled" + elif run.get("conclusion") == "success" and all(value == "success" for value in outcomes): + outcome = "success" + else: + outcome = "failure" + return {"outcome": outcome, "workflowConclusion": run.get("conclusion"), "checks": checks} + + +def freshness(manifest: dict, context_head: str, mains: dict, prs: dict) -> list[str]: + errors = [] + if context_head != manifest["contextRevision"]: + errors.append("Context main이 바뀌었습니다. 새 Context SHA로 다시 전파·검증하세요.") + for key in REPOSITORIES: + if mains.get(key) != manifest["bases"][key]: + errors.append(f"{key} main이 바뀌었습니다. 최신 base를 포함해 다시 검증하세요.") + if key not in manifest["pullRequests"]: + if mains.get(key) != manifest["heads"][key]: + errors.append(f"{key}의 변경 없는 후보 SHA가 더 이상 최신 main이 아닙니다.") + continue + pr = prs[key] + if (pr["state"] != "open" or pr["head"]["sha"] != manifest["heads"][key] + or pr["head"]["ref"] != BRANCH or pr["base"]["ref"] != "main" + or pr["head"]["repo"]["full_name"] != pr["base"]["repo"]["full_name"]): + errors.append(f"{key} PR head/base/state가 바뀌었습니다. 다시 검증하세요.") + return errors + + +def render_body(manifest: dict, key: str, result: dict | None = None) -> str: + owner = manifest["repository"].split("/")[0] + run_url = (f"https://github.com/{manifest['repository']}/actions/runs/" + f"{manifest['runId']}/attempts/{manifest['runAttempt']}") + status = result["outcome"] if result else "pending" + current = result.get("freshness", "unconfirmed") if result else "unconfirmed" + lines = [marker(manifest), "## Context source", "", + f"원본 Context SHA: `{manifest['contextRevision']}`", + f"이 PR의 후보 head SHA: `{manifest['heads'][key]}`", "", + "## Candidate set", "", + "변경이 없는 저장소도 고정 SHA로 검증합니다. 변경 묶음 전체를 검토하세요.", "", + "| Repository | Candidate SHA | Snapshot PR |", "| --- | --- | --- |", + f"| Context | `{manifest['contextRevision']}` | 원본 |"] + for repo in REPOSITORIES: + pr = manifest.get("pullRequests", {}).get(repo) + link = (f"[#{pr['number']}](https://github.com/{owner}/{REMOTE_NAMES[repo]}/pull/{pr['number']})" + if pr else "스냅샷 PR 연결 대기 중" if repo in manifest.get("changedFiles", {}) + else "스냅샷 변경 없음 (main 고정)") + lines.append(f"| {REMOTE_NAMES[repo]} | `{manifest['heads'][repo]}` | {link} |") + lines += ["", "## Verification evidence", "", + f"후보 조합 검증: **{status}** / 최신성: **{current}**", + f"[검증 실행 및 artifact]({run_url}) · 실행 시도 `{manifest['runAttempt']}`", + f"워크플로 소스 SHA: `{manifest['workflowSha']}`", + "전파 시작이나 스냅샷 복사만으로 검증 성공을 뜻하지 않습니다.", + "완료 관찰 워크플로의 contract-candidate-evidence artifact에 최종 JSON/Markdown을 보관합니다."] + if result: + lines.append("") + for name, check in result.get("checks", {}).items(): + lines.append(f"- [{name}]({check['url']}): **{check['outcome']}**") + lines.extend(f" - {error}" for error in check["errors"]) + lines.extend(f"- {error}" for error in result.get("freshnessErrors", [])) + if result.get("reportUrl"): + lines.append(f"[완료 증거 artifact 및 요약]({result['reportUrl']})") + lines += ["", "## Changed files", ""] + lines.extend(f"- `{path}`" for path in manifest.get("changedFiles", {}).get(key, [])) + lines += ["", "## Maintainer review", "", + "- [ ] 원본 Context diff, 이 PR head, 모든 후보 SHA와 완료 증거가 일치하는지 병합 직전에 확인했습니다.", + "- [ ] 모든 필수 후보 검증이 success이며 최신성이 current인지 확인했습니다. pending/failure/cancelled/timed_out/미확인은 통과가 아닙니다.", + "- [ ] OpenAPI·규범 문서·fixture의 의미와 Core/Compute/Web 구현 영향을 검토했습니다. 파일 종류로 승인하지 않았습니다.", + "- [ ] 구현 PR, 중간 호환성, 병합 순서, 부분 실패 시 중단·복구 계획을 변경 묶음 전체에 기록했습니다.", + "- [ ] head/base/Context 변경 시 재검증했습니다. integration main drift 결과를 후보 성공과 구분했습니다.", + "- [ ] 필수 리뷰와 저장소 보호 규칙을 충족했으며 담당자가 수동 병합합니다.", + "", "절차: integration의 `docs/CONTRACT_PROPAGATION.md`를 확인하세요.", ""] + return "\n".join(lines) + + +def render_report(manifest: dict, result: dict) -> str: + body = render_body(manifest, "integration", result).replace( + "이 PR의 후보 head SHA:", "integration 후보 SHA:", 1) + lines = [body, "## Files by repository", ""] + for key, files in manifest.get("changedFiles", {}).items(): + lines.append(f"- {REMOTE_NAMES[key]}") + lines.extend(f" - `{path}`" for path in files) + lines += ["", "## Publication status", ""] + lines.extend(f"- {key}: {status}" for key, status in result["prUpdates"].items()) + return "\n".join(lines) + "\n" + + +def record_missing_tuple(repository: str, run_id: int, attempt: int, + run: dict, jobs: list, output: Path) -> None: + conclusions = [job.get("conclusion") for job in jobs] + outcome = ("timed_out" if "timed_out" in conclusions else + "cancelled" if run.get("conclusion") == "cancelled" else "failure") + result = { + "schemaVersion": 2, "repository": repository, "runId": run_id, + "runAttempt": attempt, "workflowSha": run["head_sha"], + "outcome": outcome, "workflowConclusion": run.get("conclusion"), + "freshness": "unconfirmed", "tupleAvailable": False, + "contextRevision": None, "heads": None, "jobs": jobs, + "error": "후보 manifest가 없습니다. SHA 조합의 검증 여부는 미확인입니다.", + } + output.mkdir(parents=True, exist_ok=True) + (output / "contract-candidate-evidence.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output / "contract-candidate-evidence.md").write_text( + f"## Verification evidence\n\n실행 종료: **{outcome}**\n\n" + f"{result['error']} 성공으로 간주하지 마세요.\n\n" + f"[전파 실행]({run['html_url']}) · 실행 시도 `{attempt}`\n", encoding="utf-8") + + +def reconcile(repository: str, run_id: int, attempt: int, output: Path) -> dict: + endpoint = f"repos/{repository}/actions/runs/{run_id}" + run = gh_json(f"{endpoint}/attempts/{attempt}") + if (run["path"] != ".github/workflows/contract-propagation.yml" + or run["event"] not in ("workflow_dispatch", "repository_dispatch") + or run["head_branch"] != "main" or run["repository"]["full_name"] != repository + or run["status"] != "completed" or run["run_attempt"] != attempt): + raise ContractSyncError("Only completed trusted main propagation runs may be reported") + jobs = pages(f"{endpoint}/attempts/{attempt}/jobs", "jobs") + artifacts = pages(f"{endpoint}/artifacts", "artifacts") + manifest = artifact_json(repository, artifacts, f"contract-propagation-result-{attempt}", + "contract-propagation.json") + if manifest is None: + record_missing_tuple(repository, run_id, attempt, run, jobs, output) + raise ContractSyncError("No propagation manifest: candidate verification is unconfirmed") + validate_manifest(manifest, repository, run_id, attempt) + if manifest["workflowSha"] != run["head_sha"]: + raise ContractSyncError("Manifest workflow source mismatch") + evidence = {key: artifact_json(repository, artifacts, f"{name}-{attempt}", "compatibility.json") + for key, (name, _) in CHECKS.items()} + result = {**manifest, **evaluate(manifest, run, jobs, evidence), + "generatedAtUtc": datetime.now(timezone.utc).isoformat()} + result.update(freshness="unconfirmed", freshnessErrors=["최신성은 미확인입니다."], + reportUrl=os.environ.get("CONTRACT_REPORT_URL"), prUpdates={}) + # Persist the result before PR writes so write failures cannot erase evidence. + output.mkdir(parents=True, exist_ok=True) + def save(): + (output / "contract-candidate-evidence.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output / "contract-candidate-evidence.md").write_text( + render_report(manifest, result), encoding="utf-8") + save() + owner = repository.split("/")[0] + try: + prs = {key: gh_json(f"repos/{owner}/{REMOTE_NAMES[key]}/pulls/{pr['number']}") + for key, pr in manifest["pullRequests"].items()} + mains = {key: gh_json(f"repos/{owner}/{name}/git/ref/heads/main")["object"]["sha"] + for key, name in REMOTE_NAMES.items()} + context_head = gh_json(f"repos/{owner}/context/git/ref/heads/main")["object"]["sha"] + errors = freshness(manifest, context_head, mains, prs) + result.update(freshness="stale" if errors else "current", freshnessErrors=errors, + freshnessCheckedAtUtc=datetime.now(timezone.utc).isoformat()) + except ContractSyncError as error: + result["freshnessErrors"] = [f"최신성 조회 실패: {error}"] + save() + raise + save() + for key, pr in manifest["pullRequests"].items(): + full_name = f"{owner}/{REMOTE_NAMES[key]}" + latest = gh_json(f"repos/{full_name}/pulls/{pr['number']}") + # A delayed observer must not overwrite a newer propagation's PR body. + if (latest["state"] != "open" or latest["head"]["sha"] != manifest["heads"][key] + or latest["head"]["ref"] != BRANCH or latest["base"]["ref"] != "main" + or latest["head"]["repo"]["full_name"] != full_name + or latest["base"]["repo"]["full_name"] != full_name + or marker(manifest) not in (latest.get("body") or "")): + result["prUpdates"][key] = "skipped: PR changed or newer propagation owns body" + else: + try: + gh_json(f"repos/{full_name}/pulls/{pr['number']}", + payload={"body": render_body(manifest, key, result)}) + result["prUpdates"][key] = "updated" + except ContractSyncError as error: + result["prUpdates"][key] = f"failed: {error}" + save() + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True) + parser.add_argument("--run-id", required=True, type=int) + parser.add_argument("--attempt", required=True, type=int) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + if not re.fullmatch(r"[A-Za-z0-9_.-]+/integration", args.repository): + parser.error("Expected owner/integration") + result = reconcile(args.repository, args.run_id, args.attempt, args.output) + if any(value.startswith("failed:") for value in result["prUpdates"].values()): + raise ContractSyncError("Evidence saved, but one or more PR body updates failed") + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_contract_sync.py b/scripts/publish_contract_sync.py index b3af1a8..fc09047 100644 --- a/scripts/publish_contract_sync.py +++ b/scripts/publish_contract_sync.py @@ -8,6 +8,8 @@ import argparse import json +import os +import re import subprocess import tempfile from pathlib import Path @@ -16,17 +18,17 @@ REPOSITORIES, ContractSyncError, apply_plan, context_files, default_roots, expand_map, git, load_map, plan, ) +from contract_evidence import BRANCH, REMOTE_NAMES, render_body -BRANCH = "automation/context-snapshots" -REMOTE_NAMES = { - "coreApi": "core-api", "computeApi": "compute-api", - "web": "web", "integration": "integration", -} +def assert_remote_main(root: Path, expected: str) -> None: + latest = run("git", "ls-remote", "origin", "refs/heads/main", cwd=root).split() + if not latest or latest[0] != expected: + raise ContractSyncError(f"main changed during propagation: {root}; rerun with latest SHAs") def run(*args: str, cwd: Path | None = None) -> str: - result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, check=False) + result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, encoding="utf-8", check=False) if result.returncode: raise ContractSyncError( f"{' '.join(args)} failed: {result.stderr.strip() or result.stdout.strip()}" @@ -49,6 +51,8 @@ def prepare_branch( run("git", "fetch", "origin", f"{BRANCH}:refs/remotes/origin/{BRANCH}", cwd=root) marker = manifest["repositories"][repository]["revision"] pinned = git(root, "show", f"origin/{BRANCH}:{marker}").decode("ascii").strip() + if not re.fullmatch(r"[0-9a-f]{40}", pinned): + raise ContractSyncError(f"Invalid Context marker on bot branch: {repository}") previous = expand_map( manifest, context_files(context, pinned), allow_missing=True ) @@ -71,20 +75,17 @@ def prepare_branch( run("git", "switch", "-c", BRANCH, "origin/main", cwd=root) -def upsert_pr(root: Path, owner: str, repository: str, revision: str, files: list[str]) -> None: +def upsert_pr(root: Path, owner: str, repository: str, manifest: dict) -> dict: full_name = f"{owner}/{REMOTE_NAMES[repository]}" existing = json.loads(run( "gh", "pr", "list", "--repo", full_name, "--head", BRANCH, - "--state", "open", "--json", "number", cwd=root, + "--base", "main", "--state", "open", "--json", "number", cwd=root, )) + if len(existing) > 1: + raise ContractSyncError(f"Ambiguous bot PRs: {full_name}") + revision = manifest["contextRevision"] title = f"chore(contract): sync Context {revision[:12]}" - body = ( - f"Context source commit: `{revision}`\n\n" - "Mechanical snapshot propagation only. Review API and normative document " - "changes for product-code impact before merging.\n\n" - "Copied snapshot paths:\n" - + "".join(f"- `{path}`\n" for path in files) - ) + body = render_body(manifest, repository) with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", suffix=".md", delete=False ) as handle: @@ -92,6 +93,10 @@ def upsert_pr(root: Path, owner: str, repository: str, revision: str, files: lis body_path = Path(handle.name) try: if existing: + current = json.loads(run("gh", "api", f"repos/{full_name}/pulls/{existing[0]['number']}")) + if (current["head"]["sha"] != manifest["heads"][repository] + or current["head"]["repo"]["full_name"] != full_name): + raise ContractSyncError(f"PR head changed: {full_name}") run("gh", "pr", "edit", str(existing[0]["number"]), "--repo", full_name, "--title", title, "--body-file", str(body_path), cwd=root) @@ -103,10 +108,24 @@ def upsert_pr(root: Path, owner: str, repository: str, revision: str, files: lis ) finally: body_path.unlink() + prs = json.loads(run("gh", "pr", "list", "--repo", full_name, "--head", BRANCH, + "--base", "main", "--state", "open", "--json", "number")) + if len(prs) != 1: + raise ContractSyncError(f"Expected one bot PR: {full_name}") + pr = json.loads(run("gh", "api", f"repos/{full_name}/pulls/{prs[0]['number']}")) + if (pr["head"]["sha"] != manifest["heads"][repository] + or pr["head"]["repo"]["full_name"] != full_name + or pr["base"]["repo"]["full_name"] != full_name + or pr["head"]["ref"] != BRANCH or pr["base"]["ref"] != "main"): + raise ContractSyncError(f"Unexpected PR head/base: {full_name}") + return {"number": pr["number"], "head": pr["head"]["sha"], + "base": manifest["bases"][repository]} def publish(context: Path, revision: str, roots: dict[str, Path], owner: str) -> dict: manifest = load_map() + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ContractSyncError("Dispatch requires a full Context commit SHA") revision = git(context, "rev-parse", "--verify", f"{revision}^{{commit}}")\ .decode("ascii").strip() run("git", "fetch", "origin", "main", cwd=context) @@ -117,24 +136,27 @@ def publish(context: Path, revision: str, roots: dict[str, Path], owner: str) -> ) if merged.returncode: raise ContractSyncError(f"Context commit is not merged into main: {revision}") + assert_remote_main(context, revision) replicas = expand_map(manifest, context_files(context, revision)) + allowed_by_repo = {} + bases = {} for repository in REPOSITORIES: allowed = { replica.target for replica in replicas if replica.repository == repository } allowed.add(manifest["repositories"][repository]["revision"]) prepare_branch(roots[repository], repository, allowed, context, manifest) + allowed_by_repo[repository] = allowed + bases[repository] = run("git", "rev-parse", "origin/main", cwd=roots[repository]) impact = plan(manifest, context, revision, roots) - changed_sources: set[str] = set() - pull_requests: dict[str, str] = {} + changed_files = {} heads: dict[str, str] = {} for repository in REPOSITORIES: root = roots[repository] changes = impact["repositories"].get(repository, []) if changes: files = apply_plan(manifest, context, roots, impact, repository) - changed_sources.update(change["source"] for change in changes) run("git", "add", "--", *files, cwd=root) run("git", "-c", "user.name=refinvest-contract-bot", "-c", "user.email=refinvest-contract-bot@users.noreply.github.com", @@ -142,30 +164,41 @@ def publish(context: Path, revision: str, roots: dict[str, Path], owner: str) -> branch_diff = run( "git", "diff", "--name-only", "origin/main...HEAD", cwd=root ).splitlines() + outside = set(branch_diff) - allowed_by_repo[repository] + if outside: + raise ContractSyncError(f"Outgoing non-snapshot files in {repository}: {sorted(outside)}") + assert_remote_main(context, revision) + assert_remote_main(root, bases[repository]) if branch_diff: run("git", "push", "origin", f"HEAD:refs/heads/{BRANCH}", cwd=root) - marker = root / manifest["repositories"][repository]["revision"] - upsert_pr( - root, owner, repository, - marker.read_text(encoding="ascii").strip(), branch_diff, - ) - pull_requests[repository] = BRANCH - changed_sources.update( - replica.source for replica in replicas - if replica.repository == repository and replica.target in branch_diff - ) + changed_files[repository] = branch_diff heads[repository] = run( "git", "rev-parse", "HEAD" if branch_diff else "origin/main", cwd=root ) result = { + "schemaVersion": 2, + "repository": f"{owner}/integration", + "runId": int(os.environ["GITHUB_RUN_ID"]), + "runAttempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), + "workflowSha": os.environ["GITHUB_SHA"], "contextRevision": revision, "heads": heads, - "pullRequestBranches": pull_requests, - "runApiSmoke": any( - source.startswith(("openapi/", "fixtures/")) for source in changed_sources - ), + "bases": bases, + "changedFiles": changed_files, + "pullRequests": {}, + "outcome": "pending", + "runApiSmoke": True, } + # Resolve the entire tuple before rendering any PR. A second pass links all PRs. + for repository in changed_files: + assert_remote_main(context, revision) + assert_remote_main(roots[repository], bases[repository]) + result["pullRequests"][repository] = upsert_pr(roots[repository], owner, repository, result) + for repository in changed_files: + assert_remote_main(context, revision) + assert_remote_main(roots[repository], bases[repository]) + upsert_pr(roots[repository], owner, repository, result) return result diff --git a/scripts/test_contract_evidence.py b/scripts/test_contract_evidence.py new file mode 100644 index 0000000..1e6488b --- /dev/null +++ b/scripts/test_contract_evidence.py @@ -0,0 +1,333 @@ +import copy +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parent)) +import contract_evidence as evidence +from contract_sync import ContractSyncError, REPOSITORIES + + +def candidate(): + heads = dict(zip(REPOSITORIES, (char * 40 for char in "bcde"))) + return { + "schemaVersion": 2, "repository": "example/integration", "runId": 42, + "runAttempt": 2, "workflowSha": "9" * 40, "contextRevision": "a" * 40, + "heads": heads, "bases": {**heads, "coreApi": "f" * 40}, + "pullRequests": {"coreApi": {"number": 7, "head": heads["coreApi"], "base": "f" * 40}}, + "changedFiles": {"coreApi": ["docs/RULE.md", "CONTEXT_REVISION"]}, + } + + +def completed_run(): + return {"status": "completed", "conclusion": "success", "html_url": "https://example/run/42", + "path": ".github/workflows/contract-propagation.yml", "head_branch": "main", + "head_sha": "9" * 40, "event": "repository_dispatch", "run_attempt": 2, + "repository": {"full_name": "example/integration"}} + + +def completed_jobs(): + return [{"name": f"{key} / {name}", "status": "completed", "conclusion": "success", + "html_url": f"https://example/jobs/{key}"} + for key, (_, name) in evidence.CHECKS.items()] + + +def checkout_evidence(manifest): + repos = {key: {"available": True, "revision": sha, "worktree": "clean"} + for key, sha in {"context": manifest["contextRevision"], **manifest["heads"]}.items()} + return {key: {"verification": {"name": name, "outcome": "success"}, + "validation": {"outcome": "success"}, "repositories": copy.deepcopy(repos)} + for key, (_, name) in evidence.CHECKS.items()} + + +def live_pr(manifest): + return {"number": 7, "state": "open", "body": evidence.render_body(manifest, "coreApi"), + "head": {"sha": manifest["heads"]["coreApi"], "ref": evidence.BRANCH, + "repo": {"full_name": "example/core-api"}}, + "base": {"ref": "main", "repo": {"full_name": "example/core-api"}}} + + +class OutcomeTest(unittest.TestCase): + def setUp(self): + self.manifest = candidate() + self.run = completed_run() + self.jobs = completed_jobs() + self.data = checkout_evidence(self.manifest) + + def outcome(self): + return evidence.evaluate(self.manifest, self.run, self.jobs, self.data) + + def test_success_requires_completed_jobs_and_exact_five_checkout_shas(self): + result = self.outcome() + self.assertEqual(result["outcome"], "success") + self.assertEqual(result["checks"]["candidate-sync"]["checkedOutRevisions"]["web"], "d" * 40) + + def test_dispatch_and_running_are_pending_even_with_previous_success_data(self): + for state in ("queued", "in_progress"): + with self.subTest(state=state): + self.run.update(status=state, conclusion=None) + self.assertEqual(self.outcome()["outcome"], "pending") + + def test_failure_cancelled_and_timeout_are_distinct(self): + for conclusion, workflow_conclusion in (("failure", "failure"), ("cancelled", "cancelled"), + ("timed_out", "failure")): + with self.subTest(conclusion=conclusion): + self.run["conclusion"] = workflow_conclusion + self.jobs[0]["conclusion"] = conclusion + self.assertEqual(self.outcome()["outcome"], conclusion) + + def test_cancel_before_jobs_start_is_cancelled_without_fabricated_checkout(self): + self.run["conclusion"] = "cancelled" + self.jobs = [] + self.data = {} + self.assertEqual(self.outcome()["outcome"], "cancelled") + self.assertEqual(self.outcome()["checks"]["candidate-sync"]["checkedOutRevisions"], {}) + + def test_propagation_timeout_after_manifest_is_also_timeout(self): + self.run["conclusion"] = "failure" + self.jobs.append({"name": "propagate", "status": "completed", "conclusion": "timed_out"}) + self.assertEqual(self.outcome()["outcome"], "timed_out") + + def test_skipped_missing_ambiguous_and_in_progress_checks_never_pass(self): + for mode in ("skipped", "missing", "ambiguous", "in_progress"): + with self.subTest(mode=mode): + self.jobs = completed_jobs() + if mode == "missing": + self.jobs.pop() + elif mode == "ambiguous": + self.jobs.append(copy.deepcopy(self.jobs[-1])) + elif mode == "in_progress": + self.jobs[-1]["status"] = mode + else: + self.jobs[-1]["conclusion"] = mode + self.assertEqual(self.outcome()["outcome"], "failure") + + def test_missing_evidence_and_any_wrong_or_dirty_checkout_fail_closed(self): + for key in ("context", *REPOSITORIES): + with self.subTest(repository=key): + self.data = checkout_evidence(self.manifest) + self.data["candidate-smoke"]["repositories"][key]["revision"] = "0" * 40 + self.assertEqual(self.outcome()["outcome"], "failure") + self.data = checkout_evidence(self.manifest) + self.data["candidate-sync"]["repositories"]["web"]["worktree"] = "dirty" + self.assertEqual(self.outcome()["outcome"], "failure") + self.data = {} + self.assertEqual(self.outcome()["outcome"], "failure") + + def test_inner_verification_or_validation_failure_never_passes(self): + for field in ("verification", "validation"): + self.data = checkout_evidence(self.manifest) + self.data["candidate-sync"][field]["outcome"] = "failure" + self.assertEqual(self.outcome()["outcome"], "failure") + + +class FreshnessTest(unittest.TestCase): + def setUp(self): + self.manifest = candidate() + self.context = self.manifest["contextRevision"] + self.mains = self.manifest["bases"].copy() + self.prs = {"coreApi": live_pr(self.manifest)} + + def errors(self): + return evidence.freshness(self.manifest, self.context, self.mains, self.prs) + + def test_docs_only_pr_keeps_unchanged_repositories_in_tuple(self): + self.assertEqual(self.errors(), []) + body = evidence.render_body(self.manifest, "coreApi") + for sha in self.manifest["heads"].values(): + self.assertIn(sha, body) + self.assertIn("스냅샷 변경 없음", body) + self.assertIn("**pending**", body) + for heading in ("Context source", "Candidate set", "Verification evidence", + "Changed files", "Maintainer review"): + self.assertIn("## " + heading, body) + pending = copy.deepcopy(self.manifest) + pending["pullRequests"] = {} + self.assertIn("스냅샷 PR 연결 대기 중", evidence.render_body(pending, "coreApi")) + + def test_pr_head_change_invalidates_existing_success(self): + self.prs["coreApi"]["head"]["sha"] = "1" * 40 + self.assertTrue(self.errors()) + + def test_new_context_commit_invalidates_existing_success(self): + self.context = "1" * 40 + self.assertTrue(self.errors()) + + def test_changed_main_invalidates_with_and_without_snapshot_pr(self): + for key in REPOSITORIES: + with self.subTest(repository=key): + self.mains = self.manifest["bases"].copy() + self.mains[key] = "1" * 40 + self.assertTrue(self.errors()) + + def test_closed_or_retargeted_pr_cannot_reuse_evidence(self): + self.prs["coreApi"]["state"] = "closed" + self.assertTrue(self.errors()) + self.prs["coreApi"] = live_pr(self.manifest) + self.prs["coreApi"]["base"]["ref"] = "release" + self.assertTrue(self.errors()) + + +class CompletionObserverTest(unittest.TestCase): + def setUp(self): + self.manifest = candidate() + self.run = completed_run() + self.pr = live_pr(self.manifest) + self.updates = [] + self.data = checkout_evidence(self.manifest) + + def api(self, endpoint, *, payload=None): + if "/attempts/" in endpoint: + return self.run + if "/pulls/" in endpoint: + if payload: + self.updates.append(payload) + return self.pr + if "/context/" in endpoint: + return {"object": {"sha": self.manifest["contextRevision"]}} + for key, name in evidence.REMOTE_NAMES.items(): + if f"/{name}/git/ref/" in endpoint: + return {"object": {"sha": self.manifest["bases"][key]}} + self.fail(f"Unexpected endpoint: {endpoint}") + + def artifact(self, repository, artifacts, name, filename): + if name == "contract-propagation-result-2": + return self.manifest + for key, (artifact, _) in evidence.CHECKS.items(): + if name == artifact + "-2": + return self.data[key] + self.fail(f"Unexpected artifact: {name}") + + def collect(self, output): + with patch.object(evidence, "gh_json", side_effect=self.api), \ + patch.object(evidence, "pages", side_effect=lambda url, key: completed_jobs() if key == "jobs" else []), \ + patch.object(evidence, "artifact_json", side_effect=self.artifact): + return evidence.reconcile("example/integration", 42, 2, output) + + def test_completed_run_updates_same_pr_and_persists_json_markdown(self): + with tempfile.TemporaryDirectory() as directory: + result = self.collect(Path(directory)) + self.assertEqual(result["outcome"], "success") + self.assertEqual(result["freshness"], "current") + self.assertEqual(len(self.updates), 1) + self.assertIn("**success**", self.updates[0]["body"]) + saved = json.loads((Path(directory) / "contract-candidate-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(saved["prUpdates"]["coreApi"], "updated") + self.assertTrue((Path(directory) / "contract-candidate-evidence.md").is_file()) + + def test_delayed_observer_never_overwrites_newer_pending_body_or_head(self): + for change in ("body", "head"): + with self.subTest(change=change), tempfile.TemporaryDirectory() as directory: + self.pr = live_pr(self.manifest) + if change == "body": + self.pr["body"] = "" + else: + self.pr["head"]["sha"] = "0" * 40 + result = self.collect(Path(directory)) + self.assertEqual(self.updates, []) + self.assertTrue(result["prUpdates"]["coreApi"].startswith("skipped:")) + if change == "head": + self.assertEqual(result["freshness"], "stale") + + def test_observer_rejects_untrusted_workflow_branch_event_and_attempt(self): + for field, value in (("head_branch", "feature"), ("event", "pull_request"), + ("path", ".github/workflows/other.yml"), ("run_attempt", 1)): + with self.subTest(field=field), tempfile.TemporaryDirectory() as directory: + self.run = completed_run() + self.run[field] = value + with self.assertRaises(ContractSyncError): + self.collect(Path(directory)) + self.assertEqual(self.updates, []) + + def test_no_manifest_is_unconfirmed_and_does_not_write_pr(self): + with patch.object(evidence, "gh_json", return_value=self.run), \ + patch.object(evidence, "pages", return_value=[]), \ + patch.object(evidence, "artifact_json", return_value=None), \ + tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ContractSyncError, "unconfirmed"): + evidence.reconcile("example/integration", 42, 2, Path(directory)) + saved = json.loads((Path(directory) / "contract-candidate-evidence.json").read_text(encoding="utf-8")) + self.assertFalse(saved["tupleAvailable"]) + self.assertEqual(saved["freshness"], "unconfirmed") + self.assertEqual(saved["outcome"], "failure") + + def test_cancel_or_timeout_before_manifest_records_termination_without_inventing_shas(self): + for conclusion in ("cancelled", "timed_out"): + run = completed_run() + run["conclusion"] = "cancelled" if conclusion == "cancelled" else "failure" + jobs = [{"name": "propagate", "conclusion": conclusion}] + with self.subTest(conclusion=conclusion), tempfile.TemporaryDirectory() as directory, \ + patch.object(evidence, "gh_json", return_value=run), \ + patch.object(evidence, "pages", side_effect=lambda url, key: jobs if key == "jobs" else []), \ + patch.object(evidence, "artifact_json", return_value=None): + with self.assertRaisesRegex(ContractSyncError, "unconfirmed"): + evidence.reconcile("example/integration", 42, 2, Path(directory)) + saved = json.loads((Path(directory) / "contract-candidate-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(saved["outcome"], conclusion) + self.assertIsNone(saved["heads"]) + + def test_freshness_api_failure_preserves_job_result_as_unconfirmed(self): + original = self.api + def api(endpoint, *, payload=None): + if "/context/" in endpoint: + raise ContractSyncError("Context read unavailable") + return original(endpoint, payload=payload) + self.api = api + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ContractSyncError, "unavailable"): + self.collect(Path(directory)) + saved = json.loads((Path(directory) / "contract-candidate-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(saved["outcome"], "success") + self.assertEqual(saved["freshness"], "unconfirmed") + self.assertEqual(self.updates, []) + + def test_pr_write_failure_does_not_erase_final_evidence(self): + original = self.api + def api(endpoint, *, payload=None): + if payload: + raise ContractSyncError("PR write unavailable") + return original(endpoint, payload=payload) + self.api = api + with tempfile.TemporaryDirectory() as directory: + result = self.collect(Path(directory)) + self.assertTrue(result["prUpdates"]["coreApi"].startswith("failed:")) + saved = json.loads((Path(directory) / "contract-candidate-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(saved["outcome"], "success") + self.assertEqual(saved["prUpdates"], result["prUpdates"]) + + def test_manifest_run_attempt_and_tuple_are_validated(self): + for field, value in (("runAttempt", 1), ("repository", "other/integration"), + ("contextRevision", "main"), ("heads", {})): + manifest = candidate() + manifest[field] = value + with self.subTest(field=field), self.assertRaises(ContractSyncError): + evidence.validate_manifest(manifest, "example/integration", 42, 2) + + +class ArtifactTest(unittest.TestCase): + def test_json_is_read_without_extracting_archive_files(self): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as bundle: + bundle.writestr("compatibility.json", '{"ok":true}') + bundle.writestr("../../untrusted.py", "raise RuntimeError('must not run')") + item = {"name": "evidence-2", "expired": False, "id": 12, "size_in_bytes": len(buffer.getvalue())} + with patch.object(evidence.subprocess, "run", return_value=subprocess.CompletedProcess( + [], 0, stdout=buffer.getvalue(), stderr=b"")): + self.assertEqual(evidence.artifact_json("example/integration", [item], "evidence-2", "compatibility.json"), {"ok": True}) + + def test_expired_missing_and_ambiguous_artifacts_are_not_success(self): + self.assertIsNone(evidence.artifact_json("example/integration", [], "evidence-2", "compatibility.json")) + item = {"name": "evidence-2", "expired": False, "id": 12, "size_in_bytes": 10} + with self.assertRaises(ContractSyncError): + evidence.artifact_json("example/integration", [item, item], "evidence-2", "compatibility.json") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_contract_workflows.py b/scripts/test_contract_workflows.py new file mode 100644 index 0000000..d1872de --- /dev/null +++ b/scripts/test_contract_workflows.py @@ -0,0 +1,54 @@ +"""Check the workflow wiring that binds evidence to the candidate run.""" + +import sys +import unittest +from pathlib import Path + +import yaml + +sys.path.insert(0, str(Path(__file__).parent)) +from contract_evidence import CHECKS + +WORKFLOWS = Path(__file__).resolve().parents[1] / ".github/workflows" + + +def workflow(name): + # BaseLoader preserves GitHub's `on` key instead of YAML 1.1 boolean coercion. + return yaml.load((WORKFLOWS / name).read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + + +class CandidateWorkflowTest(unittest.TestCase): + def test_checks_are_called_with_all_five_pinned_outputs(self): + propagation = workflow("contract-propagation.yml") + for key in CHECKS: + job = propagation["jobs"][key] + callee = workflow(Path(job["uses"]).name) + self.assertEqual(set(job["with"]), set(callee["on"]["workflow_call"]["inputs"])) + self.assertEqual(len(job["with"]), 5) + for name, value in job["with"].items(): + self.assertEqual(value, "${{ needs.propagate.outputs." + name + " }}") + self.assertEqual(callee["on"]["workflow_call"]["inputs"][name]["required"], "true") + self.assertIn("candidate-sync", propagation["jobs"]["candidate-smoke"]["needs"]) + + def test_attempt_names_match_collector_and_existing_entrypoints_remain(self): + for key, filename in (("candidate-sync", "contract-sync.yml"), ("candidate-smoke", "api-smoke.yml")): + data = workflow(filename) + self.assertTrue({"pull_request", "push", "workflow_dispatch", "workflow_call"} <= set(data["on"])) + uploads = [step["with"]["name"] for job in data["jobs"].values() for step in job["steps"] + if step.get("uses", "").startswith("actions/upload-artifact")] + self.assertIn(CHECKS[key][0] + "-${{ github.run_attempt }}", uploads) + + def test_observer_runs_after_completion_and_executes_only_default_branch_code(self): + data = workflow("contract-evidence.yml") + self.assertEqual(data["on"]["workflow_run"]["types"], ["completed"]) + self.assertEqual(data["on"]["workflow_run"]["workflows"], ["Propagate Context snapshots"]) + job = data["jobs"]["record"] + self.assertIn("head_branch == 'main'", job["if"]) + checkout = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/checkout")) + self.assertEqual(checkout["with"]["ref"], "refs/heads/main") + self.assertEqual(checkout["with"]["persist-credentials"], "false") + self.assertEqual(data["permissions"]["actions"], "read") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_publish_contract_sync.py b/scripts/test_publish_contract_sync.py new file mode 100644 index 0000000..31ebea7 --- /dev/null +++ b/scripts/test_publish_contract_sync.py @@ -0,0 +1,175 @@ +"""Exercise publishing against local bare remotes; GitHub writes are mocked.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parent)) +import publish_contract_sync as publisher +from contract_sync import ContractSyncError, REPOSITORIES + + +def git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args], stderr=subprocess.PIPE).decode().strip() + + +class PublishingTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.context = self.repository("context") + (self.context / "docs").mkdir() + (self.context / "docs/RULE.md").write_text("first\n") + self.commit(self.context, "first") + self.first = git(self.context, "rev-parse", "HEAD") + self.roots = {} + for key in REPOSITORIES: + root = self.repository(key) + (root / "CONTEXT_REVISION").write_text(self.first + "\n") + if key == "coreApi": + (root / "docs").mkdir() + (root / "docs/RULE.md").write_text("first\n") + self.commit(root, "initial") + self.roots[key] = root + (self.context / "docs/RULE.md").write_text("second\n") + self.commit(self.context, "second") + self.second = git(self.context, "rev-parse", "HEAD") + self.manifest = { + "schemaVersion": 1, + "repositories": {key: {"revision": "CONTEXT_REVISION"} for key in REPOSITORIES}, + "mappings": [{"sourceDirectory": "docs", "files": "*.md", "mode": "text", + "targets": [{"repository": "coreApi", "directory": "docs"}]}], + "unmappedContractPrefixes": ["docs/"], "informationalFiles": [], + } + self.bodies = [] + + def repository(self, name): + root = self.root / name + root.mkdir() + remote = self.root / (name + ".git") + remote.mkdir() + git(remote, "init", "--bare", "-q", "--initial-branch=main") + git(root, "init", "-q", "--initial-branch=main") + git(root, "config", "user.name", "Contract test") + git(root, "config", "user.email", "test@example.invalid") + git(root, "remote", "add", "origin", str(remote)) + return root + + def commit(self, root, title): + git(root, "add", "-A") + git(root, "commit", "-qm", title) + git(root, "push", "-q", "origin", "HEAD") + + def github(self, root, owner, key, result): + self.bodies.append(publisher.render_body(result, key)) + return {"number": 7 + REPOSITORIES.index(key), "head": result["heads"][key], "base": result["bases"][key]} + + def publish(self, revision=None): + with patch.object(publisher, "load_map", return_value=self.manifest), \ + patch.object(publisher, "upsert_pr", side_effect=self.github), \ + patch.dict(os.environ, {"GITHUB_RUN_ID": "42", "GITHUB_RUN_ATTEMPT": "1", "GITHUB_SHA": "9" * 40}): + return publisher.publish(self.context, revision or self.second, self.roots, "example") + + def test_docs_only_publish_updates_one_pr_and_pins_all_other_mains(self): + result = self.publish() + self.assertEqual(set(result["pullRequests"]), {"coreApi"}) + self.assertEqual(set(result["heads"]), set(REPOSITORIES)) + self.assertTrue(result["runApiSmoke"], "Normative documents must also run smoke") + self.assertEqual(result["outcome"], "pending") + self.assertEqual(result["contextRevision"], self.second) + self.assertEqual(len(self.bodies), 2) + self.assertIn("[\u00237]", self.bodies[-1]) + self.assertEqual(set(result["changedFiles"]["coreApi"]), {"docs/RULE.md", "CONTEXT_REVISION"}) + for key in REPOSITORIES: + if key != "coreApi": + self.assertEqual(result["heads"][key], result["bases"][key]) + self.assertEqual(git(self.roots["coreApi"], "show", "HEAD:docs/RULE.md"), "second") + + def test_no_snapshot_change_still_records_full_tuple_without_pr(self): + # An informational-only Context change creates no snapshot PRs. + core = self.roots["coreApi"] + (core / "docs/RULE.md").write_text("second\n") + (core / "CONTEXT_REVISION").write_text(self.second + "\n") + self.commit(core, "already synchronized") + (self.context / "AGENTS.md").write_text("informational\n") + self.commit(self.context, "info") + revision = git(self.context, "rev-parse", "HEAD") + result = self.publish(revision) + self.assertEqual(result["pullRequests"], {}) + self.assertEqual(result["heads"], result["bases"]) + self.assertEqual(self.bodies, []) + + def test_multiple_changed_repositories_link_the_whole_set(self): + self.manifest["mappings"][0]["targets"].append({"repository": "web", "directory": "docs"}) + result = self.publish() + self.assertEqual(set(result["pullRequests"]), {"coreApi", "web"}) + for body in self.bodies[-2:]: + self.assertIn("example/core-api/pull/7", body) + self.assertIn("example/web/pull/9", body) + for sha in result["heads"].values(): + self.assertIn(sha, body) + + def test_old_context_commit_is_rejected_before_any_target_push(self): + with self.assertRaisesRegex(ContractSyncError, "main changed"): + self.publish(self.first) + for root in self.roots.values(): + self.assertEqual(git(root, "branch", "--show-current"), "main") + + def test_mutable_context_ref_is_rejected(self): + with self.assertRaisesRegex(ContractSyncError, "full Context"): + self.publish("main") + + def test_dirty_target_fails_before_applying_snapshots(self): + (self.roots["coreApi"] / "user-work.txt").write_text("preserve\n") + with self.assertRaisesRegex(ContractSyncError, "not clean"): + self.publish() + self.assertEqual((self.roots["coreApi"] / "user-work.txt").read_text(), "preserve\n") + + def test_existing_bot_branch_with_business_code_is_rejected(self): + root = self.roots["coreApi"] + git(root, "switch", "-c", publisher.BRANCH) + (root / "business.py").write_text("not a snapshot\n") + self.commit(root, "unexpected code") + git(root, "switch", "main") + with self.assertRaisesRegex(ContractSyncError, "non-snapshot"): + self.publish() + + def test_remote_main_move_during_publishing_aborts_without_pr_writes(self): + original = publisher.assert_remote_main + checks = [] + def check(root, expected): + checks.append(root) + if root == self.roots["coreApi"]: + raise ContractSyncError("main changed during propagation") + original(root, expected) + with patch.object(publisher, "assert_remote_main", side_effect=check): + with self.assertRaisesRegex(ContractSyncError, "main changed"): + self.publish() + self.assertEqual(self.bodies, []) + + def test_changed_pr_head_prevents_body_update(self): + calls = [] + def cli(*args, **kwargs): + calls.append(args) + if args[:3] == ("gh", "pr", "list"): + return json.dumps([{"number": 7}]) + if args[:2] == ("gh", "api"): + return json.dumps({"head": {"sha": "c" * 40, "repo": {"full_name": "example/core-api"}}}) + self.fail(f"Must not write PR: {args}") + # render_body needs the complete tuple; only the API head is changed. + from test_contract_evidence import candidate + result = candidate() + with patch.object(publisher, "run", side_effect=cli): + with self.assertRaisesRegex(ContractSyncError, "head changed"): + publisher.upsert_pr(self.roots["coreApi"], "example", "coreApi", result) + self.assertFalse(any("edit" in call or "create" in call for call in calls)) + + +if __name__ == "__main__": + unittest.main()