From 5483c38279fc95f5f99c330f16f5f32e9eee910e Mon Sep 17 00:00:00 2001 From: jbeckwith-oai <269036200+jbeckwith-oai@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:46:37 +0000 Subject: [PATCH] chore(api): Retire obsolete customization reporting Castiron-Internal-PR: https://github.com/openai/openai-node-internal/pull/400 Castiron-Source-SHA: d7a4bc8b719b8dfe03d9b5ac38f6321e7e0aa23a Castiron-Public-Base-SHA: 4e95fe2bd074edf5337406b28b111141fb3c245a --- .castiron.stats.yml | 7 +- .../castiron-custom-code-comment.yml | 904 ---------- .github/workflows/castiron-custom-code.yml | 116 -- scripts/castiron/CUSTOM_CODE.md | 298 --- scripts/castiron/README.md | 29 - scripts/castiron/custom_code_budget.py | 978 ---------- scripts/castiron/custom_code_policy.py | 731 -------- scripts/castiron/custom_code_report.py | 1534 ---------------- scripts/castiron/custom_code_test_support.py | 116 -- .../castiron/fixtures/github_publisher.cjs | 183 -- scripts/castiron/generated-files.json | 76 - scripts/castiron/ruff.toml | 3 - scripts/castiron/test_custom_code_budget.py | 430 ----- scripts/castiron/test_custom_code_github.py | 1048 ----------- scripts/castiron/test_custom_code_policy.py | 988 ---------- .../test_custom_code_policy_source.py | 198 -- .../castiron/test_custom_code_publication.py | 600 ------- scripts/castiron/test_custom_code_report.py | 1591 ----------------- .../test_custom_code_review_revocation.py | 986 ---------- .../test_custom_code_status_publication.py | 782 -------- scripts/castiron/test_custom_code_strict.py | 774 -------- 21 files changed, 3 insertions(+), 12369 deletions(-) delete mode 100644 .github/workflows/castiron-custom-code-comment.yml delete mode 100644 .github/workflows/castiron-custom-code.yml delete mode 100644 scripts/castiron/CUSTOM_CODE.md delete mode 100644 scripts/castiron/README.md delete mode 100755 scripts/castiron/custom_code_budget.py delete mode 100755 scripts/castiron/custom_code_policy.py delete mode 100755 scripts/castiron/custom_code_report.py delete mode 100644 scripts/castiron/custom_code_test_support.py delete mode 100644 scripts/castiron/fixtures/github_publisher.cjs delete mode 100644 scripts/castiron/ruff.toml delete mode 100644 scripts/castiron/test_custom_code_budget.py delete mode 100644 scripts/castiron/test_custom_code_github.py delete mode 100644 scripts/castiron/test_custom_code_policy.py delete mode 100644 scripts/castiron/test_custom_code_policy_source.py delete mode 100644 scripts/castiron/test_custom_code_publication.py delete mode 100644 scripts/castiron/test_custom_code_report.py delete mode 100644 scripts/castiron/test_custom_code_review_revocation.py delete mode 100644 scripts/castiron/test_custom_code_status_publication.py delete mode 100644 scripts/castiron/test_custom_code_strict.py diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 98712a5783..b29ff02961 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,8 +1,7 @@ schema_version: 1 -generation_id: 2428792f-6df7-44d0-b2c7-92a44663a53d +generation_id: 0f3a113a-8c3e-4be1-8bf2-ac4eed2efb58 openapi_spec_hash: 2c78c708cab402e8904f555026c78020 openapi_transformed_spec_hash: 2231f30a57a7a7afe95cb0d0e305c18d config_hash: cd0aed93d2e3579562e42f5a5e8d2112 -codegen_sha: fe5d1e3d7cd5be6d987dfa5637565092197e5ce9 -codegen_hash: 9ee4dd63f01c9769c3fef3f62a5047ee332b009de2d4e4a5ae58e3036f36195c -public_codegen_sha: f17901f871361f6cdc52c77d0f27e447d26f89c2 +codegen_sha: 71acebef46c022e49845778711ba35231bd0f715 +codegen_hash: 1929596c816dc7ee14d558cfaf310b06a5d8e24d08e155ab38830b8aeb741971 diff --git a/.github/workflows/castiron-custom-code-comment.yml b/.github/workflows/castiron-custom-code-comment.yml deleted file mode 100644 index 74f04360a2..0000000000 --- a/.github/workflows/castiron-custom-code-comment.yml +++ /dev/null @@ -1,904 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -name: Castiron custom code comment - -on: - workflow_run: - workflows: [Castiron custom code] - types: [completed] - -permissions: {} - -concurrency: - group: castiron-custom-code-comment-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }} - cancel-in-progress: false - queue: max # Preserve newer pending evaluations if older runs arrive out of order. - -jobs: - review-change: - name: Invalidate reviewed policy statuses - if: github.event.workflow_run.event == 'pull_request_review' - runs-on: ubuntu-latest - timeout-minutes: 5 - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - permissions: - actions: write - contents: read - pull-requests: read - statuses: write - steps: - - name: Re-evaluate current PR and queue after review changes - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - // Trusted main orchestration only. Signal fields never choose status targets. - const repo = `${context.repo.owner}/${context.repo.repo}`; - const signal = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: signal.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.repository.full_name !== repo || run.event !== 'pull_request_review' || - run.status !== 'completed' || run.id !== signal.id || run.run_attempt !== signal.run_attempt || - run.head_sha !== signal.head_sha || run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') { - throw new Error('Unexpected or superseded review signal'); - } - async function mergeBase(base, head) { - const {data: comparison} = await github.rest.repos.compareCommitsWithBasehead({ - ...context.repo, basehead: `${base}...${head}`, per_page: 1, - }); - const sha = comparison.merge_base_commit?.sha; - if (!/^[0-9a-f]{40}$/.test(sha || '')) throw new Error('Invalid policy merge base'); - return sha; - } - async function requiresHuman(pr, targetSha, mainSha) { - if (!/^[0-9a-f]{40}$/.test(targetSha || '')) throw new Error('Invalid authenticated target'); - if (!/^[0-9a-f]{40}$/.test(mainSha || '')) throw new Error('Invalid authenticated main'); - const path = require('node:path'); - const fs = require('node:fs'); - const root = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP, 'castiron-review-')); - try { - // Bind executable policy and its dependency to the authenticated - // main snapshot for this decision, including after a retry. - for (const name of ['custom_code_policy.py', 'custom_code_report.py']) { - const {data: file} = await github.rest.repos.getContent({ - ...context.repo, path: `scripts/castiron/${name}`, ref: mainSha, - }); - if (file.type !== 'file' || file.encoding !== 'base64' || - !Number.isSafeInteger(file.size) || file.size <= 0 || file.size > 1048576 || - typeof file.content !== 'string') throw new Error('Invalid trusted policy file'); - const content = Buffer.from(file.content, 'base64'); - if (content.length !== file.size) throw new Error('Invalid trusted policy content'); - fs.writeFileSync(path.join(root, name), content, {flag: 'wx'}); - } - const result = await exec.getExecOutput('python3', ['-I', - path.join(root, 'custom_code_policy.py'), 'review-required', - '--repository', repo, '--repo', path.join(root, 'objects.git'), - '--base', targetSha, '--head', pr.head.sha, '--main', mainSha], {silent: true}); - const decision = JSON.parse(result.stdout); - if (typeof decision.requires_human !== 'boolean') throw new Error('Invalid review decision'); - return decision.requires_human; - } finally { - fs.rmSync(root, {recursive: true, force: true}); - } - } - const dirtyTargets = new Set(); - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.ref === run.head_branch && pr.head?.repo?.id === run.head_repository.id && - pr.head?.repo?.full_name === run.head_repository.full_name); - // Retry discovery within this job timeout while targets are changing. - while (true) { - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests?.length ? run.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: run.head_sha}); - let hasAssociation = true; - let matching = []; - for (let attempt = 0; attempt < 2; attempt++) { - const candidates = []; - for (const item of pulls) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: item.number}); - if (pr.state !== 'open' || pr.base.ref !== 'next' || pr.base.repo.full_name !== repo || - (item.head?.sha && pr.head.sha !== item.head.sha) || !/^[0-9a-f]{40}$/.test(pr.head.sha)) continue; - if (!matchesSource(pr)) continue; - if (run.head_sha !== pr.head.sha) { - const {data: commit} = await github.rest.repos.getCommit({...context.repo, ref: run.head_sha}); - if (commit.parents.length !== 2 || commit.parents[1].sha !== pr.head.sha || - !/^[0-9a-f]{40}$/.test(commit.parents[0].sha) || - await mergeBase(commit.parents[0].sha, pr.base.sha) !== commit.parents[0].sha) continue; - // Associated reviews may name an older merge after next advances. - // Branch-only discovery still requires the live merge and base. - if (!hasAssociation) { - const {data: merge} = await github.rest.git.getRef({...context.repo, ref: `pull/${pr.number}/merge`}); - if (merge.object.sha !== run.head_sha || commit.parents[0].sha !== pr.base.sha) continue; - } - } - candidates.push(pr); - } - matching = candidates.length === 1 ? candidates : candidates.filter(pr => - Number.isSafeInteger(run.head_repository?.id) && run.head_repository.id > 0 && - pr.head.repo?.id === run.head_repository.id && typeof run.head_branch === 'string' && - run.head_branch.length > 0 && pr.head.ref === run.head_branch); - if (matching.length || explicitAssociation || attempt) break; - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - hasAssociation = false; - } - if (!matching.length) return; - if (matching.length !== 1) throw new Error('Review signal identifies ambiguous next PRs'); - const pr = matching[0]; - const files = await github.paginate(github.rest.pulls.listFiles, {...context.repo, pull_number: pr.number}); - if (!files.some(file => file.filename === '.castiron-ratchet.json')) return; - const {data: target} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - const targetSha = target.object.sha; - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - const mainSha = main.object.sha; - let requiresReview; - try { - requiresReview = await requiresHuman(pr, targetSha, mainSha); - } catch (error) { - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - if (currentMain.object.sha !== mainSha) continue; - throw error; - } - if (!requiresReview) { - const [{data: current}, {data: target}, main] = await Promise.all([ - github.rest.pulls.get({...context.repo, pull_number: pr.number}), - github.rest.git.getRef({...context.repo, ref: 'heads/next'}), - mainSha && github.rest.git.getRef({...context.repo, ref: 'heads/main'}), - ]); - if (current.state !== 'open' || current.head.sha !== pr.head.sha || current.base.ref !== 'next' || - current.base.repo.full_name !== repo || target.object.sha !== targetSha || - (main && main.data.object.sha !== mainSha)) continue; - return; - } - // Read authoritative reviews, including edits: submittedAt alone misses - // removal/restoration of an attestation. Never trust candidate event artifacts. - let reviewCursor = null; - const latest = new Map(); - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$number:Int!,$cursor:String) { - repository(owner:$owner,name:$name) { pullRequest(number:$number) { - reviews(first:100,after:$cursor) { nodes { fullDatabaseId state body updatedAt - author { login __typename } authorAssociation commit { oid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, number: pr.number, cursor: reviewCursor}); - const page = result.repository.pullRequest.reviews; - for (const review of page.nodes) { - const value = review.fullDatabaseId; - if (!(typeof value === 'string' || Number.isSafeInteger(value)) || - !/^[1-9][0-9]*$/.test(String(value))) throw new Error('Invalid review identity'); - const identity = String(value); - if (['PENDING', 'COMMENTED'].includes(review.state) || !review.author) continue; - const previous = latest.get(review.author.login); - if (!previous || BigInt(identity) > BigInt(previous.identity)) latest.set(review.author.login, {...review, identity}); - } - if (!page.pageInfo.hasNextPage) break; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === reviewCursor) throw new Error('Invalid review cursor'); - reviewCursor = page.pageInfo.endCursor; - } while (true); - // Same v2 whole-body contract as the selector/publisher. This can only - // request reevaluation; the normal publisher still decides authorization. - const approvals = [...latest.values()].filter(review => - review.state === 'APPROVED' && review.commit?.oid === pr.head.sha && - review.author.__typename === 'User' && !review.author.login.endsWith('[bot]') && - review.author.login !== pr.user.login && - ['OWNER', 'MEMBER', 'COLLABORATOR'].includes(review.authorAssociation) && - typeof review.body === 'string' && - review.body.trim() === 'I personally reviewed and approve this policy change as a human: ' + pr.head.sha); - const authorized = approvals.length > 0; - if (authorized && (run.actor?.type !== 'User' || - !approvals.some(review => review.author.login === run.actor.login))) return; - for (const review of approvals) { - if (!Number.isFinite(Date.parse(review.updatedAt))) throw new Error('Missing review update time'); - } - const fingerprint = require('node:crypto').createHash('sha256').update(JSON.stringify( - [pr.head.sha, approvals.map(review => [review.identity, review.updatedAt]).sort((a, b) => a[0].localeCompare(b[0]))] - )).digest('hex'); - const marker = `[review ${authorized ? 'approve' : 'revoke'} ${fingerprint}]`; - async function queue() { - let cursor = null; - const entries = []; - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:"next") { - entries(first:100,after:$cursor) { nodes { headCommit { oid } pullRequest { number headRefOid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, cursor}); - const page = result.repository.mergeQueue?.entries; - if (!page) return []; - entries.push(...page.nodes); - if (!page.pageInfo.hasNextPage) return entries; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === cursor) throw new Error('Invalid queue cursor'); - cursor = page.pageInfo.endCursor; - } while (true); - } - const entries = await queue(); - const index = entries.findIndex(entry => entry.pullRequest?.number === pr.number && entry.pullRequest.headRefOid === pr.head.sha); - // Later queue entries can include the affected constituent. Re-evaluate all of them. - const targets = new Map([[pr.head.sha, 'pull_request']]); - for (const entry of index < 0 ? [] : entries.slice(index)) { - if (entry.headCommit === null) continue; // Queue preparation has not produced statuses yet. - if (!/^[0-9a-f]{40}$/.test(entry.headCommit.oid)) throw new Error('Invalid queue SHA'); - targets.set(entry.headCommit.oid, 'merge_group'); - } - // Revoke the authenticated snapshot before slower reads. Pending on an obsolete - // affected SHA cannot authorize a merge, and a later race must not retain success. - const url = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; - const activeTargets = new Map(); - const priorWriters = new Map(); - async function priorWriter(status) { - if (!/\[review (?:approve|revoke) [0-9a-f]{64}\]$/.test(status.description || '')) return null; - const identity = status.target_url?.slice(`${context.serverUrl}/${repo}/actions/runs/`.length); - if (!/^[1-9][0-9]*$/.test(identity || '') || !Number.isSafeInteger(Number(identity))) return null; - if (!priorWriters.has(identity)) { - const {data: definition} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code-comment.yml', - }); - let writer; - try { - ({data: writer} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: Number(identity)})); - } catch (error) { - if (error.status !== 404) throw error; - priorWriters.set(identity, null); // Deleted runs cannot authenticate a marker. - return null; - } - const authenticated = definition.path === '.github/workflows/castiron-custom-code-comment.yml' && - writer.workflow_id === definition.id && writer.repository.full_name === repo && - writer.event === 'workflow_run' && writer.id === Number(identity); - priorWriters.set(identity, authenticated ? { - completed: writer.status === 'completed' && writer.conclusion === 'success', - } : null); - } - return priorWriters.get(identity); - } - const names = ['Castiron next / budget-only change', 'Castiron next / custom-code budget']; - for (const [sha, event] of targets) { - const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, {...context.repo, ref: sha}); - const trusted = statuses.filter(status => status.creator?.login === 'github-actions[bot]' && - status.target_url?.startsWith(`${context.serverUrl}/${repo}/actions/runs/`)); - const current = new Map(); - for (const status of trusted) if (!current.has(status.context)) current.set(status.context, status); - // Only current pending markers coalesce. A later publisher failure - // supersedes recovery even if a restored review has the same timestamp. - const changed = []; - for (const name of names) { - const status = current.get(name); - // A failed/cancelled writer may have stopped after pending but - // before dispatch. Re-stamp and retry; only successful writers coalesce. - const writer = status?.state === 'pending' ? await priorWriter(status) : null; - const handled = status?.state === 'success' || - (status?.state === 'pending' && status.description?.endsWith(marker) && writer?.completed); - if (authorized ? !handled : - status?.state === 'success' || (writer && !writer.completed)) changed.push(name); - } - if (authorized && names.every(name => current.get(name)?.state === 'success')) continue; - for (const name of changed) { - await github.rest.repos.createCommitStatus({...context.repo, sha, context: name, state: 'pending', - description: `Review changed; reevaluation pending. ${marker}`, target_url: url}); - dirtyTargets.add(sha); - } - if (dirtyTargets.has(sha)) activeTargets.set(sha, event); - } - if (!activeTargets.size) return; - const reruns = []; - for (const [sha, event] of activeTargets) { - const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { - ...context.repo, workflow_id: 'castiron-custom-code.yml', head_sha: sha, event, per_page: 100, - }); - const matches = []; - for (const item of runs) { - if (item.repository.full_name !== repo || item.event !== event || item.head_sha !== sha || - item.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') continue; - if (event === 'pull_request') { - const source = item.head_repository; - if (!source?.id || !source.full_name || !item.head_branch) continue; - const explicitAssociation = Boolean(item.pull_requests?.length); - let associated = item.pull_requests?.length ? item.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: sha}); - const live = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const reference of associated) { - const {data: pull} = await github.rest.pulls.get({...context.repo, pull_number: reference.number}); - if (pull.state === 'open' && pull.head.sha === sha && pull.base.repo.full_name === repo && - pull.head.repo?.id === source.id && pull.head.repo?.full_name === source.full_name && - pull.head.ref === item.head_branch && ['main', 'next'].includes(pull.base.ref)) live.push(pull); - } - if (live.length || explicitAssociation || attempt) break; - associated = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${source.owner.login}:${item.head_branch}`, - }); - } - // Match the canonical reporter's main preference for shared-head discovery. - const main = live.filter(pull => pull.base.ref === 'main'); - const selected = main.length ? main : live; - if (selected.length !== 1 || selected[0].number !== pr.number) continue; - } - matches.push(item); - } - matches.sort((a, b) => b.id - a.id); - if (matches[0]?.status === 'completed') reruns.push(matches[0].id); - } - // Retry against the new snapshot; already-observed affected statuses stay pending. - const {data: current} = await github.rest.pulls.get({...context.repo, pull_number: pr.number}); - const {data: base} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - if (current.state !== 'open' || current.head.sha !== pr.head.sha || current.base.ref !== 'next' || - current.base.repo.full_name !== repo || base.object.sha !== targetSha || - JSON.stringify(await queue()) !== JSON.stringify(entries)) continue; - // Only normal authenticated reporter runs may restore success. Never infer approval here. - for (const run_id of reruns) await github.rest.actions.reRunWorkflow({...context.repo, run_id}); - return; - } - - main-change: - name: Invalidate next queue after main advances - if: github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' - runs-on: ubuntu-latest - timeout-minutes: 5 - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - permissions: - actions: write - contents: read - pull-requests: read - statuses: write - steps: - - name: Re-evaluate live next queue after main advances - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - // Main pushes only invalidate. The normal authenticated evaluator - // and serialized publisher are the only paths that restore success. - const repo = `${context.repo.owner}/${context.repo.repo}`; - const signal = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: signal.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.repository.full_name !== repo || run.head_repository?.full_name !== repo || - run.event !== 'push' || run.head_branch !== 'main' || run.status !== 'completed' || - run.id !== signal.id || run.run_attempt !== signal.run_attempt || run.head_sha !== signal.head_sha || - !/^[0-9a-f]{40}$/.test(run.head_sha) || - run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') { - throw new Error('Unexpected or superseded main signal'); - } - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - // A newer main push has its own signal. An old retry must not churn - // candidates already qualified against that newer snapshot. - if (run.head_sha !== main.object.sha) return; - const targets = new Set(); - let cursor = null; - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }`, {owner: context.repo.owner, name: context.repo.repo, branch: 'next', cursor}); - const page = result.repository.mergeQueue?.entries; - if (!page) break; - for (const entry of page.nodes) { - if (entry.headCommit === null) continue; // Unprepared entries have no statuses to invalidate. - const sha = entry.headCommit?.oid; - if (!/^[0-9a-f]{40}$/.test(sha || '')) throw new Error('Invalid live queue SHA'); - targets.add(sha); - } - if (!page.pageInfo.hasNextPage) break; - const next = page.pageInfo.endCursor; - if (!next || next === cursor) throw new Error('Invalid queue pagination'); - cursor = next; - } while (true); - const url = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; - // Invalidate the entire live next queue before slower rerun discovery. - // This also covers bootstraps whose policy pin is now stale. Ordinary - // pinned-policy candidates can regain success through normal checks. - for (const sha of targets) { - for (const name of ['Castiron next / budget-only change', 'Castiron next / custom-code budget']) { - await github.rest.repos.createCommitStatus({...context.repo, sha, context: name, state: 'pending', - description: `Main advanced; reevaluation pending. [main ${main.object.sha}]`, target_url: url}); - } - } - for (const sha of targets) { - const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { - ...context.repo, workflow_id: 'castiron-custom-code.yml', head_sha: sha, event: 'merge_group', per_page: 100, - }); - const matches = runs.filter(item => item.repository.full_name === repo && - item.workflow_id === workflow.id && item.event === 'merge_group' && item.head_sha === sha && - typeof item.head_branch === 'string' && item.head_branch.startsWith('gh-readonly-queue/next/') && - item.path.split('@', 1)[0] === '.github/workflows/castiron-custom-code.yml' && - Number.isSafeInteger(item.id) && item.id > 0); - matches.sort((a, b) => b.id - a.id); - // A running evaluation will check main at publication. Do not - // restart an older completed run when a newer one is in flight. - if (matches[0]?.status === 'completed') { - await github.rest.actions.reRunWorkflow({...context.repo, run_id: matches[0].id}); - } - } - - compute: - name: Compute trusted custom-code report - if: contains(fromJSON('["pull_request", "merge_group"]'), github.event.workflow_run.event) && (github.event.workflow_run.path == '.github/workflows/castiron-custom-code.yml' || startsWith(github.event.workflow_run.path, '.github/workflows/castiron-custom-code.yml@')) - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - actions: read - pull-requests: read - outputs: - selected-policy-sha: ${{ steps.select-policy.outputs.policy_sha }} - authenticated-main-sha: ${{ steps.select-policy.outputs.authenticated_main_sha }} - main-snapshot: ${{ steps.select-policy.outputs.main_snapshot }} - selected-base-sha: ${{ steps.select-policy.outputs.base_sha }} - selected-head-sha: ${{ steps.select-policy.outputs.head_sha }} - selected-branch: ${{ steps.select-policy.outputs.target_branch }} - selected-bootstrap: ${{ steps.select-policy.outputs.bootstrap }} - policy-reviews: ${{ steps.select-policy.outputs.policy_reviews }} - human-review-rules: ${{ steps.select-policy.outputs.human_review_rules }} - number: ${{ steps.report.outputs.number }} - artifact-id: ${{ steps.artifact.outputs.artifact-id }} - artifact-run-attempt: ${{ github.run_attempt }} - target-branch: ${{ steps.budget.outputs.target_branch }} - policy-sha: ${{ steps.budget.outputs.policy_sha }} - base-sha: ${{ steps.budget.outputs.base_sha }} - head-sha: ${{ steps.budget.outputs.head_sha }} - isolation: ${{ steps.budget.outputs.isolation }} - budget: ${{ steps.budget.outputs.budget }} - steps: - # Selecting main here pins executable checker code and budget policy. - # The reporter separately captures the live main/next target baseline. - - name: Check out the trusted reporter - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: main - persist-credentials: false - - # The selector and its module come from current main. The later evaluator - # loads its own policy helpers from the immutable pinned checkout. - - name: Select reviewed policy from the trusted target base - id: select-policy - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - run: | - trusted_main="$(git rev-parse HEAD)" - printf 'authenticated_main_sha=%s\n' "$trusted_main" >> "$GITHUB_OUTPUT" - python3 -I scripts/castiron/custom_code_budget.py select-policy \ - --repository "$REPOSITORY" --event-path "$GITHUB_EVENT_PATH" \ - --trusted-sha "$trusted_main" \ - --repo "$RUNNER_TEMP/castiron-policy-objects.git" \ - --out "$RUNNER_TEMP/castiron-policy-selection.json" - - - name: Check out the authenticated main policy version - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ steps.select-policy.outputs.policy_sha }} - path: .castiron-policy-checker - persist-credentials: false - - - name: Compute from trusted main and target branch Git objects - id: report - if: github.event.workflow_run.event == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 -I .castiron-policy-checker/scripts/castiron/custom_code_report.py trusted-report \ - --repo "$RUNNER_TEMP/castiron-objects.git" \ - --base "$(git -C .castiron-policy-checker rev-parse HEAD)" \ - --repository "$REPOSITORY" --run-id "$RUN_ID" --run-attempt "$RUN_ATTEMPT" \ - --out "$RUNNER_TEMP/castiron-custom-code" - if test -f "$RUNNER_TEMP/castiron-custom-code/context.json"; then - number=$(jq -er '.pr' "$RUNNER_TEMP/castiron-custom-code/context.json") - printf 'number=%s\n' "$number" >> "$GITHUB_OUTPUT" - cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload the trusted report and patch - id: artifact - if: steps.report.outputs.number != '' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/castiron-custom-code/ - if-no-files-found: error - retention-days: 7 - - # The prior step's report was computed here from - # Git objects by main's reporter, not downloaded from the candidate run. - - name: Evaluate main's custom-code budget - id: budget - if: always() - continue-on-error: true # A budget failure must not suppress the existing report comment. - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - SOURCE_EVENT: ${{ github.event.workflow_run.event }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - run: | - trusted_sha=$(git -C .castiron-policy-checker rev-parse HEAD) - reuse=() - if [[ "$SOURCE_EVENT" == pull_request ]]; then - reuse=(--trusted-report-dir "$RUNNER_TEMP/castiron-custom-code") - fi - python3 -I .castiron-policy-checker/scripts/castiron/custom_code_budget.py github \ - --repository "$REPOSITORY" --event-path "$GITHUB_EVENT_PATH" \ - --trusted-sha "$trusted_sha" --repo "$RUNNER_TEMP/castiron-objects.git" \ - --selected-policy-json "$RUNNER_TEMP/castiron-policy-selection.json" \ - "${reuse[@]}" --out "$RUNNER_TEMP/custom-code-budget" - - - name: Add the budget to the run summary - if: always() - run: | - if test -f "$RUNNER_TEMP/custom-code-budget/summary.md"; then - cat "$RUNNER_TEMP/custom-code-budget/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload trusted budget measurements - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: custom-code-budget-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/custom-code-budget/ - if-no-files-found: error - retention-days: 7 - - budget-status: - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - name: Publish custom-code budget checks - needs: compute - if: always() && !cancelled() && needs.compute.result != 'skipped' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - actions: read - pull-requests: read - statuses: write - steps: - # This workflow definition is from main. No candidate checkout/artifacts. - - name: Publish exact-head statuses after checking freshness - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - SELECTED_POLICY_SHA: ${{ needs.compute.outputs.selected-policy-sha }} - AUTHENTICATED_MAIN_SHA: ${{ needs.compute.outputs.authenticated-main-sha }} - MAIN_SNAPSHOT: ${{ needs.compute.outputs.main-snapshot }} - SELECTED_BASE_SHA: ${{ needs.compute.outputs.selected-base-sha }} - SELECTED_HEAD_SHA: ${{ needs.compute.outputs.selected-head-sha }} - SELECTED_BRANCH: ${{ needs.compute.outputs.selected-branch }} - SELECTED_BOOTSTRAP: ${{ needs.compute.outputs.selected-bootstrap }} - POLICY_REVIEWS: ${{ needs.compute.outputs.policy-reviews }} - HUMAN_REVIEW_RULES: ${{ needs.compute.outputs.human-review-rules }} - TARGET_BRANCH: ${{ needs.compute.outputs.target-branch }} - POLICY_SHA: ${{ needs.compute.outputs.policy-sha }} - BASE_SHA: ${{ needs.compute.outputs.base-sha }} - HEAD_SHA: ${{ needs.compute.outputs.head-sha }} - ISOLATION_RESULT: ${{ needs.compute.outputs.isolation }} - BUDGET_RESULT: ${{ needs.compute.outputs.budget }} - PUBLISH_ATTEMPT: ${{ github.run_attempt }} - with: - script: | - const event = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: event.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.event !== event.event || run.head_sha !== event.head_sha || run.run_attempt !== event.run_attempt || - run.status !== 'completed' || run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml' || - run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.sha === run.head_sha && pr.head?.ref === run.head_branch && - pr.head?.repo?.id === run.head_repository.id && pr.head?.repo?.full_name === run.head_repository.full_name); - const head = run.head_sha; - if (!/^[0-9a-f]{40}$/.test(head)) throw new Error('Invalid candidate SHA'); - const base = process.env.BASE_SHA; - const policy = process.env.POLICY_SHA; - const evaluatedBranch = process.env.TARGET_BRANCH; - let branch, number; - let fresh = /^[0-9a-f]{40}$/.test(base) && /^[0-9a-f]{40}$/.test(policy) && head === process.env.HEAD_SHA; - if (run.event === 'pull_request') { - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests.length ? run.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: head}); - const current = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const pull of pulls) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: pull.number}); - if (pr.state === 'open' && matchesSource(pr) && - ['main', 'next'].includes(pr.base.ref) && pr.base.repo.full_name === `${context.repo.owner}/${context.repo.repo}`) current.push(pr); - } - if (current.length || explicitAssociation || attempt) break; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - } - // Match compute's preference for required main checks when - // fallback discovery finds an advisory next PR for the same head. - const main = current.filter(pr => pr.base.ref === 'main'); - const selected = main.length ? main : current; - if (selected.length !== 1) return; - branch = selected[0].base.ref; - number = selected[0].number; - } else { - const match = run.head_branch.match(/^gh-readonly-queue\/(main|next)\//); - if (run.event !== 'merge_group' || !match) return; - branch = match[1]; - } - // Bind every target to the trusted selector, including next-to-main retargeting. - fresh = fresh && process.env.SELECTED_BRANCH === branch && - process.env.SELECTED_BASE_SHA === base && process.env.SELECTED_HEAD_SHA === head && - process.env.SELECTED_POLICY_SHA === policy; - fresh = fresh && evaluatedBranch === branch; - const bootstrap = process.env.SELECTED_BOOTSTRAP; - fresh = fresh && (bootstrap === 'false' || (branch === 'next' && bootstrap === 'true')); - const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - // Concurrency serializes writes, but does not order runs or retries. - // Keep an older evaluation from overwriting a newer result for this head. - const order = [run.id, run.run_attempt, Number(context.runId), Number(process.env.PUBLISH_ATTEMPT)]; - const marker = `[evaluation ${order.join(':')}]`; - const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, {...context.repo, ref: head}); - const prefix = branch === 'next' ? 'Castiron next' : 'Castiron'; - const names = [`${prefix} / budget-only change`, `${prefix} / custom-code budget`]; - for (const status of statuses) { - if (!names.includes(status.context) || status.creator?.login !== 'github-actions[bot]') continue; - const prior = status.description?.match(/\[evaluation (\d+):(\d+):(\d+):(\d+)\]$/); - if (!prior || status.target_url !== `${url.slice(0, url.lastIndexOf('/') + 1)}${prior[3]}`) continue; - const previous = prior.slice(1).map(Number); - const different = previous.findIndex((value, index) => value !== order[index]); - if (different !== -1 && previous[different] > order[different]) return; - } - // Selection is not a lasting approval. Recheck live reviews after - // compute and immediately before attaching required statuses. - if (branch === 'next') { - try { - const proofs = JSON.parse(process.env.POLICY_REVIEWS); - const rules = JSON.parse(process.env.HUMAN_REVIEW_RULES); - if (!Array.isArray(proofs) || rules.version !== 2 || !Array.isArray(rules.associations) || - typeof rules.negative_pattern !== 'string' || typeof rules.attestation_prefix !== 'string' || - !rules.attestation_prefix) throw new Error('Missing trusted review contract'); - const negative = new RegExp(rules.negative_pattern, 'i'); - const seen = new Set(); - for (const proof of proofs) { - if (!Number.isSafeInteger(proof.number) || proof.number <= 0 || !/^[0-9a-f]{40}$/.test(proof.head_sha) || - seen.has(proof.number) || (run.event === 'pull_request' && (proof.number !== number || proof.head_sha !== head))) { - throw new Error('Invalid policy review receipt'); - } - seen.add(proof.number); - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: proof.number}); - if (pr.state !== 'open' || pr.head.sha !== proof.head_sha || pr.base.ref !== 'next' || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) throw new Error('Policy PR changed'); - let reviewCursor = null; - const latest = new Map(); - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$number:Int!,$cursor:String) { - repository(owner:$owner,name:$name) { pullRequest(number:$number) { - reviews(first:100,after:$cursor) { nodes { fullDatabaseId state body - author { login __typename } authorAssociation commit { oid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, number: proof.number, cursor: reviewCursor}); - const page = result.repository.pullRequest.reviews; - for (const review of page.nodes) { - const value = review.fullDatabaseId; - if (!(typeof value === 'string' || Number.isSafeInteger(value)) || - !/^[1-9][0-9]*$/.test(String(value))) throw new Error('Invalid review identity'); - const identity = String(value); - if (['PENDING', 'COMMENTED'].includes(review.state) || !review.author) continue; - const previous = latest.get(review.author.login); - if (!previous || BigInt(identity) > BigInt(previous.identity)) latest.set(review.author.login, {...review, identity}); - } - if (!page.pageInfo.hasNextPage) break; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === reviewCursor) throw new Error('Invalid review cursor'); - reviewCursor = page.pageInfo.endCursor; - } while (true); - const approved = [...latest.values()].some(review => review.state === 'APPROVED' && - review.commit?.oid === proof.head_sha && review.author.__typename === 'User' && - rules.associations.includes(review.authorAssociation) && !review.author.login.endsWith('[bot]') && - review.author.login !== pr.user.login && typeof review.body === 'string' && - review.body.trim() === rules.attestation_prefix + proof.head_sha && - !negative.test(review.body)); - if (!approved) throw new Error('Human policy approval is no longer valid'); - } - } catch { fresh = false; } - } - // Recheck after pagination, immediately before attaching statuses. - if (run.event === 'pull_request') { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state !== 'open' || !matchesSource(pr) || pr.base.ref !== branch || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - } else { - // A removed/rebuilt queue entry must not receive a passing result. - let cursor = null, member = false; - do { - const response = await github.graphql(`query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }`, {owner: context.repo.owner, name: context.repo.repo, branch, cursor}); - const entries = response.repository.mergeQueue?.entries; - if (!entries) break; - member = entries.nodes.some(entry => entry.headCommit?.oid === head); - if (member || !entries.pageInfo.hasNextPage) break; - const next = entries.pageInfo.endCursor; - if (!next || next === cursor) throw new Error('Invalid queue pagination'); - cursor = next; - } while (true); - const {data: target} = await github.rest.git.getRef({...context.repo, ref: `heads/${branch}`}); - fresh = fresh && base === target.object.sha && (branch === 'next' || policy === target.object.sha); - fresh = fresh && member; - } - if (branch === 'next' && run.event === 'pull_request') { - const {data: target} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - fresh = fresh && base === target.object.sha; - } - // Main pushes invalidate next evaluations, even when their reviewed - // checker pin stays unchanged. Do not restore success from a run - // computed before that invalidation. This is the reporter checkout, - // not the independently reviewed (and possibly older) policy pin. - if (branch === 'next') { - const authenticatedMain = process.env.AUTHENTICATED_MAIN_SHA || ''; - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && /^[0-9a-f]{40}$/.test(authenticatedMain) && - currentMain.object.sha === authenticatedMain; - } - // Bootstrap may adopt only the still-current authenticated main tree. - // This value comes from our selector, never from candidate artifacts. - if (process.env.MAIN_SNAPSHOT) { - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && branch === 'next' && process.env.MAIN_SNAPSHOT === policy && - currentMain.object.sha === policy; - } - // Bootstrap authorization was evaluated against authenticated main, - // carried in SELECTED_POLICY_SHA. A newer main requires reevaluation. - if (branch === 'next' && bootstrap === 'true') { - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && policy === main.object.sha; - } - for (const [name, result] of [ - [names[0], process.env.ISOLATION_RESULT], - [names[1], process.env.BUDGET_RESULT], - ]) { - const state = fresh && result === 'success' ? 'success' : 'failure'; - const description = !fresh ? 'Evaluation unavailable or queue base changed; inspect the trusted run and rerun.' - : `${state === 'success' ? 'Passed' : 'Failed'}: ${branch} ${base.slice(0, 12)}, policy ${policy.slice(0, 12)}.`; - await github.rest.repos.createCommitStatus({...context.repo, sha: head, context: name, - state, description: `${description} ${marker}`, target_url: url}); - } - - comment: - name: Update custom-code comment - needs: compute - if: always() && !cancelled() && github.event.workflow_run.event == 'pull_request' && (needs.compute.result == 'failure' || needs.compute.outputs.number != '') - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - actions: read - pull-requests: write - steps: - - name: Check out the trusted publisher - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Download this workflow's trusted report - if: needs.compute.result == 'success' - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - artifact-ids: ${{ needs.compute.outputs.artifact-id }} - merge-multiple: true - path: ${{ runner.temp }}/castiron-custom-code - - - name: Create or update the single report comment - id: publish - if: needs.compute.result == 'success' - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - PR_NUMBER: ${{ needs.compute.outputs.number }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - ARTIFACT_RUN_ID: ${{ github.run_id }} - ARTIFACT_RUN_ATTEMPT: ${{ needs.compute.outputs.artifact-run-attempt }} - run: | - python3 -I scripts/castiron/custom_code_report.py comment \ - --report "$RUNNER_TEMP/castiron-custom-code/report.json" \ - --repository "$REPOSITORY" --pr "$PR_NUMBER" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" \ - --artifact-run-id "$ARTIFACT_RUN_ID" --artifact-run-attempt "$ARTIFACT_RUN_ATTEMPT" - - - name: Publish a trusted failure status - if: always() && !cancelled() && steps.publish.outcome != 'success' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const marker = ''; - const event = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: event.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.id !== event.id || run.event !== 'pull_request' || run.event !== event.event || - run.head_sha !== event.head_sha || run.run_attempt !== event.run_attempt || run.status !== 'completed' || - run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml' || - run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.sha === run.head_sha && pr.head?.ref === run.head_branch && - pr.head?.repo?.id === run.head_repository.id && pr.head?.repo?.full_name === run.head_repository.full_name); - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests?.length ? run.pull_requests : await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: run.head_sha}); - const current = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const number of new Set(pulls.map(pull => pull.number))) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state === 'open' && matchesSource(pr) && ['main', 'next'].includes(pr.base.ref) && - pr.base.repo.full_name === `${context.repo.owner}/${context.repo.repo}`) current.push({number, branch: pr.base.ref}); - } - if (current.length || explicitAssociation || attempt) break; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - } - const main = current.filter(pr => pr.branch === 'main'); - const selected = main.length ? main : current; - if (selected.length !== 1) return; - const {number, branch} = selected[0]; - const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: number}); - const previous = comments.find(c => c.user?.type === 'Bot' && c.user?.login === 'github-actions[bot]' && c.body?.startsWith(marker)); - const prior = previous?.body?.match(//); - if (prior && (Number(prior[1]) > run.id || (Number(prior[1]) === run.id && Number(prior[2]) > run.run_attempt))) return; - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state !== 'open' || !matchesSource(pr) || pr.base.ref !== branch || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${run.id}`; - const body = `${marker}\n\n## Castiron custom code\n\n⚠️ Report unavailable for \`${run.head_sha.slice(0, 12)}\`.\n\nThe report setup or validation failed. [Inspect the workflow run](${url}).\n\n`; - if (previous) await github.rest.issues.updateComment({...context.repo, comment_id: previous.id, body}); - else await github.rest.issues.createComment({...context.repo, issue_number: number, body}); diff --git a/.github/workflows/castiron-custom-code.yml b/.github/workflows/castiron-custom-code.yml deleted file mode 100644 index afe1536098..0000000000 --- a/.github/workflows/castiron-custom-code.yml +++ /dev/null @@ -1,116 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -name: Castiron custom code - -on: - push: - branches: [main] - pull_request: - types: [opened, synchronize, reopened, ready_for_review, edited] - pull_request_review: - types: [submitted, edited, dismissed] - # Notify the existing trusted handler for queue candidates. - merge_group: - types: [checks_requested] - -permissions: - contents: read - -concurrency: - group: castiron-custom-code-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref }} - cancel-in-progress: false - -env: - REPORTER_SHA256: ddb15c9a162d26a6c9abf1b4fefcd80f2e685d9d255af729863258b7ecd08e2d - -jobs: - queue-signal: - if: github.event_name == 'merge_group' || github.event_name == 'pull_request_review' || github.event_name == 'push' - runs-on: ubuntu-latest - timeout-minutes: 1 - permissions: {} - steps: - - run: echo 'The workflow_run handler on main evaluates this candidate independently.' - - # Candidate tests run on a separate runner and never precede a token-bearing step. - candidate-tests: - if: github.event_name == 'pull_request' - name: Castiron / reporter tests - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - name: Check out the pull request - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - - name: Test snapshot isolation and the custom-code budget - run: python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' - - report: - if: github.event_name == 'pull_request' - name: Castiron / baseline consistency - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Record the pull request - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - run: | - mkdir -p "$RUNNER_TEMP/castiron-custom-code" - jq -n --argjson pr "$PR_NUMBER" --arg repository "$REPOSITORY" \ - --argjson run "$RUN_ID" --argjson attempt "$RUN_ATTEMPT" \ - '{pr: $pr, repository: $repository, run: $run, attempt: $attempt}' \ - > "$RUNNER_TEMP/castiron-custom-code/context.json" - - - name: Check out the pull request - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Verify the reporter matches its generated workflow - run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict - - - name: Validate the codegen hash and report custom code - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} - PUBLIC_REPOSITORY: ${{ !github.event.repository.private }} - run: | - git fetch --quiet --no-tags origin "$BASE_SHA" "$HEAD_SHA" - mode=() - if [[ "$PUBLIC_REPOSITORY" == true ]]; then mode=(--public); fi - python3 -I scripts/castiron/custom_code_report.py report \ - --base "$BASE_SHA" --head "$HEAD_SHA" --target-branch "$TARGET_BRANCH" \ - --fetch --require-head-hash "${mode[@]}" \ - --out "$RUNNER_TEMP/castiron-custom-code" - - - name: Add the report to the run summary - if: always() - run: | - if test -f "$RUNNER_TEMP/castiron-custom-code/summary.md"; then - cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload report and current custom-code patch - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/castiron-custom-code/ - if-no-files-found: error - retention-days: 7 diff --git a/scripts/castiron/CUSTOM_CODE.md b/scripts/castiron/CUSTOM_CODE.md deleted file mode 100644 index 973ce72812..0000000000 --- a/scripts/castiron/CUSTOM_CODE.md +++ /dev/null @@ -1,298 +0,0 @@ - - -# Custom code - -The custom-code reporter measures the SDK's remaining customization of generated -files. The budget gate checks that measurement against the repository's policy. -See [CONTRIBUTING.md](../../.github/CONTRIBUTING.md#custom-code-budget) for budget changes -and human review, and [AGENTS.md](../../AGENTS.md#custom-code-budget) for agent rules. - -## What is counted - -Repositories may separately activate `"ratchet": "go"` in their policy after -this checker reaches trusted main. Activation must be a human-reviewed, -policy-only PR; SDK promotion does not carry policy edits. Keep the existing -`max_custom_patch_lines` ceiling. - -The Go ratchet rejects net growth in generated-owned custom patch additions -plus deletions **and physical lines of handwritten Go** compared with the -captured live target. It includes helpers, tests, comments, blank lines, nested -modules, and SDK-owned CI Go retained in private checkpoints. Non-text and -nonregular handwritten Go fail validation. Both revisions use their own verified -generated snapshots, read as Git objects without executing candidate code. - -Stale PRs compare with the live target, not their historical merge base. Merge -queues check the complete combined candidate. Every landed reduction becomes -the next baseline automatically. An activated ratchet cannot be disabled; -raising the independent patch-only ceiling does not restore retired headroom. -Existing policy isolation, human review, and snapshot ownership rules still apply. - -The budget is **added lines plus deleted lines in the entire current custom patch** -against the candidate's verified pure-generated snapshot. It is not the PR's diff -against main, and additions and deletions never cancel. Replacing 500 generated -lines with 500 handwritten lines costs 1,000 lines. Deleting a generated file -counts all its removed lines. Restoring generated content reduces the budget. - -The checker reuses `scripts/castiron/custom_code_report.py` without changing its -snapshot/hash validation or generated-file ownership. It counts generated-owned -files, including runtime code, tests, and generated documentation. Wholly -handwritten-only files are outside this existing report's scope. Changes in -generation ownership remain visible in the report; do not change exclusions or -checkpoints to hide customization. Non-text custom patches fail as uncountable. - -The ceiling is defined in [`.castiron-ratchet.json`](../../.castiron-ratchet.json). -Migration policy contains an integer `schema_version` of 1 and a -nonnegative integer `max_custom_patch_lines`. Strict schema v2 is described below. -Missing, deleted, renamed, executable, symlinked, -malformed, or unsupported policy files fail closed. There is no disable flag or -automatic budget update command. - -Budget-file isolation is checked against the complete PR diff, not just its latest -commit. Increases never apply to their own PR or merge group. Decreases must also -fit the measured usage. Human approval is contributor and agent policy plus normal -review, not an automated claim that GitHub can identify who operated an account. - -## Trusted CI - -### Strict generated-file ownership - -Schema v1 remains migration mode with its existing patch budget. An SDK with -zero customizations can opt into schema v2 through a policy-only PR: - -```json -{ "schema_version": 2, "generated_files": "strict", "max_custom_patch_lines": 0 } -``` - -First land shared compiler/verifier support and regenerate the SDK, including -`scripts/castiron/generated-files.json`. Then activate strict policy. Both the -main policy and a proposed activation are enforced; strict policy cannot be -downgraded. A policy change cannot be combined with SDK or generated-source edits. -SDK promotion rejects policy-file changes; submit the policy-only PR directly -in the SDK repository. - -Strict mode compares generated paths, final file bytes, and Git modes against -the manifest in the verified generated snapshot. The candidate's manifest is -also checked exactly. Header removal, deletion, renaming, symlink substitution, -or manifest edits cannot exempt a generated file. Generated workflow/action -hashes are covered by the manifest even though the legacy patch counter and -public snapshot publication continue to exclude those subtrees. Handwritten -files and repository-owned governance remain outside generated ownership. -Private checkpoints retain SDK-owned CI workflows and actions from their parent; -the generated ownership inventory does not claim those inherited files. - -The existing required `Castiron / custom-code budget` status fails on any strict -violation. Its trusted-main execution and merge-queue checks remain mandatory. -Local generation validates before integration and publication using compiler-owned -code. New generated snapshots still require the existing generation/promotion -provenance and review controls; a self-computed hash is not approval to publish. - -Agents must fix inputs/compiler code or handwritten extensions, regenerate, and -verify. Never patch generated files or the manifest, disable strict mode, change -ownership exclusions, or rewrite provenance to get a passing result. Verification -failure must preserve edits for inspection rather than overwrite them. - -The custom-code workflow pair separates candidate execution from trusted checks: - -- `castiron-custom-code.yml` runs proposed offline tests and the advisory report - on `pull_request` with read-only permissions. -- `castiron-custom-code-comment.yml` handles `workflow_run` from **main**. Its - read-only selector authenticates the target base and selects a main commit. - Main uses its captured current commit. Next uses the reviewed `next_policy` - version stored in its protected target base. A second checkout executes only - that authenticated main version of the reporter/checker and reads its policy. - It independently resolves the live PR target (`main` or `next`) and captures - the target baseline and candidate SHA. For main, the target baseline is the - pinned main checkout; for next, it is the live next ref, not stale PR metadata. - It reads only Git objects in a fresh bare repository, never candidate code. -- An unprivileged `merge_group` job in the first workflow signals that a candidate - needs checking. The trusted handler resolves the target from the queue ref, - requires an effective merge-queue rule on that target, and verifies current - queue membership and target ancestry. It checks isolation against the target - per constituent PR and budgets the complete candidate using trusted main policy. - A policy increase in next or the same group cannot grant a higher limit. - -A separate publisher with no checkout attaches these statuses only to PRs and -merge groups targeting `main`, after rechecking the PR head/target or queue base freshness: - -- `Castiron / budget-only change` -- `Castiron / custom-code budget` - -Next uses separate `Castiron next / budget-only change` and -`Castiron next / custom-code budget` contexts, so identical commit SHAs on the -two tracks cannot overwrite each other's authorization. The policy version is -fixed by next's protected base; advancing main does not silently change it. -Changing the reviewed pin changes next's own base and rebuilds its merge queue. -Review invalidation reads the effective checker's inert `PIN_REVIEW_POLICY` -capability marker at that exact revision. Legacy or unknown versions retain -human review for every pin edit; only the recognized effective-policy contract -allows non-increasing forward pin changes to use ordinary review. No checker -code is executed by the invalidation handler. - -The effective policy is read from the selected authenticated main commit. The -next base selects that version; a candidate never selects its own executing -checker or grants itself a larger budget. Target baseline, trusted policy, and candidate identities are recorded -separately in the report and budget result. Reusing a report requires matching -branch, policy, and head identities; retargeted PRs fail closed. - -PR results describe a captured snapshot even if a branch moves during computation. -They do not authorize merging without queue validation. Queue checks require -current target baseline and its approved policy version (current main for main, -reviewed pin for next). Immediately before publishing, the handler rechecks the -PR head/target or queue membership and the bound target baseline/policy identities. A stale queue evaluation cannot publish success. Missing -snapshots, invalid hashes, unavailable membership, and policy errors fail closed. - -The queue signal cannot supply a passing result, and removing it leaves required -statuses missing. No write-capable job executes from the candidate's workflow -definition. See GitHub's [workflow_run trust model](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) -and [merge-queue behavior](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). - -The trusted run summary reports additions, deletions, total, mixed-file count, -headroom, largest patches, and exact policy/candidate/generated revisions. The -existing custom-code comment remains unchanged, including when the budget fails. -The trusted compute job reuses its own report, never the candidate's artifacts. -The checker, workflows, and offline tests are generated from shared Castiron -templates. The budget policy remains repository-owned and is never generated. -Repository-specific customizations are preserved through the normal three-way -merge during generation. - -## Local verification - -Run from the SDK repository with Python 3.10+; no SDK dependencies are needed: - -```sh -python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' -``` - -Use a **trusted checkout's** script to inspect committed revisions: - -```sh -python3 -I scripts/castiron/custom_code_budget.py check \ - --repo /path/to/sdk --base FULL_MAIN_SHA --head FULL_CANDIDATE_SHA \ - --public --fetch --out /tmp/custom-code-budget -``` - -The inspected repo's `origin` must match its snapshot source. Omit `--public` -for the private SDK repository's checkpoints. Local checks are diagnostic and -use the supplied base as policy; CI independently binds the target, main policy, -and candidate to live GitHub metadata. - -## Activation order - -1. Merge the initial budget policy in its budget-only PR after human review. -2. Merge the tooling and guidance separately. Do not enable required - statuses before the policy and trusted checker exist on main. There is no - permanent bootstrap exemption in the checker. -3. Exercise a passing PR, over-budget PR, mixed budget/code PR, a fork PR, and a - merge-queue candidate. Policy increases must still use the old limit. -4. Have a human administrator retain/enable **require merge queue** and add both - exact status names above to the main ruleset, bound to GitHub Actions. Retain - normal review and CODEOWNERS settings. Exercise a main-policy decrease after - an unchanged PR head passed, and confirm its queued candidate uses the new limit. - The tests job is separate from these authoritative statuses. - -This is an accidental-change guardrail, not an adversarial approval boundary. -Workflow definitions and ruleset changes still have the normal repository review -and administrator controls; GitHub Actions status names alone are not a unique -workflow identity. Stronger organization-required workflow enforcement can be -added later. - -## Activating next - -Land shared tooling in Castiron first, then generate and review its SDK update -onto main. Add a reviewed policy pin in a separate `.castiron-ratchet.json`-only -PR targeting next; preserve the existing budget/schema fields: - -```json -"next_policy": { - "repository": "openai/your-sdk", - "main_sha": "" -} -``` - -The SHA must belong to that same repository's authenticated main first-parent -history and support this pin protocol. Selection reads the protected target -base, never the candidate. Pin-only updates and initial pin adoption use ordinary -CODEOWNERS and required reviews when the pin does not roll back to an older main -checker, the effective pinned budget stays the same or decreases, and its schema, -strictness and ratchet mode stay unchanged. Initial adoption compares checker age -and effective policy with authenticated bootstrap main, not the unpinned target's -local policy fields; later updates compare with the current protected -target pin and effective policy, even for stale PRs. A -checker rollback, an effective pinned-budget increase or mode transition, a local numeric budget increase, or Go -ratchet activation still requires an exact-head human approving review from a -repository owner, member, or collaborator. Bot reviews, stale reviews and dismissed -approvals do not authorize those changes. - -For those human-required changes, the approving reviewer must explicitly attest that -they personally performed the human review. The entire review body must be the -following sentence, substituting the exact full current head SHA: - -```text -I personally reviewed and approve this policy change as a human: <40-character head SHA> -``` - -Blank reviews, ordinary code-review text, quoted attestations, and attestations for -an earlier head do not qualify. Agents must never submit this attestation, even -through a human account. This is an explicit reviewer attestation, not a claim that -GitHub account type or team membership proves who operated the account. Ordinary -SDK code reviews continue to accept automation under their normal repository gates. - -For initial adoption only, a base with no pin may accept an exact policy-only -pin-initialization PR under authenticated current-main checker/policy, with the -same effective-budget and mode review requirements. The proposed pin is validated but never executed -for its own initialization. A malformed base pin cannot enter this path; other -changes without a base pin fail closed. Ordinary pin updates execute under the -previous approved version and cannot include SDK/tooling changes. - -An existing unpinned `next` may instead adopt the complete authenticated current-main -snapshot when next is an ancestor of main and main already uses strict ownership -with zero custom lines. The canonical `sdk prepare-next-baseline` command preserves -all main bytes, modes and existing public checkpoints, adding only the exact -same-main policy pin. This exception cannot include additional edits, a different -pin, a relaxed policy or multiple queue constituents. Ordinary checkpoint, manifest, -zero-budget and repository review checks remain mandatory. Main must still match -at final publication; if it moves, prepare and qualify a fresh candidate. This is -adoption of already-public output, not regeneration or a budget increase. - -Keep required status activation separate until hosted tests have exercised main -and next PRs, forks, retargeting, policy changes, and both merge queues. Require -the target-specific next contexts after observing their genuine producers; -never reuse main's context names on next. Establish missing ordinary CI triggers -through a reviewed trusted workflow path before the policy-only initialization. -Preserve reviews, CODEOWNERS, merge queues, and all existing main protections. -This policy contract does not enable package releases from next. - -For `next` policy activations and budget increases, the trusted computation also -records the exact constituent PR numbers and head SHAs requiring human approval. -The isolated status publisher rechecks those live PRs and their latest reviews -immediately before publication. Dismissed, replaced, stale, automated or missing -approvals fail closed even if ordinary automated approvals keep the queue live. -The publisher consumes the selector's versioned review rules, with cross-runtime -predicate fixtures; candidate artifacts never supply authorization or outcomes. - -### Review changes after publication - -The unprivileged reporter workflow signals submitted, edited and dismissed reviews. -For a current next PR changing the policy file, the trusted main handler marks its -required budget statuses and affected live queue statuses pending, then requests -normal reporter reruns. Only the ordinary authenticated evaluator may restore -success; the review signal never selects executable code, approval rules or an -arbitrary status target. Fork runs with empty PR lists use authenticated commit -associations and open-PR lookup, retaining live head/merge binding. The handler -marks each authenticated snapshot pending before slow run pagination, then retries -discovery if the PR or queue changes. Pending statuses on superseded affected -SHAs cannot authorize a merge; observed successes stay revoked even if a later -lookup fails. Reporter reruns must resolve to this same PR under the canonical -main-preference rule, not merely share its head SHA. Missing or ambiguous reporter -runs remain pending. Status writers share a serialization group. - -Retain native required approving reviews and CODEOWNERS at merge time. Review-event -delivery and reevaluation are asynchronous; this mechanism is not an atomic human -approval check at merge. In particular, automated reviews using human accounts do -not become human policy authorization. Do not weaken native gates or treat this -handler as their replacement. A failed invalidation or rerun must be investigated; -missing reporter runs remain pending until the normal trusted run is requested. -Install and validate the review signal on the protected target before activating -next policy checks, including actual PR and queue revocation after dismissal. - -The trusted selector also checks this exact active-checker contract before granting a pin-only review exemption. Legacy or unrecognized contracts retain the required exact-head review in the selection receipt, so the final publisher revalidates it even without a review-change event. diff --git a/scripts/castiron/README.md b/scripts/castiron/README.md deleted file mode 100644 index c445a641b3..0000000000 --- a/scripts/castiron/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Castiron custom-code reporting - -Castiron maintains shared templates for these files. Prefer changing those templates -for cross-SDK improvements; repository-specific customizations use the normal -three-way merge and are allowed. -The reporter uses Python 3.10+, Git, and `gh`; it does not import SDK code. - -Run `python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py'` -for the offline suite. Install Node.js to exercise the workflow publishers too. -See [CUSTOM_CODE.md](CUSTOM_CODE.md) for budget policy and activation. -The report comment includes commands to inspect the exact custom-code patch. -Public reporting uses only public snapshots and needs no private repository access. - -The workflow validates the recorded `codegen_hash`. -Its hash format is documented in the reporter. Only `.github/actions/` and -`.github/workflows/` are excluded from the content hash. - -The read-only pull-request workflow runs on every branch, including drafts and -forks. A separate read-only `workflow_run` job computes the authoritative report from -the captured main checkout and GitHub-associated PR head. Merge groups are -checked independently against current main. It fetches those objects into a new bare repository and never checks -out or executes PR code. The comment-writing job consumes only the artifact -from that trusted job, rechecks freshness, and links to its report and patch. -PR-produced reports are advisory run output, not the published assessment. The -publisher becomes active once its workflow is on the default branch. No branch -allowlist or repository variable is needed. Never execute PR-controlled code with -write credentials. Changing either workflow may require one-time AM permission. diff --git a/scripts/castiron/custom_code_budget.py b/scripts/castiron/custom_code_budget.py deleted file mode 100755 index 88b8085329..0000000000 --- a/scripts/castiron/custom_code_budget.py +++ /dev/null @@ -1,978 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""SDK custom-code budget gate. Run only from a trusted checkout, never PR code. - -Reuses Castiron's vendored snapshot verifier and generated-file accounting. This -file and its workflows are generated from shared Castiron templates. -""" - -from __future__ import annotations - -import argparse -import ast -import hashlib -import html -import importlib.util -import json -import os -import subprocess -import sys -from pathlib import Path -from typing import Any, cast - -# Load only the policy helpers beside this immutable checker, including under -I. -_spec = importlib.util.spec_from_file_location( - "castiron_budget_policy", Path(__file__).with_name("custom_code_policy.py") -) -assert _spec is not None and _spec.loader is not None -policy = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = policy -_spec.loader.exec_module(policy) -report = policy.report -unique_keys = policy.unique_keys -read_policy = policy.read_policy -changed_paths = policy.changed_paths -queued_entries = policy.queued_entries -is_main_snapshot_bootstrap = policy.is_main_snapshot_bootstrap - -POLICY = ".castiron-ratchet.json" -NEXT_POLICY_VERSION = 1 -ISOLATION_MESSAGE = ( - "Budget changes require a separate, budget-only PR. " - "Put the justification in the PR description." -) - - -def read_budget(repo: Path, revision: str) -> int: - return cast(int, read_policy(repo, revision)["max_custom_patch_lines"]) - - -def strict_policy(repo: Path, revision: str) -> bool: - return cast(int, read_policy(repo, revision)["schema_version"]) == 2 - - -def go_ratchet(repo: Path, revision: str) -> bool: - return cast(bool, read_policy(repo, revision).get("ratchet") == "go") - - -MANIFEST = b"scripts/castiron/generated-files.json" -# Accept the relocated tooling only from the verified checkpoint. Deploy this -# reader before changing the producer so trusted-main checks can validate moves. -MANIFEST_PATHS = (MANIFEST, b"generated/castiron/generated-files.json") -# Derived from the renderer's repository-file inventory, also used by checkpoint -# publication to distinguish generated tooling from protected SDK-owned CI. -GENERATED_GITHUB_PATHS = frozenset( - { - b".github/workflows/castiron-custom-code.yml", - b".github/workflows/castiron-custom-code-comment.yml", - } -) - - -# Audited inventory of the authenticated pre-inventory checker format. These -# historical snapshots bind the checker bytes and manifest through their codegen -# hash. Never infer their inventory from the running compiler or candidate tree. -LEGACY_CHECKPOINT_INVENTORIES = { - "e504abba2701f4733eb4b8ce26856cca596e13feab1e7e323adc8e7d2ccb64bc": frozenset( - { - b".github/workflows/castiron-custom-code-comment.yml", - b".github/workflows/castiron-custom-code.yml", - } - ), -} - - -def checkpoint_github_inventory(source: bytes) -> frozenset[bytes]: - """Read literal compiler inventory from verified checkpoint data, without execution.""" - values = [ - node.value - for node in ast.parse(source).body - if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "GENERATED_GITHUB_PATHS" - for target in node.targets - ) - ] - if not values: - historical = LEGACY_CHECKPOINT_INVENTORIES.get(hashlib.sha256(source).hexdigest()) - if historical is not None: - return historical - if len(values) != 1: - raise ValueError("checkpoint must contain one generated GitHub inventory") - value = values[0] - if ( - not isinstance(value, ast.Call) - or not isinstance(value.func, ast.Name) - or value.func.id != "frozenset" - or len(value.args) != 1 - or value.keywords - ): - raise ValueError("checkpoint generated GitHub inventory must be literal") - literal_paths: object = ast.literal_eval(value.args[0]) - # An empty renderer inventory is written with an empty pair of braces. - if isinstance(literal_paths, dict) and not literal_paths: - return frozenset() - if not isinstance(literal_paths, set) or any( - not isinstance(path, bytes) or not path.startswith(b".github/") - for path in cast(set[object], literal_paths) - ): - raise ValueError("invalid checkpoint generated GitHub inventory") - return frozenset(cast(set[bytes], literal_paths)) - - -def generated_manifest(repo: Path, generated: str) -> dict[bytes, dict[str, str]]: - """Read ownership from the verified snapshot, never the candidate's manifest.""" - entries = report.tree_entries(repo, generated, include_workflows=True) - manifests = [path for path in MANIFEST_PATHS if path in entries] - if len(manifests) > 1: - raise ValueError("checkpoint contains ambiguous generated ownership manifests") - manifest_path = manifests[0] if manifests else MANIFEST - entry = entries.get(manifest_path) - if entry is None or entry.mode != b"100644": - raise ValueError( - "strict mode requires a generated ownership manifest; regenerate with Castiron" - ) - blobs = report.read_blobs(repo, entries) - inventory_path = manifest_path.rsplit(b"/", 1)[0] + b"/custom_code_budget.py" - inventory_entry = entries.get(inventory_path) - if inventory_entry is None or inventory_entry.mode not in (b"100644", b"100755"): - raise ValueError("checkpoint is missing its generated GitHub inventory") - github_paths = checkpoint_github_inventory(blobs[inventory_entry.oid]) - value = json.loads(blobs[entry.oid], object_pairs_hook=unique_keys) - if not isinstance(value, dict) or set(cast(dict[str, Any], value)) != { - "schema_version", - "files", - }: - raise ValueError("invalid generated ownership manifest") - value = cast(dict[str, Any], value) - if ( - type(value["schema_version"]) is not int - or value["schema_version"] != 1 - or not isinstance(value["files"], dict) - ): - raise ValueError("invalid generated ownership manifest") - files: dict[bytes, dict[str, str]] = {} - for name, identity in cast(dict[Any, Any], value["files"]).items(): - if ( - not isinstance(name, str) - or not name - or "\\" in name - or "\0" in name - or any(part in ("", ".", "..", ".git") for part in name.split("/")) - ): - raise ValueError("invalid generated manifest path") - path = name.encode("utf-8") - if path in MANIFEST_PATHS: - raise ValueError("generated manifest cannot include itself") - # A matching identity cannot turn inherited, SDK-owned CI into generated - # output. Use this verified checkpoint's literal compiler inventory, not - # the executing verifier's version or candidate Python/manifest claims. - # Public checkpoints may omit the private workflow/action blobs. - if not report.included(path) and path not in github_paths: - raise ValueError(f"generated manifest claims SDK-owned GitHub path: {name}") - if not isinstance(identity, dict) or set(cast(dict[str, Any], identity)) != { - "mode", - "sha256", - }: - raise ValueError(f"invalid generated identity: {name}") - identity = cast(dict[str, Any], identity) - if ( - identity["mode"] not in ("100644", "100755") - or not isinstance(identity["sha256"], str) - or not report.HASH.fullmatch(identity["sha256"]) - ): - raise ValueError(f"invalid generated identity: {name}") - files[path] = identity - # Sanitized public checkpoints omit GitHub blobs, not their ownership entries. - # Otherwise an omitted identity would also escape strict candidate checks. - if missing := github_paths - files.keys(): - raise ValueError( - f"generated manifest omits generated GitHub inventory path: {min(missing).decode()}" - ) - # Checkpoints retain SDK-owned workflow/action subtrees from their parent. - # Those are outside generated ownership. Every other snapshot path, including - # renderer-owned GitHub files omitted from the legacy hash, must be listed. - # Public snapshots retain generated identities without the private blobs. - snapshot_paths = set(entries) - {manifest_path} - if any( - report.included(path) or path in github_paths for path in snapshot_paths - files.keys() - ) or any(report.included(path) for path in files.keys() - snapshot_paths): - raise ValueError("generated manifest does not cover the verified snapshot exactly") - for path, identity in files.items(): - if path in entries: - actual = entries[path] - if ( - actual.mode.decode() != identity["mode"] - or hashlib.sha256(blobs[actual.oid]).hexdigest() != identity["sha256"] - ): - raise ValueError(f"manifest disagrees with generated snapshot: {path.decode()}") - # The manifest is itself immutable, bound by the existing snapshot hash. - files[manifest_path] = { - "mode": "100644", - "sha256": hashlib.sha256(blobs[entry.oid]).hexdigest(), - } - return files - - -def verify_generated( - repo: Path, - generated: str, - head: str, - *, - previous: str | None = None, - worktree: bool = False, -) -> list[str]: - files = generated_manifest(repo, generated) - old = generated_manifest(repo, previous) if previous and previous != generated else files - entries = report.tree_entries(repo, head, include_workflows=True) - blobs = report.read_blobs(repo, {p: e for p, e in entries.items() if p in files}) - failures: list[str] = [] - for path, identity in files.items(): - entry = entries.get(path) - content = blobs.get(entry.oid) if entry else None - mode = entry.mode.decode() if entry else None - if worktree: - full = repo / path.decode() - # Never follow a symlink introduced by an agent or formatter. - unsafe = any( - (repo / Path(*Path(path.decode()).parts[:i])).is_symlink() - for i in range(1, len(Path(path.decode()).parts) + 1) - ) - if unsafe or not full.is_file(): - content, mode = None, None - else: - content = full.read_bytes() - mode = "100755" if full.stat().st_mode & 0o111 else "100644" - if ( - content is None - or mode != identity["mode"] - or hashlib.sha256(content).hexdigest() != identity["sha256"] - ): - failures.append( - f"{path.decode()}: differs from Castiron output; change inputs/compiler or handwritten code" - ) - for path in old.keys() - files.keys(): - retired = repo / path.decode() - present = ( - retired.is_symlink() or (retired.exists() and not retired.is_dir()) - if worktree - else path in entries - ) - if present: - failures.append(f"{path.decode()}: retired generated file remains in candidate") - return failures - - -def check_isolation(repo: Path, base: str, head: str) -> int | None: - # Validate even if the policy was removed by an earlier commit in this PR. - proposed = read_budget(repo, head) - paths = changed_paths(repo, base, head) - if POLICY.encode() not in paths: - return None - if strict_policy(repo, base) and not strict_policy(repo, head): - raise ValueError("strict generated-file ownership cannot be downgraded") - if go_ratchet(repo, base) and not go_ratchet(repo, head): - raise ValueError("Go customization ratchet cannot be disabled") - if paths != {POLICY.encode()}: - raise ValueError(ISOLATION_MESSAGE) - return proposed - - -def count_custom_lines(result: dict[str, Any]) -> tuple[int, int]: - if result.get("status") != "ok": - raise ValueError("custom-code report could not verify the generated baseline") - added = removed = 0 - for file in result["files"]: - if not file["custom_after"]: - continue - counts = [file["added"], file["removed"]] - if any(not isinstance(n, str) or not n.isascii() or not n.isdecimal() for n in counts): - raise ValueError(f"cannot count non-text customization: {file['path']}") - added += int(counts[0]) - removed += int(counts[1]) - return added, removed - - -def outcome(state: str, description: str) -> dict[str, str]: - return {"state": state, "description": description} - - -def go_customization(repo: Path, revision: str, measured: dict[str, Any]) -> dict[str, Any]: - """Count patches plus physical handwritten Go, using only verified Git objects.""" - added, removed = count_custom_lines(measured) - # Use the reporter's ownership view. Private snapshots also retain SDK-owned - # CI files excluded from that view; those Go files are still handwritten. - generated = report.tree_entries(repo, measured["after"]["commit"]) - handwritten = { - path: entry - for path, entry in report.tree_entries(repo, revision, include_workflows=True).items() - if path.endswith(b".go") and path not in generated - } - for path, entry in handwritten.items(): - if entry.mode not in {b"100644", b"100755"}: - raise ValueError(f"nonregular handwritten Go: {path!r}") - blobs = report.read_blobs(repo, handwritten) - files: list[dict[str, Any]] = [] - for path, entry in sorted(handwritten.items()): - content = blobs[entry.oid] - content.decode("utf-8") - if b"\0" in content: - raise ValueError(f"non-text handwritten Go: {path!r}") - lines = content.count(b"\n") + int(bool(content) and not content.endswith(b"\n")) - files.append({"path": path.decode("utf-8"), "lines": lines}) - handwritten_lines = sum(file["lines"] for file in files) - return { - "revision": revision, - "generated": measured["after"], - "patch_lines": added + removed, - "handwritten_lines": handwritten_lines, - "handwritten_files": files, - "total": added + removed + handwritten_lines, - } - - -def evaluate( - repo: Path, - base: str, - head: str, - *, - public: bool, - fetch: bool = False, - pull_heads: list[str] | None = None, - measurement: tuple[dict[str, Any], bytes] | None = None, - policy_sha: str | None = None, - target_branch: str = "main", - main_snapshot: str | None = None, - repository: str | None = None, -) -> tuple[dict[str, Any], bytes]: - """In a queue, isolate each PR but budget the complete merged candidate.""" - base, head = report.require_sha(base), report.require_sha(head) - policy = cast(str, report.require_sha(policy_sha or base)) - result: dict[str, Any] = { - "base_sha": base, - "target_branch": target_branch, - "policy_sha": policy, - "head_sha": head, - "checks": {}, - } - checks = result["checks"] - proposed_limits: list[int] = [] - try: - read_budget(repo, policy) - read_budget(repo, base) - read_budget(repo, head) - if pull_heads is not None and not pull_heads: - raise ValueError("merge group has no verified constituent PRs") - if main_snapshot is not None and ( - target_branch != "next" - or main_snapshot != policy - or repository is None - or (pull_heads is not None and len(pull_heads) != 1) - or not is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - policy, - constituent=pull_heads[0] - if pull_heads is not None and len(pull_heads) == 1 - else None, - ) - or any( - not is_main_snapshot_bootstrap(repo, repository, base, candidate, policy) - for candidate in (pull_heads if pull_heads is not None else [head]) - ) - ): - raise ValueError("invalid authenticated main snapshot bootstrap") - for pr_head in pull_heads if pull_heads is not None else [head]: - proposed = 0 if main_snapshot is not None else check_isolation(repo, base, pr_head) - if proposed is not None: - proposed_limits.append(proposed) - checks["isolation"] = outcome( - "success", - "Exact authenticated strict-zero main snapshot adoption." - if main_snapshot is not None - else "Budget changes are isolated; human review is required for increases.", - ) - except (report.ReportError, ValueError, UnicodeError) as exc: - checks["isolation"] = outcome("failure", str(exc)) - - patch = b"" - try: - limit = read_budget(repo, policy) - result["limit"] = limit - measured, patch = ( - measurement - if measurement is not None - else report.build_report( - repo, - base, - head, - public=public, - fetch=fetch, - require_head_hash=True, - target_branch=target_branch, - ) - ) - added, removed = count_custom_lines(measured) - total = added + removed - result.update( - additions=added, - deletions=removed, - total=total, - mixed_files=measured["counts"]["after"], - generated=measured["after"], - files=measured["files"], - ) - # Increases never apply to their own PR/group. Decreases must be viable too. - checked_limit = min([limit, *proposed_limits]) - result["checked_limit"] = checked_limit - state = "success" if total <= checked_limit else "failure" - description = f"+{added} / -{removed} = {total} custom lines; limit {checked_limit}." - if any(go_ratchet(repo, revision) for revision in (policy, base, head)): - # The report's historical merge base is for presentation only. Measure - # the captured live target against its own hash-verified snapshot. - baseline, _ = report.build_report( - repo, - base, - base, - public=public, - fetch=fetch, - require_head_hash=True, - target_branch=target_branch, - ) - before = go_customization(repo, base, baseline) - after = go_customization(repo, head, measured) - delta = after["total"] - before["total"] - result["ratchet"] = {"kind": "go", "before": before, "after": after, "delta": delta} - description += ( - f" Go customization {before['total']} -> {after['total']} (delta {delta:+d})." - ) - if delta > 0: - state = "failure" - if go_ratchet(repo, base) and not go_ratchet(repo, head): - state = "failure" - description += " Go customization ratchet cannot be disabled." - # Main's strict policy applies even if the candidate tries to remove it. - # A proposed activation must itself satisfy the stricter contract. - if strict_policy(repo, policy) or strict_policy(repo, base) or strict_policy(repo, head): - violations = verify_generated( - repo, - measured["after"]["commit"], - head, - previous=(measured["before"] or measured["adoption_baseline"])["commit"], - ) - result["strict_violations"] = violations - if violations: - state = "failure" - description = "Strict generated-file verification failed: " + "; ".join(violations) - else: - description += " All generated files exactly match the verified manifest." - checks["budget"] = outcome(state, description) - except (report.ReportError, ValueError, UnicodeError, KeyError) as exc: - checks["budget"] = outcome("failure", str(exc)) - return result, patch - - -def write_result(out: Path, result: dict[str, Any], patch: bytes = b"") -> None: - out.mkdir(parents=True, exist_ok=True) - (out / "budget.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") - (out / "custom-code.patch").write_bytes(patch) - lines = ["## Custom-code budget", ""] - for name in ("isolation", "budget"): - check = result["checks"][name] - lines.append(f"- **{name}: {check['state']}** — {html.escape(check['description'])}") - if "total" in result: - lines += [ - "", - f"Mixed files: **{result['mixed_files']}**. Base-budget headroom: **{result['limit'] - result['total']} lines**.", - ] - lines += [ - "", - f"Policy: `{result['policy_sha']}` · candidate: `{result['head_sha']}`.", - f"Target {result.get('target_branch', 'baseline')}: `{result['base_sha']}`.", - f"Verified generated snapshot: `{result['generated']['commit']}`.", - "", - "| Largest custom patches | Added | Deleted |", - "|---|---:|---:|", - ] - files = [f for f in result["files"] if f["custom_after"]] - for file in sorted(files, key=lambda f: int(f["added"]) + int(f["removed"]), reverse=True)[ - :10 - ]: - path = html.escape(file["path"]).replace("|", "|").replace("\n", " ") - lines.append(f"| {path} | {file['added']} | {file['removed']} |") - ownership = [f for f in result["files"] if f["category"] == "no_longer_generated"] - if ownership: - lines += ["", "**No longer generated (outside the current budget):**"] - lines += [f"- {html.escape(f['path'])}" for f in ownership] - lines += [ - "", - "Budget increases require a separate budget-only PR, explicit justification, and a human approving review. Agents may not approve or bypass an increase.", - ] - summary = "\n".join(lines) + "\n" - (out / "summary.md").write_text(summary) - print(summary) - - -def github_evaluate( - repo: Path, - repository: str, - event: dict[str, Any], - trusted_sha: str, - trusted_report_dir: Path | None = None, - selection: dict[str, Any] | None = None, -) -> tuple[dict[str, Any], bytes]: - """Fetch data into a fresh bare repo; the executable checkout remains trusted.""" - if not report.REPOSITORY.fullmatch(repository): - raise ValueError("invalid repository") - root = f"repos/{repository}" - metadata = report.api("GET", root) - if event["repository"]["full_name"] != repository: - raise ValueError("event repository mismatch") - branch = metadata["default_branch"] - if branch != "main": - raise ValueError("budget gate requires main as the default branch") - main = report.require_sha(trusted_sha) - signal = event["workflow_run"] - run_id = signal["id"] - if type(run_id) is not int or run_id <= 0: - raise ValueError("invalid source run ID") - run = report.api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["head_sha"] != signal["head_sha"] - or run["repository"]["full_name"] != repository - or run["path"].split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["status"] != "completed" - or run["run_attempt"] != signal["run_attempt"] - or run["event"] != signal["event"] - ): - raise ValueError("unexpected or superseded source workflow run") - head = report.require_sha(run["head_sha"]) - measured = None - if run["event"] == "pull_request": - current = report.current_pull(repository, run) - if current is None: - raise ValueError( - "source run must identify exactly one current PR targeting main or next" - ) - _, branch = current - if trusted_report_dir is not None: - measured = json.loads((trusted_report_dir / "report.json").read_text()) - base = report.require_sha(measured["target_base_sha"]) - else: - base = ( - main - if branch == "main" - else report.require_sha( - report.api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"] - ) - ) - elif run["event"] == "merge_group": - # Both the target baseline and trusted policy must still be current. - branch = next( - ( - name - for name in report.TARGET_BRANCHES - if run["head_branch"].startswith(f"gh-readonly-queue/{name}/") - ), - "", - ) - if not branch: - raise ValueError("queue signal does not target main or next") - current_main = report.require_sha( - report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"] - ) - if branch == "main" and main != current_main: - raise ValueError("trusted checkout is stale; rerun against current main") - base = ( - main - if branch == "main" - else report.require_sha( - report.api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"] - ) - ) - else: - raise ValueError("unsupported budget event") - # PR statuses can outlive their base revision. Require the queue's fresh, - # combined candidate check before merging rather than relying on those alone. - page = 1 - while True: - rules = report.api("GET", f"{root}/rules/branches/{branch}?per_page=100&page={page}") - if any(rule["type"] == "merge_queue" for rule in rules): - break - if len(rules) < 100: - raise ValueError( - f"{branch} must require a merge queue before activating the budget gate" - ) - page += 1 - # This receipt is created locally by the trusted-main selector before the - # pinned checkout. It must never come from candidate workflow artifacts. - if ( - selection is None - or type(selection.get("schema_version")) is not int - or any( - selection.get(key) != value - for key, value in { - "schema_version": 1, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": main, - }.items() - ) - ): - raise ValueError("trusted policy selection does not match evaluation") - measurement = None - if trusted_report_dir is not None: - # Only the preceding main-branch step may supply this directory. Never - # download a candidate artifact here. Reuse its verified report and bare - # object store, avoiding a second snapshot fetch/hash/diff computation. - if run["event"] != "pull_request": - raise ValueError("only PR runs can reuse the trusted report") - assert measured is not None - if ( - measured["target_branch"] != branch - or measured["policy_sha"] != main - or measured["head_sha"] != head - or (branch == "main" and base != main) - ): - raise ValueError("trusted report does not match the captured base/head") - if report.git(repo, "rev-parse", "--is-bare-repository").strip() != b"true": - raise ValueError("trusted report must use a bare object store") - measurement = (measured, (trusted_report_dir / "custom-code.patch").read_bytes()) - else: - if repo.exists(): - raise ValueError("Git object directory must be fresh") - subprocess.run(["git", "init", "--bare", str(repo)], check=True, capture_output=True) - report.git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - report.git( - repo, "fetch", "--quiet", "--no-tags", "origin", *dict.fromkeys((main, base, head)) - ) - pull_heads: list[str] | None = None - if run["event"] == "merge_group": - if report.git(repo, "merge-base", base, head).decode().strip() != base: - raise ValueError("merge group does not include current target; retry the current group") - pull_heads = [] - entries = queued_entries(repository, branch) - if not any(queue_head == head for _, queue_head in entries): - raise ValueError("merge-group head is no longer in the current queue") - for candidate, queue_head in entries: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", queue_head) - ancestor = report.git(repo, "merge-base", queue_head, head).decode().strip() - if ancestor == queue_head: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", candidate) - pull_heads.append(candidate) - if not pull_heads: - raise ValueError("cannot verify merge-group PR membership; retry the current group") - main_snapshot = selection.get("main_snapshot") or None - if main_snapshot is not None: - if ( - branch != "next" - or main_snapshot != main - or report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"] != main - ): - raise ValueError("authenticated main snapshot changed; retry bootstrap") - # This evaluator has its own bare object store, including when reusing - # the PR report. SHA fetches do not create the selector's named main ref. - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - ) - if report.git(repo, "rev-parse", "refs/remotes/origin/main").decode().strip() != main: - raise ValueError("authenticated main changed during snapshot fetch; retry") - if not is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - main, - constituent=pull_heads[0] if pull_heads is not None and len(pull_heads) == 1 else None, - ): - raise ValueError("authenticated main snapshot changed; retry bootstrap") - if branch == "next": - assert selection is not None - base_pin = read_policy(repo, base).get("next_policy") - if base_pin is None: - # evaluate() checks each verified queue constituent for policy isolation. - if ( - selection.get("bootstrap") is not True - or "next_policy" not in read_policy(repo, head) - or ( - main_snapshot is None - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ) - ): - raise ValueError("unreviewed next policy initialization") - elif selection.get("bootstrap") is not False or base_pin != { - "repository": repository, - "main_sha": main, - }: - raise ValueError("trusted next base pin changed; rerun policy selection") - result, patch = evaluate( - repo, - base, - head, - public=not metadata["private"], - fetch=True, - pull_heads=pull_heads, - measurement=measurement, - policy_sha=main, - target_branch=branch, - main_snapshot=main_snapshot, - repository=repository, - ) - result["target_branch"] = branch - return result, patch - - -def prepare_main_snapshot( - repo: Path, repository: str, base: str, authenticated_main: str, *, fetch: bool = False -) -> str: - """Create and verify an adoption commit without altering a ref, index or metadata.""" - base, authenticated_main = map(report.require_sha, (base, authenticated_main)) - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - main_policy = read_policy(repo, authenticated_main) - if { - key: value for key, value in main_policy.items() if key != "ratchet" - } != strict or "next_policy" in read_policy(repo, base): - raise ValueError("main snapshot preparation requires strict-zero main and unpinned next") - if policy.merge_base(repo, base, authenticated_main) != base: - raise ValueError("protected next must be an ancestor of authenticated main") - value = { - **main_policy, - "next_policy": {"repository": repository, "main_sha": authenticated_main}, - } - blob = report.git( - repo, - "hash-object", - "-w", - "--stdin", - input_bytes=(json.dumps(value, indent=2) + "\n").encode(), - ).strip() - entries = report.git(repo, "ls-tree", "-z", authenticated_main).split(b"\0") - # Replace only the root policy blob, preserving every other entry and mode. - replacement = b"100644 blob " + blob + b"\t" + POLICY.encode() - tree = ( - report.git( - repo, - "mktree", - "-z", - input_bytes=b"\0".join( - replacement if entry.partition(b"\t")[2] == POLICY.encode() else entry - for entry in entries - ), - ) - .decode() - .strip() - ) - head = ( - report.git( - repo, - "commit-tree", - tree, - "-p", - base, - "-p", - authenticated_main, - "-m", - "Adopt verified strict-zero main baseline on next", - ) - .decode() - .strip() - ) - if not is_main_snapshot_bootstrap(repo, repository, base, head, authenticated_main): - raise ValueError("prepared snapshot does not match authenticated main") - result, _ = evaluate( - repo, - base, - head, - public=True, - fetch=fetch, - target_branch="next", - policy_sha=authenticated_main, - main_snapshot=authenticated_main, - repository=repository, - ) - if not all(check["state"] == "success" for check in result["checks"].values()): - raise ValueError("main snapshot verification failed: " + json.dumps(result["checks"])) - return cast(str, head) - - -def read_public_api(path: str) -> Any: - # Reuse the local gh login without reading or printing its credential. - output = subprocess.run( - ["gh", "api", "--hostname", "github.com", "--method", "GET", path], - capture_output=True, - check=True, - timeout=60, - ) - return json.loads(output.stdout) - - -def prepare_next_baseline(repo: Path, repository: str, branch: str) -> dict[str, str]: - """Authenticate public refs and create one new local review branch; never push.""" - if not report.REPOSITORY.fullmatch(repository) or not branch.startswith("codex/"): - raise ValueError("expected a repository and a new codex/ review branch") - report.git(repo, "check-ref-format", "--branch", branch) - origin = report.git(repo, "remote", "get-url", "origin").decode().strip() - if origin not in report.github_origin_urls(repository): - raise ValueError("preparation requires the public repository as origin") - root = f"repos/{repository}" - metadata = read_public_api(root) - if ( - metadata["private"] - or metadata["default_branch"] != "main" - or metadata["full_name"] != repository - ): - raise ValueError("preparation requires a public repository with default main") - - def ref(name: str) -> str: - return cast( - str, - report.require_sha(read_public_api(f"{root}/git/ref/heads/{name}")["object"]["sha"]), - ) - - main, base = ref("main"), ref("next") - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - "+refs/heads/next:refs/remotes/origin/next", - ) - for name, expected in (("main", main), ("next", base)): - if ( - report.git(repo, "rev-parse", f"refs/remotes/origin/{name}").decode().strip() - != expected - ): - raise ValueError("public refs changed during preparation; retry") - head = prepare_main_snapshot(repo, repository, base, main, fetch=True) - if ref("main") != main or ref("next") != base: - raise ValueError("public refs changed during verification; retry") - # Compare-and-create only. Existing review branches and working trees are preserved. - report.git(repo, "update-ref", f"refs/heads/{branch}", head, "0" * 40) - return {"head_sha": head, "base_sha": base, "main_sha": main, "branch": branch} - - -def main() -> int: - if sys.argv[1:2] == ["select-policy"]: - return cast(int, policy.main(sys.argv[1:])) - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("review-rules") - prepare = commands.add_parser("prepare-next-baseline") - prepare.add_argument("--repo", type=Path, required=True) - prepare.add_argument("--repository", required=True) - prepare.add_argument("--branch", required=True) - generated = commands.add_parser("verify-generated") - generated.add_argument("--repo", type=Path, required=True) - generated.add_argument("--policy-ref", required=True) - generated.add_argument("--generated", required=True) - generated.add_argument("--previous-generated") - generated.add_argument("--head", required=True) - generated.add_argument("--worktree", action="store_true") - local = commands.add_parser("check") - local.add_argument("--base", required=True) - local.add_argument("--head", required=True) - local.add_argument("--public", action="store_true") - local.add_argument("--fetch", action="store_true") - local.add_argument("--target-branch", choices=sorted(report.TARGET_BRANCHES), default="main") - github = commands.add_parser("github") - github.add_argument("--repository", required=True) - github.add_argument("--event-path", type=Path, required=True) - github.add_argument("--trusted-sha", required=True) - github.add_argument("--trusted-report-dir", type=Path) - github.add_argument("--selected-policy-json", type=Path) - for command in (local, github): - command.add_argument("--repo", type=Path, required=True) - command.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - if args.command == "review-rules": - print(json.dumps(policy.HUMAN_REVIEW_RULES, separators=(",", ":"))) - return 0 - if args.command == "prepare-next-baseline": - try: - preparation = prepare_next_baseline(args.repo, args.repository, args.branch) - except (ValueError, OSError, report.ReportError, subprocess.SubprocessError) as exc: - print(str(exc), file=sys.stderr) - return 1 - print(json.dumps(preparation, sort_keys=True)) - return 0 - if args.command == "verify-generated": - # Local pipeline integration pins policy and checkpoint before merging. - # Legacy SDKs without a policy retain their existing behavior. - try: - policy_ref = report.require_sha(args.policy_ref) - if not report.git(args.repo, "ls-tree", policy_ref, "--", POLICY): - return 0 - if not strict_policy(args.repo, policy_ref): - return 0 - failures = verify_generated( - args.repo, - args.generated, - args.head, - previous=args.previous_generated, - worktree=args.worktree, - ) - if failures: - raise ValueError("\n".join(failures)) - except (ValueError, OSError, report.ReportError) as exc: - print(str(exc), file=sys.stderr) - return 1 - return 0 - try: - if args.command == "check": - result, patch = evaluate( - args.repo, - args.base, - args.head, - public=args.public, - fetch=args.fetch, - target_branch=args.target_branch, - ) - else: - result, patch = github_evaluate( - args.repo, - args.repository, - json.loads(args.event_path.read_text()), - args.trusted_sha, - args.trusted_report_dir, - json.loads(args.selected_policy_json.read_text()) - if args.selected_policy_json - else None, - ) - except ( - report.ReportError, - ValueError, - OSError, - KeyError, - TypeError, - subprocess.SubprocessError, - ) as exc: - result = { - "checks": {name: outcome("failure", str(exc)) for name in ("isolation", "budget")} - } - patch = b"" - write_result(args.out, result, patch) - if args.command == "github" and "GITHUB_OUTPUT" in os.environ: - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - output.writelines( - f"{name}={result['checks'][name]['state']}\n" for name in ("isolation", "budget") - ) - output.write(f"target_branch={result.get('target_branch', '')}\n") - output.write(f"policy_sha={result.get('policy_sha', '')}\n") - output.write(f"base_sha={result.get('base_sha', '')}\n") - output.write(f"head_sha={result.get('head_sha', '')}\n") - return 0 if all(c["state"] == "success" for c in result["checks"].values()) else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_policy.py b/scripts/castiron/custom_code_policy.py deleted file mode 100755 index 8d06a30575..0000000000 --- a/scripts/castiron/custom_code_policy.py +++ /dev/null @@ -1,731 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Select reviewed policy using current main; emit a local, versioned receipt. - -The pinned budget evaluator remains a separate executable authority. This module -never executes candidate code, and receipts must not come from candidate artifacts. -""" - -from __future__ import annotations - -import argparse -import ast -import importlib.util -import json -import os -import re -import sys -from pathlib import Path -from typing import Any, TypedDict, cast - -# Explicitly load the reporter beside this trusted script, including under python -I. -# The inspected repository is only a Git object store, not a Python import path. -_spec = importlib.util.spec_from_file_location( - "castiron_budget_reporter", Path(__file__).with_name("custom_code_report.py") -) -assert _spec is not None and _spec.loader is not None -report = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = report -_spec.loader.exec_module(report) - -POLICY = ".castiron-ratchet.json" -NEXT_POLICY_VERSION = 1 -# Inert capability marker read by trusted review invalidation at this exact revision. -PIN_REVIEW_POLICY = "effective-policy-v1" -SELECTION_RECEIPT_VERSION = 1 - - -def unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - value: dict[str, Any] = {} - for key, item in pairs: - if key in value: - raise ValueError(f"duplicate policy key: {key}") - value[key] = item - return value - - -def read_policy(repo: Path, revision: str) -> dict[str, Any]: - revision = report.require_sha(revision) - entry = report.git(repo, "ls-tree", revision, "--", POLICY).split() - if len(entry) != 4 or entry[:2] != [b"100644", b"blob"]: - raise ValueError(f"{POLICY} must exist as a regular, non-executable file at {revision}") - if int(report.git(repo, "cat-file", "-s", entry[2].decode())) > 4096: - raise ValueError("budget file exceeds 4096 bytes") - value = json.loads( - report.git(repo, "show", f"{revision}:{POLICY}"), object_pairs_hook=unique_keys - ) - if not isinstance(value, dict): - raise ValueError("budget file must be an object") # noqa: TRY004 - malformed policy is a value error - value = cast(dict[str, Any], value) - version = value.get("schema_version") - if type(version) is not int or version not in (1, 2): - raise ValueError("unsupported budget schema_version") - keys = {"schema_version", "max_custom_patch_lines"} - if "ratchet" in value: - keys.add("ratchet") - if value["ratchet"] != "go": - raise ValueError("ratchet must be go") - if version == 2: - keys.add("generated_files") - if "next_policy" in value: - keys.add("next_policy") - pin: Any = value["next_policy"] - if not isinstance(pin, dict) or set(cast(dict[str, Any], pin)) != { - "repository", - "main_sha", - }: - raise ValueError("next_policy must contain exactly repository and main_sha") - if not isinstance(pin["repository"], str) or not report.REPOSITORY.fullmatch( - pin["repository"] - ): - raise ValueError("invalid next_policy repository") - if not isinstance(pin["main_sha"], str): - raise ValueError("invalid next_policy main_sha") - report.require_sha(pin["main_sha"]) - if set(value) != keys: - raise ValueError("budget file contains missing or unsupported fields") - limit = value["max_custom_patch_lines"] - if type(limit) is not int or limit < 0: - raise ValueError("max_custom_patch_lines must be a nonnegative integer") - if version == 2 and (value["generated_files"] != "strict" or limit != 0): - raise ValueError("schema v2 requires generated_files=strict and a zero budget") - return value - - -def next_policy_sha(repo: Path, repository: str, base: str, authenticated_main: str) -> str: - """Select executable policy only from an authenticated target base, never its candidate.""" - base = report.require_sha(base) - authenticated_main = report.require_sha(authenticated_main) - if not report.REPOSITORY.fullmatch(repository): - raise ValueError("invalid policy repository") - origin = report.git(repo, "remote", "get-url", "origin").decode().strip() - if origin not in report.github_origin_urls(repository): - raise ValueError("policy repository origin mismatch") - fetched_main = report.git(repo, "rev-parse", "refs/remotes/origin/main").decode().strip() - if fetched_main != authenticated_main: - raise ValueError("authenticated main changed during policy selection; retry") - pin = read_policy(repo, base).get("next_policy") - # read_policy validates the shape of every present next_policy pin. - if pin is None: - raise ValueError("trusted next base has no reviewed policy pin") - if pin["repository"] != repository: - raise ValueError("next policy pin belongs to another repository") - policy = cast(str, report.require_sha(pin["main_sha"])) - # A reachable side-branch commit is not an adopted main policy revision. - adopted = report.git(repo, "rev-list", "--first-parent", authenticated_main).splitlines() - if policy.encode() not in adopted: - raise ValueError("next policy pin is not in authenticated main first-parent history") - read_policy(repo, policy) - # Reject pre-protocol checkers rather than executing code that cannot bind a next pin. - source = report.git(repo, "show", f"{policy}:scripts/castiron/custom_code_budget.py") - tree = ast.parse(source) - capable = any( - isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "NEXT_POLICY_VERSION" - for target in node.targets - ) - and isinstance(node.value, ast.Constant) - and type(node.value.value) is int - and node.value.value == NEXT_POLICY_VERSION - for node in tree.body - ) - if not capable: - raise ValueError("pinned checker does not support reviewed next policies") - return policy - - -def supports_effective_pin_review(repo: Path, revision: str) -> bool: - """Read the effective checker's inert contract; never execute its source.""" - revision = report.require_sha(revision) - path = "scripts/castiron/custom_code_policy.py" - entry = report.git(repo, "ls-tree", revision, "--", path).split() - if len(entry) != 4 or entry[0] not in (b"100644", b"100755") or entry[1] != b"blob": - return False - source = report.git(repo, "show", f"{revision}:{path}") - declarations = re.findall(rb"^PIN_REVIEW_POLICY = .*$", source, re.MULTILINE) - return declarations == [b'PIN_REVIEW_POLICY = "effective-policy-v1"'] - - -class HumanReviewRules(TypedDict): - version: int - associations: list[str] - negative_pattern: str - attestation_prefix: str - - -HUMAN_REVIEW_RULES: HumanReviewRules = { - "version": 2, - "attestation_prefix": "I personally reviewed and approve this policy change as a human: ", - "associations": ["OWNER", "MEMBER", "COLLABORATOR"], - "negative_pattern": ( - r"automated[\s_:-]*(?:source[\s_:-]*)?review|source[- ]only review|" - r"(?:not|isn't|does not constitute|does not substitute for)\s+(?:an?\s+)?human\s+approv|" - r"non[- ]human[- ]approval" - ), -} - - -def has_human_approval(reviews: list[dict[str, Any]], author: str, head: str) -> bool: - """Versioned review predicate; publisher consumes these same trusted rules.""" - latest: dict[str, dict[str, Any]] = {} - for review in reviews: - # Discussion does not replace an approval; blocking or dismissed - # reviews still supersede it, and edited attestations are checked below. - if review["state"] not in ("PENDING", "COMMENTED") and review.get("user") is not None: - login = review["user"]["login"] - if login not in latest or review["id"] > latest[login]["id"]: - latest[login] = review - return any( - review["state"] == "APPROVED" - and review["commit_id"] == head - and review["user"]["type"] == "User" - and review.get("author_association") in HUMAN_REVIEW_RULES["associations"] - and not review["user"]["login"].endswith("[bot]") - and review["user"]["login"] != author - and (review.get("body") or "").strip() == HUMAN_REVIEW_RULES["attestation_prefix"] + head - and not re.search( - HUMAN_REVIEW_RULES["negative_pattern"], review.get("body") or "", re.IGNORECASE - ) - for review in latest.values() - ) - - -def require_human_policy_review(repository: str, head: str, number: int | None = None) -> int: - """Only live, exact-head human reviews authorize a next policy activation/update.""" - root = f"repos/{repository}" - pulls = ( - [{"number": number}] - if number is not None - else report.api("GET", f"{root}/commits/{head}/pulls?per_page=100") - ) - if not pulls: - # Fork runs can have no commit association. Discover candidates from - # the authenticated base repository, then apply the same live checks - # below. Never accept an approval based only on search metadata. - page = 1 - while True: - candidates = report.api( - "GET", f"{root}/pulls?state=open&base=next&per_page=100&page={page}" - ) - pulls.extend(item for item in candidates if item["head"]["sha"] == head) - if len(candidates) < 100: - break - page += 1 - current: list[dict[str, Any]] = [] - for item in pulls: - pull = report.api("GET", f"{root}/pulls/{item['number']}") - if ( - pull["state"] == "open" - and pull["head"]["sha"] == head - and pull["base"]["ref"] == "next" - and pull["base"]["repo"]["full_name"] == repository - ): - current.append(pull) - if len(current) != 1: - raise ValueError("policy change must identify one open next PR") - pull = current[0] - all_reviews: list[dict[str, Any]] = [] - page = 1 - while True: - reviews = report.api( - "GET", f"{root}/pulls/{pull['number']}/reviews?per_page=100&page={page}" - ) - all_reviews.extend(reviews) - if len(reviews) < 100: - break - page += 1 - if not has_human_approval(all_reviews, pull["user"]["login"], head): - raise ValueError("next policy changes require an exact-head human approving review") - return int(pull["number"]) - - -def requires_human_policy_review( - repo: Path, repository: str, base: str, head: str, authenticated_main: str -) -> bool: - """Decide review requirements from Git data using the selector's policy rules.""" - base_policy = read_policy(repo, base) - candidate_policy = read_policy(repo, head) - candidate_base_policy = read_policy(repo, merge_base(repo, base, head)) - budget_increased = ( - candidate_policy["max_custom_patch_lines"] > candidate_base_policy["max_custom_patch_lines"] - ) - ratchet_activated = ( - candidate_policy.get("ratchet") == "go" and candidate_base_policy.get("ratchet") != "go" - ) - pin_changed = candidate_policy.get("next_policy") != candidate_base_policy.get("next_policy") - if not pin_changed: - return budget_increased or ratchet_activated - policy = ( - next_policy_sha(repo, repository, base, authenticated_main) - if "next_policy" in base_policy - else authenticated_main - ) - candidate = head - proposed = read_policy(repo, next_policy_sha(repo, repository, candidate, authenticated_main)) - # Authored changes come from the merge base, but their safety must be - # measured against the active policy: the protected target's pin, or - # authenticated main during bootstrap. - previous = read_policy(repo, policy) - # Bootstrap executes authenticated main; initial adoption must not silently - # replace it with older checker code, even when policy fields match. - previous_checker = ( - base_policy["next_policy"]["main_sha"] - if "next_policy" in base_policy - else authenticated_main - ) - rolled_back = bool( - report.git( - repo, - "rev-list", - "--first-parent", - f"{candidate_policy['next_policy']['main_sha']}..{previous_checker}", - ).strip() - ) - # Compare effective policies, not the candidate's unchanged local ceiling. - # Schema/strictness or ratchet transitions also retain human authorization. - pin_requires_human = ( - not supports_effective_pin_review(repo, previous_checker) - or rolled_back - or proposed["max_custom_patch_lines"] > previous["max_custom_patch_lines"] - or proposed["schema_version"] != previous["schema_version"] - or proposed.get("generated_files") != previous.get("generated_files") - or proposed.get("ratchet") != previous.get("ratchet") - ) - return budget_increased or ratchet_activated or pin_requires_human - - -def is_main_snapshot_bootstrap( - repo: Path, - repository: str, - base: str, - head: str, - authenticated_main: str, - *, - constituent: str | None = None, -) -> bool: - """Recognize only an exact, strict-zero authenticated main snapshot adoption. - - This does not verify generated ownership: the ordinary budget evaluator must - still authenticate the unchanged main checkpoints and their complete manifest. - No candidate file is executed and no generation metadata is rewritten. - """ - base, head, authenticated_main = map(report.require_sha, (base, head, authenticated_main)) - if "next_policy" in read_policy(repo, base): - return False - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - main_policy = read_policy(repo, authenticated_main) - if {key: value for key, value in main_policy.items() if key != "ratchet"} != strict: - return False - expected_policy = { - **main_policy, - "next_policy": {"repository": repository, "main_sha": authenticated_main}, - } - if read_policy(repo, head) != expected_policy: - return False - # Also validates the configured repository, fetched main and protocol marker. - if next_policy_sha(repo, repository, head, authenticated_main) != authenticated_main: - return False - if merge_base(repo, base, authenticated_main) != base or merge_base(repo, base, head) != base: - return False - paths = set( - report.git( - repo, - "diff", - "--name-only", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - authenticated_main, - head, - "--", - ).split(b"\0") - ) - {b""} - if paths != {POLICY.encode()}: - return False - parents = report.git(repo, "rev-list", "--parents", "-n", "1", head).decode().split()[1:] - expected = list(dict.fromkeys((base, authenticated_main))) - if constituent is not None and head != constituent: - if not is_main_snapshot_bootstrap(repo, repository, base, constituent, authenticated_main): - return False - expected = [base, constituent] - if parents != expected: - return False - # The complete tree comparison binds stats/inventory bytes and file modes. - # Require an existing public snapshot; this is not legacy metadata recovery. - return bool(report.read_stats(repo, authenticated_main).get("public_codegen_sha")) - - -def select_policy( - repo: Path, - repository: str, - branch: str, - base: str, - head: str, - authenticated_main: str, - *, - pull_heads: list[str] | None = None, - pull_numbers: dict[str, int] | None = None, -) -> dict[str, Any]: - """The trusted selector runs before checking out any pinned executable code.""" - base, head, authenticated_main = map(report.require_sha, (base, head, authenticated_main)) - if branch == "main": - return { - "schema_version": SELECTION_RECEIPT_VERSION, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": authenticated_main, - "bootstrap": False, - } - if branch != "next": - raise ValueError("unsupported policy target") - # Malformed policies are errors, never a reason to enter bootstrap mode. - base_policy = read_policy(repo, base) - head_policy = read_policy(repo, head) - bootstrap = "next_policy" not in base_policy - snapshot_bootstrap = bootstrap and is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - authenticated_main, - constituent=pull_heads[0] if pull_heads is not None and len(pull_heads) == 1 else None, - ) - if ( - snapshot_bootstrap - and pull_heads is not None - and ( - len(pull_heads) != 1 - or not is_main_snapshot_bootstrap( - repo, repository, base, pull_heads[0], authenticated_main - ) - ) - ): - raise ValueError("main snapshot bootstrap requires one exact verified constituent") - if bootstrap: - if "next_policy" not in head_policy or ( - not snapshot_bootstrap - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ): - raise ValueError("missing next pin allows only reviewed policy-only initialization") - policy = authenticated_main - else: - policy = next_policy_sha(repo, repository, base, authenticated_main) - # No candidate-selected version executes, including in a policy-only PR. - head_base_policy = read_policy(repo, merge_base(repo, base, head)) - if head_policy.get("next_policy") != head_base_policy.get("next_policy"): - if "next_policy" not in head_policy or ( - not snapshot_bootstrap - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ): - raise ValueError("next pin changes must be policy-only and cannot remove the pin") - next_policy_sha(repo, repository, head, authenticated_main) - policy_reviews: list[dict[str, Any]] = [] - bootstrap_validated = False - candidates = pull_heads if pull_heads is not None else [head] - if not candidates: - raise ValueError("next evaluation has no verified constituent PR") - for candidate in candidates: - candidate_policy = read_policy(repo, candidate) - candidate_base = merge_base(repo, base, candidate) - candidate_base_policy = read_policy(repo, candidate_base) - pin_changed = candidate_policy.get("next_policy") != candidate_base_policy.get( - "next_policy" - ) - budget_increased = ( - candidate_policy["max_custom_patch_lines"] - > candidate_base_policy["max_custom_patch_lines"] - ) - ratchet_activated = ( - candidate_policy.get("ratchet") == "go" and candidate_base_policy.get("ratchet") != "go" - ) - if ( - not snapshot_bootstrap - and (pin_changed or budget_increased or ratchet_activated) - and changed_paths(repo, base, candidate) != {POLICY.encode()} - ): - raise ValueError("each next policy activation or increase must be policy-only") - if requires_human_policy_review(repo, repository, base, candidate, authenticated_main): - number = require_human_policy_review( - repository, candidate, pull_numbers[candidate] if pull_numbers is not None else None - ) - policy_reviews.append({"number": number, "head_sha": candidate}) - # Keep activation tied to its verified policy-only constituent, even - # when current main's effective-policy rules require no new approval. - if pin_changed and candidate_policy.get("next_policy") == head_policy.get("next_policy"): - bootstrap_validated = True - if bootstrap and not bootstrap_validated: - raise ValueError("bootstrap requires a verified policy-only constituent") - return { - "schema_version": SELECTION_RECEIPT_VERSION, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": policy, - "bootstrap": bootstrap, - "main_snapshot": authenticated_main if snapshot_bootstrap else "", - "policy_reviews": policy_reviews, - "human_review_rules": HUMAN_REVIEW_RULES, - } - - -def select_github_policy( - repo: Path, - repository: str, - event: dict[str, Any], - trusted_sha: str, - *, - source_workflow: str = ".github/workflows/castiron-custom-code.yml", -) -> dict[str, Any]: - """Authenticate event, refs and graph in a fresh store before choosing checker code.""" - if source_workflow not in ( - ".github/workflows/castiron-custom-code.yml", - ".github/workflows/ci.yml", - ): - raise ValueError("unsupported policy source workflow") - if ( - not report.REPOSITORY.fullmatch(repository) - or event["repository"]["full_name"] != repository - ): - raise ValueError("policy event repository mismatch") - root = f"repos/{repository}" - metadata = report.api("GET", root) - if metadata["default_branch"] != "main": - raise ValueError("policy selector requires the main default branch") - signal = event["workflow_run"] - run = report.api("GET", f"{root}/actions/runs/{signal['id']}") - if ( - run["repository"]["full_name"] != repository - or run["head_sha"] != signal["head_sha"] - or run["status"] != "completed" - or run["event"] != signal["event"] - or run["run_attempt"] != signal["run_attempt"] - or run["path"].split("@", 1)[0] != source_workflow - ): - raise ValueError("unexpected or superseded policy source run") - workflow = report.api("GET", f"{root}/actions/workflows/{source_workflow.rsplit('/', 1)[1]}") - if ( - type(run.get("workflow_id")) is not int - or run["workflow_id"] != workflow["id"] - or workflow["path"] != source_workflow - ): - raise ValueError("policy source run is not the canonical workflow") - head = report.require_sha(run["head_sha"]) - pull_numbers: dict[str, int] = {} - if run["event"] == "pull_request": - current = report.current_pull(repository, run) - if current is None: - raise ValueError("policy run must identify one current target PR") - number, branch = current - pull_numbers[head] = number - elif run["event"] == "merge_group": - branch = next( - ( - name - for name in report.TARGET_BRANCHES - if run["head_branch"].startswith(f"gh-readonly-queue/{name}/") - ), - "", - ) - if not branch: - raise ValueError("policy queue target is unsupported") - else: - raise ValueError("unsupported policy source event") - trusted_sha = report.require_sha(trusted_sha) - if branch == "main": - if source_workflow != ".github/workflows/castiron-custom-code.yml": - raise ValueError("CI policy adapter is restricted to next") - return select_policy(repo, repository, branch, trusted_sha, head, trusted_sha) - main = report.require_sha(report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"]) - base = report.require_sha(report.api("GET", f"{root}/git/ref/heads/next")["object"]["sha"]) - if repo.exists(): - raise ValueError("policy object directory must be fresh") - repo.mkdir() - report.git(repo, "init", "--quiet", "--bare") - report.git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - "+refs/heads/next:refs/remotes/origin/next", - head, - ) - if report.git(repo, "rev-parse", "refs/remotes/origin/next").decode().strip() != base: - raise ValueError("next base changed during selection; retry") - heads: list[str] | None = None - if run["event"] == "merge_group": - if report.git(repo, "merge-base", base, head).decode().strip() != base: - raise ValueError("policy queue does not contain current next base") - entries = queued_pull_entries(repository, branch) - if not any(queue == head for _, queue, _ in entries): - raise ValueError("policy queue entry is no longer live") - heads = [] - for candidate, queue, number in entries: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", queue) - if report.git(repo, "merge-base", queue, head).decode().strip() == queue: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", candidate) - if candidate in pull_numbers: - raise ValueError("policy queue has multiple PRs for one head") - pull_numbers[candidate] = number - heads.append(candidate) - if not heads: - raise ValueError("policy queue has no verified constituent PRs") - return select_policy( - repo, repository, branch, base, head, main, pull_heads=heads, pull_numbers=pull_numbers - ) - - -def merge_base(repo: Path, base: str, head: str) -> str: - """Distinguish actual PR policy edits from later changes on the target.""" - return cast( - str, - report.git(repo, "merge-base", report.require_sha(base), report.require_sha(head)) - .decode() - .strip(), - ) - - -def changed_paths(repo: Path, base: str, head: str) -> set[bytes]: - """Use the entire PR diff, not its last commit or a truncated API file list.""" - start = merge_base(repo, base, head) - return set( - report.git( - repo, - "diff", - "--name-only", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - start, - head, - "--", - ).split(b"\0") - ) - {b""} - - -def queued_entries(repository: str, branch: str) -> list[tuple[str, str]]: - return [(head, queue) for head, queue, _ in queued_pull_entries(repository, branch)] - - -def queued_pull_entries(repository: str, branch: str) -> list[tuple[str, str, int]]: - """Query the actual queue; never infer PR membership from commit messages.""" - owner, name = repository.split("/") - query = """query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { pullRequest { number headRefOid } headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }""" - cursor = None - heads: list[tuple[str, str, int]] = [] - while True: - response = report.api( - "POST", - "graphql", - { - "query": query, - "variables": {"owner": owner, "name": name, "branch": branch, "cursor": cursor}, - }, - ) - entries = response["data"]["repository"]["mergeQueue"]["entries"] - for node in entries["nodes"]: - # Entries still awaiting preparation have no synthetic queue commit. - if node["headCommit"] is not None: - heads.append( - ( - report.require_sha(node["pullRequest"]["headRefOid"]), - report.require_sha(node["headCommit"]["oid"]), - int(node["pullRequest"]["number"]), - ) - ) - page = entries["pageInfo"] - if not page["hasNextPage"]: - return heads - if not page["endCursor"] or page["endCursor"] == cursor: - raise ValueError("invalid merge-queue pagination") - cursor = page["endCursor"] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - review = commands.add_parser("review-required") - review.add_argument("--repo", type=Path, required=True) - review.add_argument("--repository", required=True) - review.add_argument("--base", required=True) - review.add_argument("--head", required=True) - review.add_argument("--main", required=True) - selection = commands.add_parser("select-policy") - selection.add_argument("--repo", type=Path, required=True) - selection.add_argument("--repository", required=True) - selection.add_argument("--event-path", type=Path, required=True) - selection.add_argument("--trusted-sha", required=True) - selection.add_argument("--out", type=Path, required=True) - selection.add_argument( - "--source-workflow", - choices=(".github/workflows/castiron-custom-code.yml", ".github/workflows/ci.yml"), - default=".github/workflows/castiron-custom-code.yml", - ) - args = parser.parse_args(argv) - if args.command == "review-required": - if not report.REPOSITORY.fullmatch(args.repository): - raise ValueError("invalid policy repository") - base, head, main = map(report.require_sha, (args.base, args.head, args.main)) - if args.repo.exists(): - raise ValueError("policy object directory must be fresh") - args.repo.mkdir() - report.git(args.repo, "init", "--quiet", "--bare") - report.git( - args.repo, "remote", "add", "origin", f"https://github.com/{args.repository}.git" - ) - report.git( - args.repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - base, - head, - ) - if report.git(args.repo, "rev-parse", "refs/remotes/origin/main").decode().strip() != main: - raise ValueError("authenticated main changed during review evaluation; retry") - print( - json.dumps( - { - "requires_human": requires_human_policy_review( - args.repo, args.repository, base, head, main - ) - } - ) - ) - return 0 - selected = select_github_policy( - args.repo, - args.repository, - json.loads(args.event_path.read_text()), - args.trusted_sha, - source_workflow=args.source_workflow, - ) - args.out.write_text(json.dumps(selected, sort_keys=True) + "\n") - if "GITHUB_OUTPUT" in os.environ: - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - output.writelines( - f"{key}={json.dumps(value) if isinstance(value, (bool, list, dict)) else value}\n" - for key, value in selected.items() - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_report.py b/scripts/castiron/custom_code_report.py deleted file mode 100755 index 28b827905b..0000000000 --- a/scripts/castiron/custom_code_report.py +++ /dev/null @@ -1,1534 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regenerate with Castiron; do not edit this file by hand. -"""Standalone, vendorable Castiron generated-content hashing and reporting. - -Hash v1: SHA256(domain || u64(entry count) || entries), where each entry is -u64(path length) || path bytes || six ASCII Git mode bytes || u64(blob length) -|| blob bytes. Integers are unsigned big-endian. Entries sort by raw Git path. -Only .github/actions/ and .github/workflows/ are excluded. No SDK imports. -""" - -from __future__ import annotations - -import argparse -import base64 -import collections -import hashlib -import html -import json -import os -import re -import struct -import subprocess -import sys -import tempfile -from contextlib import ExitStack -from dataclasses import dataclass -from pathlib import Path -from typing import Any, cast -from urllib.parse import urlencode - -DOMAIN = b"castiron-codegen-v1\0" -MARKER = "" -RUN_MARKER = re.compile(r"") -SHA = re.compile(r"[0-9a-f]{40}\Z") -HASH = re.compile(r"[0-9a-f]{64}\Z") -REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") -TARGET_BRANCHES = ("main", "next") -EXCLUDED = (b".github/actions/", b".github/workflows/") -PUBLIC_BLOCKED_PATHS = { - b".castiron", - b".github", - b".gitmodules", - b"CODEOWNERS", - b"SECURITY.md", - b"docs/CODEOWNERS", - b"docs/SECURITY.md", -} -LABELS = { - "newly_customized": "Newly customized", - "removed": "Customization removed", - "existing_changed": "Existing customization changed", - "first_adoption": "Current customization; legacy baseline unavailable", - "baseline_changed": "Generated baseline changed", - "unchanged": "Existing customization unchanged", - "no_longer_generated": "No longer generated", - "newly_generation_owned": "Now generated; already existed", -} - - -class ReportError(ValueError): - """The report cannot establish a trustworthy comparison.""" - - -def github_origin_urls(repository: str) -> set[str]: - """Accepted URL spellings for one exact GitHub repository.""" - if not REPOSITORY.fullmatch(repository): - raise ReportError("invalid GitHub repository") - return { - f"https://github.com/{repository}.git", - f"https://github.com/{repository}", - f"git@github.com:{repository}.git", - } - - -@dataclass(frozen=True) -class Entry: - mode: bytes - oid: str - - -def git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - env = os.environ.copy() - # Preserve ordinary Git authentication/configuration, but do not inherit - # repository routing from the caller (for example, a Git hook). - for name in ( - "GIT_DIR", - "GIT_COMMON_DIR", - "GIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_PREFIX", - "GIT_OBJECT_DIRECTORY", - "GIT_ALTERNATE_OBJECT_DIRECTORIES", - ): - env.pop(name, None) - env["GIT_NO_REPLACE_OBJECTS"] = "1" - with ExitStack() as resources: - stdin = None - if input_bytes is not None: - # Batched object reads can fill both pipes on macOS. Feed requests - # through a private, automatically removed file while draining output. - stdin = resources.enter_context(tempfile.TemporaryFile()) - stdin.write(input_bytes) - stdin.seek(0) - result = subprocess.run( - ["git", "--no-replace-objects", "-C", str(repo), *args], - stdin=stdin, - capture_output=True, - env=env, - check=False, - ) - if result.returncode: - raise ReportError(f"git {args[0]} failed: {result.stderr.decode(errors='replace').strip()}") - return result.stdout - - -def require_sha(value: str) -> str: - if not SHA.fullmatch(value): - raise ReportError("expected a full Git commit SHA") - return value - - -def included(path: bytes) -> bool: - return not path.startswith(EXCLUDED) - - -def tree_entries(repo: Path, commit: str, *, include_workflows: bool = False) -> dict[bytes, Entry]: - result: dict[bytes, Entry] = {} - for record in git(repo, "ls-tree", "-r", "-z", require_sha(commit)).split(b"\0"): - if not record: - continue - metadata, path = record.split(b"\t", 1) - mode, kind, oid = metadata.split(b" ") - if not include_workflows and not included(path): - continue - if kind != b"blob" or mode not in {b"100644", b"100755", b"120000"}: - raise ReportError(f"unsupported Git entry: {path!r}") - result[path] = Entry(mode, oid.decode("ascii")) - return result - - -def read_blobs(repo: Path, entries: dict[bytes, Entry]) -> dict[str, bytes]: - oids = sorted({entry.oid for entry in entries.values()}) - output = git( - repo, "cat-file", "--batch", input_bytes="".join(f"{oid}\n" for oid in oids).encode() - ) - result: dict[str, bytes] = {} - offset = 0 - for oid in oids: - end = output.index(b"\n", offset) - actual, kind, size = output[offset:end].split(b" ") - if actual.decode() != oid or kind != b"blob": - raise ReportError("unexpected Git object response") - offset = end + 1 - length = int(size) - result[oid] = output[offset : offset + length] - offset += length - if output[offset : offset + 1] != b"\n": - raise ReportError("truncated Git object response") - offset += 1 - if offset != len(output): - raise ReportError("unexpected trailing Git object data") - return result - - -def hash_codegen_commit(repo: Path, commit: str) -> str: - entries = tree_entries(repo, commit) - blobs = read_blobs(repo, entries) - digest = hashlib.sha256(DOMAIN + struct.pack(">Q", len(entries))) - for path, entry in sorted(entries.items()): - content = blobs[entry.oid] - digest.update(struct.pack(">Q", len(path))) - digest.update(path) - digest.update(entry.mode) - digest.update(struct.pack(">Q", len(content))) - digest.update(content) - return digest.hexdigest() - - -def read_stats(repo: Path, commit: str) -> dict[str, str]: - entry = git(repo, "ls-tree", require_sha(commit), "--", ".castiron.stats.yml") - if not entry.startswith((b"100644 blob ", b"100755 blob ")): - raise ReportError("Castiron stats must be a regular Git file") - text = git(repo, "show", f"{require_sha(commit)}:.castiron.stats.yml").decode("utf-8") - values: dict[str, str] = {} - for line in text.splitlines(): - if not line.strip() or line.lstrip().startswith("#"): - continue - key, sep, value = line.partition(":") - if not sep or key.strip() in values: - raise ReportError("invalid or duplicate stats field") - values[key.strip()] = value.strip().strip("\"'") - if values.get("schema_version") != "1": - raise ReportError("unsupported Castiron stats schema") - require_sha(values.get("codegen_sha", "")) - if not re.fullmatch( - r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", - values.get("generation_id", ""), - ): - raise ReportError("invalid generation ID") - if "codegen_hash" in values and not HASH.fullmatch(values["codegen_hash"]): - raise ReportError("invalid codegen_hash") - if "public_codegen_sha" in values: - require_sha(values["public_codegen_sha"]) - return values - - -def public_metadata(repo: Path, commit: str) -> dict[str, str]: - message = git(repo, "show", "-s", "--format=%B", require_sha(commit)).decode("utf-8") - fields: dict[str, str] = {} - for key in ("Version", "Generation-ID", "Hash", "Branch"): - matches = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) - if len(matches) != 1: - raise ReportError("invalid public codegen metadata") - fields[key] = matches[0] - if fields["Version"] != "1" or not HASH.fullmatch(fields["Hash"]): - raise ReportError("unsupported public codegen metadata") - targets = re.findall(r"^Castiron-Public-Codegen-Target: (.*)$", message, re.MULTILINE) - if targets: - if len(targets) != 1 or targets[0] not in TARGET_BRANCHES: - raise ReportError("invalid public codegen target") - fields["Target"] = targets[0] - if not fields["Branch"].startswith("codegen/"): - raise ReportError("invalid public codegen branch") - git(repo, "check-ref-format", f"refs/heads/{fields['Branch']}") - for key in ("Adoption-Base", "Adoption-Source"): - values = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) - if values: - if len(values) != 1: - raise ReportError("duplicate adoption metadata") - fields[key] = require_sha(values[0]) - if ("Adoption-Base" in fields) != ("Adoption-Source" in fields): - raise ReportError("incomplete adoption metadata") - return fields - - -def legacy_snapshot_on_main(repo: Path, commit: str, *, fetch: bool) -> bool: - """Legacy snapshots have no target; trust only adoption in public main history.""" - if fetch: - git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - ) - revisions = ( - git( - repo, - "log", - "--first-parent", - "--format=%H", - f"-G{require_sha(commit)}", - "refs/remotes/origin/main", - "--", - ".castiron.stats.yml", - ) - .decode() - .splitlines() - ) - for revision in revisions: - try: - if read_stats(repo, revision).get("public_codegen_sha") == commit: - return True - except ReportError: - continue - return False - - -def validate_public_target( - repo: Path, - commit: str, - metadata: dict[str, str], - *, - target_branch: str, - fetch: bool, - bootstrap_lineage: str | None = None, -) -> None: - if target_branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - if target_branch != "next" and ( - metadata["Branch"] == "codegen/castiron/next" - or metadata["Branch"].startswith("codegen/castiron/next/") - ): - raise ReportError("next checkpoint cannot authorize a main report") - if target_branch == "main": - if metadata.get("Target") == "next": - raise ReportError("next checkpoint cannot authorize a main report") - if "Target" not in metadata and not legacy_snapshot_on_main(repo, commit, fetch=fetch): - if bootstrap_lineage is None: - raise ReportError("legacy snapshot has no verified main adoption") - # Pre-bootstrap SDK stats have no public snapshot pointer. Their - # baseline may be an ancestor of an adopted snapshot, never adopted - # directly. Validate the lineage independently before trusting it. - validate_public_target( - repo, - bootstrap_lineage, - public_metadata(repo, bootstrap_lineage), - target_branch=target_branch, - fetch=fetch, - ) - git(repo, "merge-base", "--is-ancestor", commit, bootstrap_lineage) - - -def resolve_public_baseline( - repo: Path, - revision: str, - *, - fetch: bool, - require_hash: bool, - lineage: str | None = None, - target_branch: str = "main", -) -> dict[str, Any]: - stats = read_stats(repo, revision) - commit = stats.get("public_codegen_sha") - if commit is None: - # Bootstrap old public main without changing its schema-v1 stats. Search - # only the already-verified public snapshot lineage supplied by PR head. - if lineage is None: - raise ReportError("stats do not contain public_codegen_sha") - for candidate in git(repo, "rev-list", require_sha(lineage)).decode().splitlines(): - if public_metadata(repo, candidate)["Generation-ID"] == stats["generation_id"]: - commit = candidate - break - if commit is None: - raise ReportError("public snapshot lineage does not contain the base generation") - elif fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) - metadata = public_metadata(repo, commit) - validate_public_target( - repo, - commit, - metadata, - target_branch=target_branch, - fetch=fetch, - bootstrap_lineage=lineage if stats.get("public_codegen_sha") is None else None, - ) - if metadata["Generation-ID"] != stats["generation_id"]: - raise ReportError("public codegen generation ID does not match stats") - ref = f"refs/heads/{metadata['Branch']}" - tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") - else: - tracking = f"refs/remotes/origin/{metadata['Branch']}" - git(repo, "merge-base", "--is-ancestor", commit, tracking) - actual = hash_codegen_commit(repo, commit) - if actual != metadata["Hash"]: - raise ReportError("public codegen content does not match snapshot metadata") - expected = stats.get("codegen_hash") - if expected is None and require_hash: - raise ReportError("head stats do not contain codegen_hash") - if expected is not None and expected != actual: - raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") - return { - "commit": commit, - "generation_id": stats["generation_id"], - "hash": actual, - "hash_recorded": expected is not None, - } - - -def public_snapshot_entries( - source: Path, - commit: str, - destination: Path, - public_base: str, - approved_tree: str | None = None, -) -> dict[bytes, Entry]: - entries = tree_entries(source, commit) - approved = tree_entries(destination, public_base) - if approved_tree is not None: - approved.update(tree_entries(destination, approved_tree)) - for path in entries: - if path in PUBLIC_BLOCKED_PATHS or path.startswith((b".castiron/", b".github/")): - raise ReportError(f"refusing to publish private or governance snapshot path: {path!r}") - if path not in approved: - raise ReportError( - f"snapshot path is absent from the approved public SDK tree: {path!r}" - ) - return entries - - -def copy_generated_tree( - source: Path, - commit: str, - destination: Path, - public_base: str, - approved_tree: str | None = None, -) -> str: - """Transfer approved blobs/trees only, never source commits or ancestors.""" - entries = public_snapshot_entries(source, commit, destination, public_base, approved_tree) - for oid, blob in read_blobs(source, entries).items(): - actual = git(destination, "hash-object", "-w", "--stdin", input_bytes=blob).decode().strip() - if actual != oid: - raise ReportError("source and destination Git object formats differ") - directories: dict[bytes, list[bytes]] = {b"": []} - for path, entry in entries.items(): - parent, _, name = path.rpartition(b"/") - directories.setdefault(parent, []).append( - entry.mode + b" blob " + entry.oid.encode() + b"\t" + name + b"\0" - ) - while parent: - parent = parent.rpartition(b"/")[0] - directories.setdefault(parent, []) - for directory in sorted(directories, key=lambda p: (p.count(b"/"), len(p)), reverse=True): - tree = git( - destination, "mktree", "-z", input_bytes=b"".join(directories[directory]) - ).strip() - if not directory: - return tree.decode() - parent, _, name = directory.rpartition(b"/") - directories[parent].append(b"040000 tree " + tree + b"\t" + name + b"\0") - raise ReportError("could not construct public generated tree") - - -def create_public_snapshot( - destination: Path, - tree: str, - generation: str, - content_hash: str, - branch: str, - parent: str | None, - adoption: tuple[str, str] | None = None, -) -> str: - target = ( - "next" - if branch == "codegen/castiron/next" or branch.startswith("codegen/castiron/next/") - else "main" - ) - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - f"Castiron-Public-Codegen-Target: {target}\n" - ) - if adoption is not None: - if target != "next": - raise ReportError("adoption snapshot must target next") - message += f"Castiron-Public-Codegen-Adoption-Base: {require_sha(adoption[0])}\n" - message += f"Castiron-Public-Codegen-Adoption-Source: {require_sha(adoption[1])}\n" - parents = ["-p", require_sha(parent)] if parent is not None else [] - snapshot = ( - git( - destination, - "-c", - "user.name=Castiron", - "-c", - "user.email=noreply@openai.com", - "commit-tree", - tree, - *parents, - "-m", - message, - ) - .decode() - .strip() - ) - require_sha(snapshot) - if hash_codegen_commit(destination, snapshot) != content_hash: - raise ReportError("public snapshot content hash differs from private checkpoint") - return snapshot - - -def first_adoption_source(repo: Path, base: str, *, target_branch: str, fetch: bool) -> str: - """Authority is the protected target base policy, never a candidate receipt.""" - base = require_sha(base) - if target_branch != "next": - raise ReportError("first adoption is next-only") - if git(repo, "ls-tree", base, "--", ".castiron.stats.yml"): - raise ReportError("existing/corrupt Castiron stats require ordinary recovery") - if git(repo, "log", "--full-history", "--format=%H", base, "--", ".castiron.stats.yml").strip(): - raise ReportError("deleted Castiron stats cannot authorize first adoption") - if not git(repo, "ls-tree", base, "--", ".stats.yml").startswith(b"100644 blob "): - raise ReportError("first adoption requires a legacy SDK target") - if not git(repo, "ls-tree", base, "--", ".castiron-ratchet.json").startswith(b"100644 blob "): - raise ReportError("first adoption requires an approved target policy") - - def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ReportError("duplicate policy field") - result[key] = value - return result - - policy = json.loads( - git(repo, "show", f"{base}:.castiron-ratchet.json"), object_pairs_hook=unique - ) - pin = policy.get("next_policy") - if not isinstance(pin, dict) or set(cast(dict[str, Any], pin)) != {"repository", "main_sha"}: - raise ReportError("first adoption requires the protected next-policy pin") - pin = cast(dict[str, Any], pin) - repository = pin["repository"] - if repository not in {"openai/openai-java", "openai/openai-cli"}: - raise ReportError("first adoption is only supported for legacy Java/CLI") - remote = git(repo, "remote", "get-url", "origin").decode().strip() - allowed = github_origin_urls(repository) | github_origin_urls(f"{repository}-internal") - if remote not in allowed: - raise ReportError("adoption policy repository does not match SDK origin") - source = require_sha(pin["main_sha"]) - main_ref = "refs/castiron/adoption-main" if fetch else "refs/remotes/origin/main" - if fetch: - git( - repo, - "fetch", - "--quiet", - "--no-tags", - f"https://github.com/{repository}.git", - f"+refs/heads/main:{main_ref}", - ) - if source not in git(repo, "rev-list", "--first-parent", main_ref).decode().splitlines(): - raise ReportError("adoption source is not on authenticated main history") - source_policy = json.loads( - git(repo, "show", f"{source}:.castiron-ratchet.json"), object_pairs_hook=unique - ) - if {key: value for key, value in policy.items() if key != "next_policy"} != source_policy: - raise ReportError("first adoption must preserve approved policy limits") - if not read_stats(repo, source).get("public_codegen_sha"): - raise ReportError( - "approved main must have a genuine public checkpoint before first adoption" - ) - return source - - -def adoption_head( - repo: Path, - base: str, - source: str, - head: str, - after: dict[str, Any], - *, - public: bool, -) -> None: - git(repo, "merge-base", "--is-ancestor", base, head) - if not public: - # Private initialization retains the authenticated source parent. Normal - # promotion deliberately publishes a single-parent target-branch commit. - git(repo, "merge-base", "--is-ancestor", source, head) - if git(repo, "show", f"{base}:.castiron-ratchet.json") != git( - repo, "show", f"{head}:.castiron-ratchet.json" - ): - raise ReportError("first adoption must preserve protected target policy") - original, current = read_stats(repo, source), read_stats(repo, head) - if original["generation_id"] == current["generation_id"]: - raise ReportError("first adoption requires a new generation") - for key in ("config_hash", "openapi_spec_hash"): - if not original.get(key) or current.get(key) != original[key]: - raise ReportError("first adoption must preserve authenticated main inputs") - if public: - # Public authorization comes from the protected pin and verified public - # snapshot ancestry, never from a candidate-supplied adoption receipt. - git(repo, "merge-base", "--is-ancestor", original["public_codegen_sha"], after["commit"]) - metadata = public_metadata(repo, after["commit"]) - if ( - metadata.get("Target") != "next" - or metadata.get("Adoption-Base") != base - or metadata.get("Adoption-Source") != source - ): - raise ReportError("snapshot does not bind the authorized first adoption") - else: - message = git(repo, "show", "-s", "--format=%B", current["codegen_sha"]).decode() - bindings = re.findall(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) - if len(bindings) != 1: - raise ReportError("invalid adoption generation binding") - envelope = json.loads(base64.b64decode(bindings[0], validate=True)) - record = envelope.get("record", {}) - if envelope.get("release_target") != "next" or record.get("public_baseline_commit") != base: - raise ReportError("new checkpoint must bind the old target base") - pin = json.loads(git(repo, "show", f"{base}:.castiron-ratchet.json"))["next_policy"] - sdk = pin["repository"].split("/")[1] - if record.get("target") != sdk or record.get("language") != sdk.removeprefix("openai-"): - raise ReportError("adoption generation SDK mismatch") - - -def prepare_public_snapshot( - source: Path, - source_base: str, - source_head: str, - destination: Path, - public_base: str, - branch: str, -) -> dict[str, Any]: - """Prepare public-only generated history; never push anything.""" - after = read_stats(source, source_head) - target_branch = ( - "next" if branch == "castiron/next" or branch.startswith("castiron/next/") else "main" - ) - adoption_source = None - if not git(destination, "ls-tree", public_base, "--", ".castiron.stats.yml"): - adoption_source = first_adoption_source( - destination, public_base, target_branch=target_branch, fetch=True - ) - else: - read_stats(source, source_base) - public = read_stats(destination, adoption_source or public_base) - baseline = resolve_checkpoint( - source, after, fetch=True, require_hash=True, target_branch=target_branch - ) - if adoption_source: - # Public source commit is already fetched into the private promotion clone. - adoption_head(source, public_base, adoption_source, source_head, baseline, public=False) - - codegen_branch = f"codegen/{branch}" - ref = f"refs/heads/{codegen_branch}" - git(destination, "check-ref-format", ref) - remote = git(destination, "ls-remote", "--refs", "origin", ref).decode().splitlines() - previous = require_sha(remote[0].split()[0]) if remote else "" - upgrade_legacy_parent = False - if previous: - git(destination, "fetch", "--quiet", "--no-tags", "origin", previous) - metadata = public_metadata(destination, previous) - # A pending legacy promotion has not been adopted on main. It may be - # retained as public history, but never reused to authorize the new - # snapshot: rebuild that from the verified private checkpoint below. - upgrade_legacy_parent = ( - target_branch == "main" - and "Target" not in metadata - and metadata["Branch"] == codegen_branch - ) - if not upgrade_legacy_parent: - validate_public_target( - destination, previous, metadata, target_branch=target_branch, fetch=True - ) - # Stats pin the snapshot, not the SDK branch that currently uses it. - # Another promotion may legitimately reuse this public commit; its - # original branch remains the locator used by resolve_public_baseline. - if hash_codegen_commit(destination, previous) != metadata["Hash"]: - raise ReportError("existing public codegen branch has invalid provenance") - - base_snapshot = public.get("public_codegen_sha") - if base_snapshot: - base_snapshot = resolve_public_baseline( - destination, - adoption_source or public_base, - fetch=True, - require_hash=False, - target_branch="main" if adoption_source else target_branch, - )["commit"] - elif previous: - try: - base_snapshot = resolve_public_baseline( - destination, - public_base, - fetch=True, - require_hash=False, - lineage=previous, - target_branch=target_branch, - )["commit"] - except ReportError: - base_snapshot = None - if base_snapshot is None: - old = resolve_checkpoint( - source, public, fetch=True, require_hash=False, target_branch=target_branch - ) - # Initial bootstrap is limited to paths already present on public main. - tree = copy_generated_tree(source, old["commit"], destination, public_base) - base_snapshot = create_public_snapshot( - destination, tree, old["generation_id"], old["hash"], codegen_branch, None - ) - - # Normal promotion has already applied the reviewed patch to the public - # index. Private HEAD can contain unchanged private-only paths, so it is - # not an approval boundary for publication. - approved_tree = git(destination, "write-tree").decode().strip() - tree = copy_generated_tree(source, baseline["commit"], destination, public_base, approved_tree) - parent = base_snapshot - if previous: - ancestors = git(destination, "rev-list", previous).decode().splitlines() - if base_snapshot in ancestors: - parent = previous - metadata = public_metadata(destination, previous) - if ( - not upgrade_legacy_parent - and metadata["Generation-ID"] == after["generation_id"] - and ( - not adoption_source - or ( - metadata.get("Adoption-Base") == public_base - and metadata.get("Adoption-Source") == adoption_source - ) - ) - and metadata["Hash"] == baseline["hash"] - and git(destination, "rev-parse", f"{previous}^{{tree}}").decode().strip() == tree - and base_snapshot in ancestors - ): - snapshot = previous - else: - snapshot = create_public_snapshot( - destination, - tree, - after["generation_id"], - baseline["hash"], - codegen_branch, - parent, - (public_base, adoption_source) if adoption_source else None, - ) - elif ( - public_metadata(destination, base_snapshot)["Generation-ID"] == after["generation_id"] - and git(destination, "rev-parse", f"{base_snapshot}^{{tree}}").decode().strip() == tree - ): - snapshot = base_snapshot - else: - snapshot = create_public_snapshot( - destination, - tree, - after["generation_id"], - baseline["hash"], - codegen_branch, - parent, - (public_base, adoption_source) if adoption_source else None, - ) - stats_path = destination / ".castiron.stats.yml" - current = git(source, "show", f"{require_sha(source_head)}:.castiron.stats.yml").decode("utf-8") - lines = [line for line in current.splitlines() if not line.startswith("public_codegen_sha:")] - stats_path.write_text("\n".join(lines) + f"\npublic_codegen_sha: {snapshot}\n") - git(destination, "add", "--", ".castiron.stats.yml") - return { - "commit": snapshot, - "base_commit": base_snapshot, - "branch": codegen_branch, - "previous": previous, - "hash": baseline["hash"], - } - - -def resolve_baseline( - repo: Path, revision: str, *, fetch: bool, require_hash: bool, target_branch: str = "main" -) -> dict[str, Any]: - return resolve_checkpoint( - repo, - read_stats(repo, revision), - fetch=fetch, - require_hash=require_hash, - target_branch=target_branch, - ) - - -def resolve_checkpoint( - repo: Path, - stats: dict[str, str], - *, - fetch: bool, - require_hash: bool, - target_branch: str = "main", -) -> dict[str, Any]: - if target_branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - commit = stats["codegen_sha"] - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) - message = git(repo, "show", "-s", "--format=%B", commit).decode("utf-8") - match = re.search(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) - if not match: - raise ReportError("codegen checkpoint has no generation metadata") - try: - decoded: object = json.loads(base64.b64decode(match[1], validate=True)) - except (ValueError, TypeError) as exc: - raise ReportError("invalid codegen checkpoint metadata") from exc - if not isinstance(decoded, dict): - raise ReportError("invalid codegen checkpoint metadata") - metadata = cast(dict[str, Any], decoded) - namespace = "" - if "schema_version" in metadata: - if ( - set(metadata) != {"schema_version", "release_target", "record"} - or metadata["schema_version"] != "castiron-checkpoint/v2" - or metadata["release_target"] != "next" - or not isinstance(metadata["record"], dict) - ): - raise ReportError("invalid versioned checkpoint envelope") - if target_branch != "next": - raise ReportError("next checkpoint cannot authorize a main report") - metadata = cast(dict[str, Any], metadata["record"]) - namespace = "castiron/next/" - if {"release_target", "schema_version", "record"}.intersection(metadata): - raise ReportError("checkpoint target metadata must use the versioned envelope") - if metadata.get("generation_id") != stats["generation_id"]: - raise ReportError("codegen checkpoint generation ID does not match stats") - branch = metadata.get("source_branch") - if not isinstance(branch, str) or not branch: - raise ReportError("codegen checkpoint has no source branch") - if not namespace and (branch == "castiron/next" or branch.startswith("castiron/next/")): - raise ReportError("legacy checkpoint cannot use the reserved next namespace") - ref = f"refs/heads/codegen/{namespace}{branch}" - git(repo, "check-ref-format", ref) - tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") - else: - tracking = f"refs/remotes/origin/codegen/{namespace}{branch}" - git(repo, "merge-base", "--is-ancestor", commit, tracking) - actual = hash_codegen_commit(repo, commit) - expected = stats.get("codegen_hash") - if expected is None and require_hash: - raise ReportError("head stats do not contain codegen_hash") - if expected is not None and actual != expected: - raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") - return { - "commit": commit, - "generation_id": stats["generation_id"], - "hash": actual, - "hash_recorded": expected is not None, - } - - -def numstats(repo: Path, generated: str, integrated: str) -> dict[bytes, tuple[str, str]]: - result: dict[bytes, tuple[str, str]] = {} - output = git( - repo, - "diff", - "--numstat", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - generated, - integrated, - "--", - ) - for record in output.split(b"\0"): - if record: - added, removed, path = record.split(b"\t", 2) - result[path] = (added.decode("ascii"), removed.decode("ascii")) - return result - - -def classify(g0: Entry | None, i0: Entry | None, g1: Entry | None, i1: Entry | None) -> str | None: - before = g0 is not None and g0 != i0 - after = g1 is not None and g1 != i1 - if g0 is not None and g1 is None and (before or i1 is not None): - return "no_longer_generated" - if g0 is None and g1 is not None and i0 is not None: - return "newly_generation_owned" - if not before and not after: - return None - if g0 is not None and g1 is not None and g0 != g1: - return "baseline_changed" - if not before: - return "newly_customized" - if not after: - return "removed" - return "unchanged" if i0 == i1 else "existing_changed" - - -def build_report( - repo: Path, - base: str, - head: str, - *, - fetch: bool = False, - require_head_hash: bool = False, - public: bool = False, - target_branch: str = "main", -) -> tuple[dict[str, Any], bytes]: - base, head = require_sha(base), require_sha(head) - comparison_base = git(repo, "merge-base", base, head).decode().strip() - adoption_source = None - if not git(repo, "ls-tree", comparison_base, "--", ".castiron.stats.yml"): - if comparison_base != base: - raise ReportError( - "first-adoption target advanced; regenerate against the protected base" - ) - adoption_source = first_adoption_source( - repo, comparison_base, target_branch=target_branch, fetch=fetch - ) - if public: - after = resolve_public_baseline( - repo, - head, - fetch=fetch, - require_hash=require_head_hash, - target_branch=target_branch, - ) - before = resolve_public_baseline( - repo, - adoption_source or comparison_base, - fetch=fetch, - require_hash=False, - lineage=after["commit"], - target_branch="main" if adoption_source else target_branch, - ) - else: - before = resolve_baseline( - repo, - adoption_source or comparison_base, - fetch=fetch, - require_hash=False, - target_branch="main" if adoption_source else target_branch, - ) - after = resolve_baseline( - repo, - head, - fetch=fetch, - require_hash=require_head_hash, - target_branch=target_branch, - ) - if adoption_source: - adoption_head(repo, comparison_base, adoption_source, head, after, public=public) - g0, i0 = ( - ({}, {}) - if adoption_source - else (tree_entries(repo, before["commit"]), tree_entries(repo, comparison_base)) - ) - g1, i1 = tree_entries(repo, after["commit"]), tree_entries(repo, head) - stats = numstats(repo, after["commit"], head) - files: list[dict[str, Any]] = [] - current_paths: list[str] = [] - for path in sorted(g0.keys() | g1.keys()): - category = classify(g0.get(path), i0.get(path), g1.get(path), i1.get(path)) - if category is None: - continue - custom_before = path in g0 and g0[path] != i0.get(path) - custom_after = path in g1 and g1[path] != i1.get(path) - added, removed = stats.get(path, ("0", "0")) if custom_after else ("0", "0") - files.append( - { - "path": path.decode("utf-8"), - "category": ( - ("first_adoption" if custom_after else "unchanged") - if adoption_source - else category - ), - "custom_before": None if adoption_source else custom_before, - "custom_after": custom_after, - "added": added, - "removed": removed, - } - ) - if custom_after: - current_paths.append(path.decode("utf-8")) - patch = b"" - if current_paths: - patch = git( - repo, - "--literal-pathspecs", - "diff", - "--binary", - "--full-index", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - "--no-color", - after["commit"], - head, - "--", - *current_paths, - ) - return { - "schema_version": 1, - "status": "ok", - "public": public, - "target_base_sha": base, - "base_sha": comparison_base, - "head_sha": head, - "before": None if adoption_source else before, - "adoption_baseline": before if adoption_source else None, - "adoption_source": adoption_source, - "before_status": "legacy_generation_unavailable" if adoption_source else "verified", - "after": after, - "files": files, - "counts": { - "before": None if adoption_source else sum(f["custom_before"] for f in files), - "after": sum(f["custom_after"] for f in files), - **dict(collections.Counter(f["category"] for f in files)), - }, - }, patch - - -def render_report( - report: dict[str, Any], - run_url: str = "", - *, - repository: str = "", - run_id: int = 0, - run_attempt: int = 0, -) -> str: - head = require_sha(report["head_sha"]) - lines = [MARKER, "", "## Castiron custom code", ""] - branch = report.get("target_branch", "main") - if branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - lines.extend([f"Evaluated {branch}: `{require_sha(report['target_base_sha'])}`.", ""]) - if "policy_sha" in report: - lines.extend([f"Trusted main policy: `{require_sha(report['policy_sha'])}`.", ""]) - if report.get("status") != "ok": - reason = html.escape(str(report.get("error", "Report could not be computed"))) - lines.extend([f"⚠️ Report unavailable for `{head[:12]}`.", "", reason]) - else: - files = report["files"] - counts: collections.Counter[str] = collections.Counter() - for file in files: - if file["category"] not in LABELS: - raise ReportError("unknown report category") - counts[file["category"]] += 1 - before = sum(bool(file["custom_before"]) for file in files) - after = sum(bool(file["custom_after"]) for file in files) - base = require_sha(report["base_sha"]) - newly_customized = sum( - bool(file["custom_after"]) and not file["custom_before"] for file in files - ) - removed = sum( - bool(file["custom_before"]) - and not file["custom_after"] - and file["category"] != "no_longer_generated" - for file in files - ) - if report.get("before_status") == "legacy_generation_unavailable": - lines.extend( - [ - "Legacy before-generation unavailable; no historical customization claim.", - "", - f"Current mixed files: {after}. Approved main reference: `{require_sha(report['adoption_source'])}`.", - ] - ) - elif newly_customized == 0: - remaining = ( - f"{after} mixed file{'s' if after != 1 else ''} remain{'s' if after == 1 else ''}" - ) - changed = counts["existing_changed"] - details = [f"{changed} existing customization{'s' if changed != 1 else ''} changed"] - if removed: - details.append(f"{removed} customizations removed") - if counts["baseline_changed"]: - details.append(f"{counts['baseline_changed']} generated baselines changed") - lines.extend( - [ - "✅ No new custom-code files detected.", - "", - remaining + "; " + "; ".join(details) + ".", - ] - ) - else: - lines.extend( - [ - f"**Mixed files: {before} → {after}**", - "", - ( - f"{newly_customized} newly customized · {removed} customizations removed · " - f"{counts['existing_changed']} existing customizations changed · {counts['baseline_changed']} generated baselines changed" - ), - ] - ) - verification = ( - "Current generation and approved main reference verified." - if report["before"] is None - else "Generated baselines verified." - ) - lines.extend(["", f"Compared `{base[:12]}` → `{head[:12]}`. {verification}"]) - if ( - not (report["before"] or report["adoption_baseline"])["hash_recorded"] - or not report["after"]["hash_recorded"] - ): - lines.extend( - [ - "", - "Legacy baseline: content hash computed from the pinned checkpoint; not yet recorded in that revision’s stats.", - ] - ) - interesting = [file for file in files if file["category"] != "unchanged"] - if interesting: - lines.extend(["", "| File | Result | Current custom patch |", "| --- | --- | --- |"]) - for file in interesting[:40]: - path = ( - html.escape(str(file["path"])) - .replace("|", "|") - .replace("\n", " ") - .replace("\r", " ") - ) - added, removed = str(file["added"]), str(file["removed"]) - if not re.fullmatch(r"\d+|-", added) or not re.fullmatch(r"\d+|-", removed): - raise ReportError("invalid line counts") - delta = f"+{added} / −{removed}" if file["custom_after"] else "None" - lines.append(f"| {path} | {LABELS[file['category']]} | {delta} |") - if len(interesting) > 40: - lines.extend( - [ - "", - f"{len(interesting) - 40} more affected files are listed in the full report.", - ] - ) - if counts["unchanged"]: - lines.extend( - [ - "", - f"
{counts['unchanged']} existing customizations unchanged", - "", - ] - ) - for file in [file for file in files if file["category"] == "unchanged"][:40]: - lines.append(f"- {html.escape(str(file['path']))}") - if counts["unchanged"] > 40: - lines.extend(["", f"{counts['unchanged'] - 40} more in the full report."]) - lines.extend(["", "
"]) - lines.extend( - [ - "", - "A changed generated baseline means this report cannot reliably identify which handwritten lines changed.", - ] - ) - target_base = require_sha(report["target_base_sha"]) - lines.extend(["", "
Inspect the custom-code diff", ""]) - if repository: - if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: - raise ReportError("invalid artifact download target") - artifact = f"castiron-custom-code-{run_id}-{run_attempt}" - lines.extend( - [ - "Download the exact patch produced by this run (requires repository access):", - "", - "```sh", - f"gh run download {run_id} --repo {repository} \\", - f" --name {artifact} --dir /tmp/{artifact}", - f"git apply --stat /tmp/{artifact}/custom-code.patch", - f"cat /tmp/{artifact}/custom-code.patch", - "```", - "", - ] - ) - lines.extend( - [ - "Or reproduce it from an SDK checkout containing the vendored reporter:", - "", - "```sh", - f"git fetch --no-tags origin {target_base} {head}", - "python3 scripts/castiron/custom_code_report.py report \\", - f" --base {target_base} \\", - f" --head {head} --target-branch {branch} --fetch --require-head-hash{' --public' if report.get('public') else ''} \\", - f" --out /tmp/castiron-custom-code-{head[:12]}", - f"cat /tmp/castiron-custom-code-{head[:12]}/custom-code.patch", - "```", - "", - "This is the **current full custom patch** for mixed files, not an attribution of only the handwritten lines changed by this PR.", - "", - "
", - ] - ) - if run_url: - lines.extend(["", f"[Full report and patch]({run_url})"]) - return "\n".join(lines) + "\n" - - -def api(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - if not token: - raise ReportError("GitHub token is required for comment publication") - args = ["gh", "api", "--hostname", "github.com", "--method", method, path] - if payload is not None: - args.extend(["--input", "-"]) - result = subprocess.run( - args, - input=json.dumps(payload).encode() if payload is not None else None, - capture_output=True, - check=False, - timeout=60, - ) - if result.returncode: - raise ReportError( - f"GitHub API request failed: {result.stderr.decode(errors='replace').strip()}" - ) - return json.loads(result.stdout) - - -def associated_pulls( - repository: str, run: dict[str, Any], *, branch_only: bool = False -) -> list[dict[str, Any]]: - root = f"repos/{repository}" - associated: list[dict[str, Any]] = ( - [] - if branch_only - else run["pull_requests"] - or api("GET", f"{root}/commits/{require_sha(run['head_sha'])}/pulls?per_page=100") - ) - if associated: - return associated - # Fork runs can be absent from both association endpoints. Discover candidates - # by branch; callers still verify the current PR head and target through GitHub. - source: dict[str, Any] = run.get("head_repository") or {} - if not ( - source.get("id") - and source.get("full_name") - and source.get("owner", {}).get("login") - and run.get("head_branch") - ): - return [] - head = f"{source['owner']['login']}:{run['head_branch']}" - for page in range(1, 101): - query = urlencode({"state": "open", "head": head, "per_page": 100, "page": page}) - pulls = api("GET", f"{root}/pulls?{query}") - associated.extend(pulls) - if len(pulls) < 100: - return associated - raise ReportError("too many pull requests for source branch") - - -def matches_source(pull: dict[str, Any], run: dict[str, Any]) -> bool: - """A shared commit is discovery metadata, not authority over another PR.""" - source: dict[str, Any] = run.get("head_repository") or {} - head: dict[str, Any] = pull.get("head") or {} - repository: dict[str, Any] = head.get("repo") or {} - return bool( - source.get("id") - and source.get("full_name") - and run.get("head_branch") - and repository.get("id") == source["id"] - and repository.get("full_name") == source["full_name"] - and head.get("ref") == run["head_branch"] - and head.get("sha") == run["head_sha"] - ) - - -def current_pull(repository: str, run: dict[str, Any]) -> tuple[int, str] | None: - """Resolve one live supported target; association metadata is discovery only.""" - current: list[tuple[int, str]] = [] - pulls = associated_pulls(repository, run) - for attempt in range(2): - for number in sorted({int(pr["number"]) for pr in pulls}): - if number <= 0: - raise ReportError("invalid associated pull request") - pull = api("GET", f"repos/{repository}/pulls/{number}") - if ( - pull["state"] == "open" - and matches_source(pull, run) - and pull["base"]["repo"]["full_name"] == repository - and pull["base"]["ref"] in TARGET_BRANCHES - ): - current.append((number, pull["base"]["ref"])) - if current or run["pull_requests"] or attempt: - break - # Nonempty commit associations can contain only unrelated PRs. - pulls = associated_pulls(repository, run, branch_only=True) - # Fallback discovery can find both targets for one head. Main's required - # checks take precedence over next's advisory report; keep same-target - # ambiguity fail-closed. Explicit run associations still limit discovery. - current = [pull for pull in current if pull[1] == "main"] or current - if len(current) > 1: - raise ReportError("workflow run has multiple current pull requests") - return current[0] if current else None - - -def publish_comment( - report: dict[str, Any], - repository: str, - number: int, - run_id: int, - run_attempt: int, - *, - artifact_run_id: int = 0, - artifact_run_attempt: int = 0, -) -> str: - if not REPOSITORY.fullmatch(repository) or min(number, run_id, run_attempt) <= 0: - raise ReportError("invalid GitHub publication target") - root = f"repos/{repository}" - pull = api("GET", f"{root}/pulls/{number}") - if ( - pull["state"] != "open" - or pull["head"]["sha"] != report["head_sha"] - or pull["base"]["ref"] != report.get("target_branch", "main") - or pull["base"]["ref"] not in TARGET_BRANCHES - or pull["base"]["repo"]["full_name"] != repository - ): - return "Skipped stale report" - run = api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["event"] != "pull_request" - or run.get("path", "").split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["head_sha"] != report["head_sha"] - ): - raise ReportError("workflow run does not match report PR/head") - if run["run_attempt"] != run_attempt or not matches_source(pull, run): - return "Skipped stale report" - if current_pull(repository, run) != (number, pull["base"]["ref"]): - raise ReportError("workflow run does not match report PR/head") - artifact_run_id = artifact_run_id or run_id - artifact_run_attempt = artifact_run_attempt or run_attempt - body = render_report( - report, - f"https://github.com/{repository}/actions/runs/{artifact_run_id}", - repository=repository, - run_id=artifact_run_id, - run_attempt=artifact_run_attempt, - ) - body += f"\n\n" - found = None - for page in range(1, 101): - comments = api("GET", f"{root}/issues/{number}/comments?per_page=100&page={page}") - for comment in comments: - if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( - MARKER - ): - found = comment - break - if found is not None or len(comments) < 100: - break - if found is not None: - previous = RUN_MARKER.search(found["body"]) - if previous and tuple(map(int, previous.groups())) > (run_id, run_attempt): - return "Skipped stale report" - if found["body"] == body: - return str(found["html_url"]) - # The workflow serializes publishers; recheck after pagination before writing. - pull = api("GET", f"{root}/pulls/{number}") - if ( - pull["state"] != "open" - or pull["head"]["sha"] != report["head_sha"] - or pull["base"]["ref"] != report.get("target_branch", "main") - or pull["base"]["ref"] not in TARGET_BRANCHES - or pull["base"]["repo"]["full_name"] != repository - or not matches_source(pull, run) - ): - return "Skipped stale report" - if found is not None: - result = api("PATCH", f"{root}/issues/comments/{found['id']}", {"body": body}) - else: - result = api("POST", f"{root}/issues/{number}/comments", {"body": body}) - return str(result["html_url"]) - - -def write_report( - repo: Path, - base: str, - head: str, - out: Path, - *, - fetch: bool, - require_head_hash: bool, - public: bool, - target_branch: str = "main", - policy_sha: str | None = None, -) -> dict[str, Any]: - out.mkdir(parents=True, exist_ok=True) - try: - report, patch = build_report( - repo, - base, - head, - fetch=fetch, - require_head_hash=require_head_hash, - public=public, - target_branch=target_branch, - ) - except (ReportError, UnicodeError, KeyError, ValueError) as exc: - report = { - "schema_version": 1, - "status": "error", - "target_base_sha": require_sha(base), - "head_sha": require_sha(head), - "error": str(exc), - } - patch = b"" - report["target_branch"] = target_branch - if policy_sha is not None: - report["policy_sha"] = require_sha(policy_sha) - (out / "report.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - (out / "custom-code.patch").write_bytes(patch) - summary = render_report(report) - (out / "summary.md").write_text(summary) - sys.stdout.write(summary) - return report - - -def trusted_report( - repo: Path, repository: str, run_id: int, run_attempt: int, out: Path, *, base: str -) -> None: - """Recompute from GitHub-associated Git objects, never from PR-produced artifacts.""" - # Executable code and policy always come from the pinned main checkout. - # The target branch is independently resolved from live GitHub metadata. - policy_sha = require_sha(base) - if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: - raise ReportError("invalid GitHub report target") - root = f"repos/{repository}" - run = api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["event"] != "pull_request" - or run.get("path", "").split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["status"] != "completed" - ): - raise ReportError("unexpected source workflow run") - if run["run_attempt"] != run_attempt: - return - head = require_sha(run["head_sha"]) - current = current_pull(repository, run) - if current is None: - return - number, branch = current - metadata = api("GET", root) - if metadata["default_branch"] != "main" or run["repository"]["full_name"] != repository: - raise ReportError("trusted reports require the repository's main default branch") - public = not metadata["private"] - base = ( - policy_sha - if branch == "main" - else require_sha(api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"]) - ) - # This must be a new, bare repository: no PR worktree, hooks, configuration, - # submodules, or Python imports can affect the trusted reporter. - repo.mkdir() - git(repo, "init", "--quiet", "--bare") - git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - git(repo, "fetch", "--quiet", "--no-tags", "origin", *dict.fromkeys((base, head, policy_sha))) - write_report( - repo, - base, - head, - out, - fetch=True, - require_head_hash=True, - public=public, - target_branch=branch, - policy_sha=policy_sha, - ) - (out / "context.json").write_text( - json.dumps({"pr": number, "repository": repository, "run": run_id, "attempt": run_attempt}) - + "\n" - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - hashing = commands.add_parser("hash") - hashing.add_argument("--repo", type=Path, default=Path.cwd()) - hashing.add_argument("--commit", required=True) - reporting = commands.add_parser("report") - reporting.add_argument("--repo", type=Path, default=Path.cwd()) - reporting.add_argument("--base", required=True) - reporting.add_argument("--head", required=True) - reporting.add_argument("--out", type=Path, required=True) - reporting.add_argument("--fetch", action="store_true") - reporting.add_argument("--require-head-hash", action="store_true") - reporting.add_argument("--public", action="store_true") - reporting.add_argument("--target-branch", choices=sorted(TARGET_BRANCHES), default="main") - trusted = commands.add_parser("trusted-report") - trusted.add_argument("--repo", type=Path, required=True) - trusted.add_argument("--repository", required=True) - trusted.add_argument("--run-id", type=int, required=True) - trusted.add_argument("--run-attempt", type=int, required=True) - trusted.add_argument("--out", type=Path, required=True) - trusted.add_argument("--base", required=True, help="immutable main SHA of the trusted checkout") - selecting = commands.add_parser("promotion-base") - selecting.add_argument("--repo", type=Path, required=True) - selecting.add_argument("--base", required=True) - selecting.add_argument("--branch", required=True) - preparing = commands.add_parser("prepare-public") - preparing.add_argument("--source-repo", type=Path, required=True) - preparing.add_argument("--source-base", required=True) - preparing.add_argument("--source-head", required=True) - preparing.add_argument("--public-repo", type=Path, required=True) - preparing.add_argument("--public-base", required=True) - preparing.add_argument("--branch", required=True) - commenting = commands.add_parser("comment") - commenting.add_argument("--report", type=Path, required=True) - commenting.add_argument("--repository", required=True) - commenting.add_argument("--pr", type=int, required=True) - commenting.add_argument("--run-id", type=int, required=True) - commenting.add_argument("--run-attempt", type=int, required=True) - commenting.add_argument("--artifact-run-id", type=int, default=0) - commenting.add_argument("--artifact-run-attempt", type=int, default=0) - args = parser.parse_args() - try: - if args.command == "hash": - sys.stdout.write(hash_codegen_commit(args.repo, args.commit) + "\n") - elif args.command == "promotion-base": - base = require_sha(args.base) - if not git(args.repo, "ls-tree", base, "--", ".castiron.stats.yml"): - target = ( - "next" - if args.branch == "castiron/next" or args.branch.startswith("castiron/next/") - else "main" - ) - base = first_adoption_source(args.repo, base, target_branch=target, fetch=True) - sys.stdout.write(base + "\n") - elif args.command == "prepare-public": - sys.stdout.write( - json.dumps( - prepare_public_snapshot( - args.source_repo, - args.source_base, - args.source_head, - args.public_repo, - args.public_base, - args.branch, - ) - ) - + "\n" - ) - elif args.command == "trusted-report": - trusted_report( - args.repo, - args.repository, - args.run_id, - args.run_attempt, - args.out, - base=args.base, - ) - elif args.command == "comment": - if args.report.stat().st_size > 5_000_000: - raise ReportError("report artifact is too large") - report = json.loads(args.report.read_text()) - sys.stdout.write( - publish_comment( - report, - args.repository, - args.pr, - args.run_id, - args.run_attempt, - artifact_run_id=args.artifact_run_id, - artifact_run_attempt=args.artifact_run_attempt, - ) - + "\n" - ) - else: - report = write_report( - args.repo, - args.base, - args.head, - args.out, - fetch=args.fetch, - require_head_hash=args.require_head_hash, - public=args.public, - target_branch=args.target_branch, - ) - return 0 if report["status"] == "ok" else 1 - except (ReportError, OSError, subprocess.TimeoutExpired) as exc: - sys.stderr.write(f"Castiron custom-code report: {exc}\n") - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_test_support.py b/scripts/castiron/custom_code_test_support.py deleted file mode 100644 index 812ec45acd..0000000000 --- a/scripts/castiron/custom_code_test_support.py +++ /dev/null @@ -1,116 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Small Git/checkpoint fixture shared by the offline Castiron tests.""" - -from __future__ import annotations - -import base64 -import json -import subprocess -import tempfile -import unittest -from pathlib import Path -from typing import Any, cast - -import custom_code_report as report - -GENERATION = "550e8400-e29b-41d4-a716-446655440000" - - -class GitTestCase(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.repo = Path(self.temporary.name) - self.git("init", "-q", "-b", "main") - self.git("config", "user.name", "Castiron test") - self.git("config", "user.email", "castiron@example.test") - # Finish automatic maintenance before the temporary repository is removed. - self.git("config", "maintenance.autoDetach", "false") - self.git("config", "gc.autoDetach", "false") - - def git(self, *args: str) -> str: - return subprocess.run( - ["git", "-C", str(self.repo), *args], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - def write(self, path: str, body: str) -> None: - target = self.repo / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(body) - - def commit(self, message: str = "fixture") -> str: - self.git("add", "-A") - self.git("commit", "-q", "--allow-empty", "-m", message) - return self.git("rev-parse", "HEAD") - - def baseline(self) -> tuple[str, str]: - self.write("generated.py", "generated\n") - metadata = { - "generation_id": GENERATION, - "source_branch": "test", - "target": "openai-python", - "language": "python", - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - generated = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/test", generated) - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {GENERATION}\ncodegen_sha: {generated}\ncodegen_hash: {report.hash_codegen_commit(self.repo, generated)}\n", - ) - return generated, self.commit("integrated") - - -def source_run(head: str, event: str = "pull_request") -> dict[str, Any]: - return { - "id": 123, - "workflow_id": 456, - "event": event, - "head_sha": head, - "head_branch": "gh-readonly-queue/main/pr-7-example" if event == "merge_group" else "sdk", - "repository": {"full_name": "openai/example"}, - "head_repository": { - "id": 10, - "full_name": "contributor/example", - "owner": {"login": "contributor"}, - }, - "path": ".github/workflows/castiron-custom-code.yml", - "status": "completed", - "run_attempt": 1, - "pull_requests": [{"number": 3}] if event == "pull_request" else [], - "conclusion": "failure", # The candidate's result is deliberately ignored. - } - - -def source_head(sha: str) -> dict[str, Any]: - return {"sha": sha, "ref": "sdk", "repo": source_run(sha)["head_repository"]} - - -WORKFLOW = ( - Path(__file__).resolve().parents[2] / ".github/workflows/castiron-custom-code-comment.yml" -) - - -def run_publisher(step: str, payload: dict[str, Any]) -> list[dict[str, Any]]: - section = WORKFLOW.read_text().split(f"- name: {step}\n", 1)[1] - script = section.split(" script: |\n", 1)[1] - lines: list[str] = [] - for line in script.splitlines(): - if line and not line.startswith(" "): - break - lines.append(line[12:]) - return run_workflow_script("\n".join(lines), payload) - - -def run_workflow_script(script: str, payload: dict[str, Any]) -> list[dict[str, Any]]: - output = subprocess.run( - ["node", str(Path(__file__).with_name("fixtures") / "github_publisher.cjs")], - input=json.dumps({**payload, "script": script}), - text=True, - capture_output=True, - check=True, - ) - return cast(list[dict[str, Any]], json.loads(output.stdout)) diff --git a/scripts/castiron/fixtures/github_publisher.cjs b/scripts/castiron/fixtures/github_publisher.cjs deleted file mode 100644 index 58324d2496..0000000000 --- a/scripts/castiron/fixtures/github_publisher.cjs +++ /dev/null @@ -1,183 +0,0 @@ -// File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -// Execute the real workflow script with an offline GitHub API and capture writes. -const fs = require('node:fs'); -const data = JSON.parse(fs.readFileSync(0, 'utf8')); -const writes = []; -const github = { - rest: { - actions: { - getWorkflowRun: async ({ run_id }) => { - if (data.run_errors?.[run_id]) - throw Object.assign(new Error('Workflow run lookup failed'), { - status: data.run_errors[run_id], - }); - return { data: data.runs_by_id?.[run_id] || data.run || data.context?.payload?.workflow_run }; - }, - getWorkflow: async ({ workflow_id }) => ({ - data: { - id: workflow_id === 'castiron-custom-code-comment.yml' ? 789 : 456, - path: `.github/workflows/${workflow_id}`, - }, - }), - listWorkflowRuns: 'workflow-runs', - reRunWorkflow: async (value) => { - if (data.fail_rerun) throw new Error('Simulated dispatch failure'); - writes.push({ operation: 'rerun', ...value }); - }, - }, - pulls: { - get: async ({ pull_number }) => ({ - data: data.current_sequence?.shift() || data.pulls_by_number?.[pull_number] || data.current, - }), - list: 'pulls', - listReviews: 'reviews', - listFiles: 'files', - }, - git: { - getRef: async ({ ref }) => ({ - data: { - object: { - sha: ref.startsWith('pull/') - ? data.merge_sha - : ref === 'heads/main' && (data.main_sha_sequence?.length || data.main_sha) - ? data.main_sha_sequence?.shift() || data.main_sha - : ref === 'heads/next' && (data.target_sha_sequence?.length || data.target_sha) - ? data.target_sha_sequence?.shift() || data.target_sha - : data.current.base.sha, - }, - }, - }), - }, - repos: { - getContent: async ({ path, ref }) => { - if ( - !['scripts/castiron/custom_code_policy.py', 'scripts/castiron/custom_code_report.py'].includes(path) - ) - throw new Error('Unexpected trusted policy path'); - const content = Buffer.from(JSON.stringify({ ref, path })); - return { - data: { - type: 'file', - encoding: 'base64', - size: content.length, - content: content.toString('base64'), - ...(data.policy_file_override || {}), - }, - }; - }, - getCommit: async () => ({ data: data.merge_commit }), - compareCommitsWithBasehead: async ({ basehead }) => ({ - data: data.comparisons?.[basehead] || { - merge_base_commit: { sha: basehead.split('...')[0] }, - }, - }), - createCommitStatus: async (value) => { - writes.push(value); - data.statuses_by_sha ||= {}; - data.statuses_by_sha[value.sha] = [ - { ...value, creator: { login: 'github-actions[bot]' } }, - ...(data.statuses_by_sha[value.sha] || data.previous_statuses || []), - ]; - }, - listCommitStatusesForRef: 'statuses', - listPullRequestsAssociatedWithCommit: 'associations', - }, - issues: { - listComments: 'comments', - createComment: async (value) => writes.push({ operation: 'create', ...value }), - updateComment: async (value) => writes.push({ operation: 'update', ...value }), - }, - }, - graphql: async (_query, { branch, cursor, number }) => { - if (_query.includes('reviews(first:100')) { - const page = data.review_pages?.shift(); - if (page && page.expected_cursor !== cursor) throw new Error('Wrong review cursor'); - if (page?.after_read) Object.assign(data, page.after_read); - return { - repository: { - pullRequest: { - reviews: page || { - nodes: data.review_nodes_by_number?.[number] || data.review_nodes || [], - pageInfo: { hasNextPage: false, endCursor: null }, - }, - }, - }, - }; - } - if (data.forbid_queue) throw new Error('Unexpected queue discovery'); - if ((branch || 'next') !== data.current.base.ref) throw new Error('Wrong queue target'); - const page = data.queue_pages?.shift(); - if (page && page.expected_cursor !== cursor) throw new Error('Wrong queue cursor'); - return { - repository: { - mergeQueue: { - entries: page || { - nodes: - data.queue_nodes || (data.queue_absent ? [] : [{ headCommit: { oid: data.run.head_sha } }]), - pageInfo: { hasNextPage: false, endCursor: null }, - }, - }, - }, - }; - }, - paginate: async (method, params) => { - if (method === 'files') return data.files || [{ filename: '.castiron-ratchet.json' }]; - if (method === 'workflow-runs') return data.runs_by_sha?.[params.head_sha] || []; - if (method === 'reviews') return data.reviews_by_number?.[params.pull_number] || data.reviews || []; - if (method === 'statuses') return data.statuses_by_sha?.[params.ref] || data.previous_statuses || []; - if (method === 'comments') return data.comments_by_number?.[params.issue_number] || data.comments || []; - if (method === 'associations') return data.associated_pulls || []; - if (method === 'pulls' && params.head === 'contributor:sdk' && params.state === 'open') { - return data.fallback_pulls || []; - } - throw new Error(`Unexpected GitHub lookup: ${method}`); - }, -}; -const exec = { - getExecOutput: async (command, args) => { - if ( - command !== 'python3' || - args[0] !== '-I' || - require('node:path').basename(args[1]) !== 'custom_code_policy.py' || - args[2] !== 'review-required' || - args[3] !== '--repository' || - args[4] !== 'openai/example' || - args[5] !== '--repo' || - args[7] !== '--base' || - args[9] !== '--head' || - args[11] !== '--main' || - ![args[8], args[10], args[12]].every((value) => /^[0-9a-f]{40}$/.test(value)) - ) { - throw new Error('Unexpected trusted policy invocation'); - } - for (const name of ['custom_code_policy.py', 'custom_code_report.py']) { - const file = require('node:path').join(require('node:path').dirname(args[1]), name); - const source = JSON.parse(fs.readFileSync(file, 'utf8')); - if (source.ref !== args[12] || source.path !== `scripts/castiron/${name}`) - throw new Error('Executed stale or mismatched trusted policy'); - } - if (data.policy_error_sequence?.shift() || data.policy_error) - throw new Error('Trusted policy evaluation failed'); - const expectedBase = data.expected_policy_bases?.shift() || data.expected_policy_base; - if (expectedBase && args[8] !== expectedBase) throw new Error('Evaluated stale target policy'); - return { - stdout: JSON.stringify({ - requires_human: data.requires_human_by_main?.[args[12]] ?? data.requires_human ?? true, - }), - }; - }, -}; -const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; -new AsyncFunction('github', 'context', 'process', 'require', 'exec', data.script)( - github, - data.context, - { env: { RUNNER_TEMP: require('node:os').tmpdir(), ...(data.env || {}) } }, - require, - exec, -) - .then(() => process.stdout.write(JSON.stringify(writes))) - .catch((error) => { - if (data.capture_writes) fs.writeFileSync(data.capture_writes, JSON.stringify(writes)); - console.error(error); - process.exitCode = 1; - }); diff --git a/scripts/castiron/generated-files.json b/scripts/castiron/generated-files.json index a2a325638e..4255b330c6 100644 --- a/scripts/castiron/generated-files.json +++ b/scripts/castiron/generated-files.json @@ -1,86 +1,10 @@ { "schema_version": 1, "files": { - ".github/workflows/castiron-custom-code-comment.yml": { - "mode": "100644", - "sha256": "ec43a2286a8f61efc2bfade540b4ccf572aac51fe0d2796121700f64f5bfce9b" - }, - ".github/workflows/castiron-custom-code.yml": { - "mode": "100644", - "sha256": "b52dcb3feb424790aff1f6f7cff7f385bd921369257b8e59be2e3860ec1f4926" - }, "api.md": { "mode": "100644", "sha256": "fc38e1d873ac4544fa16fe74478a228d86058ad681f2c7b939dff463015c928b" }, - "scripts/castiron/CUSTOM_CODE.md": { - "mode": "100644", - "sha256": "3014fb451a9bf3e924d3fc4f3eb65e9688aff7e8a109f20d90dad56a4736bcc4" - }, - "scripts/castiron/README.md": { - "mode": "100644", - "sha256": "4aff82cb988d01babde79902ae20de842fdd4b2b447d5f4cb3c5ce0a36d6fab8" - }, - "scripts/castiron/custom_code_budget.py": { - "mode": "100755", - "sha256": "b85ae4eab2a84dfbcd1f197f87077f686bc4f3347dbf74cca3abb1f99ab50b9a" - }, - "scripts/castiron/custom_code_policy.py": { - "mode": "100755", - "sha256": "890ecfc963b8a2461e7e801c9071735a975ed9876de89c80323da10960965295" - }, - "scripts/castiron/custom_code_report.py": { - "mode": "100755", - "sha256": "ddb15c9a162d26a6c9abf1b4fefcd80f2e685d9d255af729863258b7ecd08e2d" - }, - "scripts/castiron/custom_code_test_support.py": { - "mode": "100644", - "sha256": "b90c31b6b7347a3bc254b9fec228d5ccad3025306c60465a027a239420276671" - }, - "scripts/castiron/fixtures/github_publisher.cjs": { - "mode": "100644", - "sha256": "c84bd86518a70a5b7bbf877d8c13129debface0c220e5118510b0ef17d9ab707" - }, - "scripts/castiron/ruff.toml": { - "mode": "100644", - "sha256": "933d12c5582988097578a13689e1f7c877c36baf2c8c0e7123393e272ca023bd" - }, - "scripts/castiron/test_custom_code_budget.py": { - "mode": "100644", - "sha256": "7fdb3a9282978e6b1c0d84d73c43bb4adf96ff035663e77bc8e45f0fe082be7d" - }, - "scripts/castiron/test_custom_code_github.py": { - "mode": "100644", - "sha256": "815e8db60dd81f69895696813f6b612856eb899803e3e9b85df22ec688073b87" - }, - "scripts/castiron/test_custom_code_policy.py": { - "mode": "100644", - "sha256": "fd503013d604cf9659df95a3a47cecfc17da364397625d95ef6121aa790e6a05" - }, - "scripts/castiron/test_custom_code_policy_source.py": { - "mode": "100644", - "sha256": "dcb0be5e8157791dc02d1ae1d6b82cc703d4bc7b3c4797bc00bd9d2672ad13a2" - }, - "scripts/castiron/test_custom_code_publication.py": { - "mode": "100644", - "sha256": "4533c8c852d590446180715adbd00a63c12af29c53316d3e91d129890ccadbc6" - }, - "scripts/castiron/test_custom_code_report.py": { - "mode": "100644", - "sha256": "e1fbafb5242cac6128757339d78902b12e7658c82e5f157cc2ca7bef80347838" - }, - "scripts/castiron/test_custom_code_review_revocation.py": { - "mode": "100644", - "sha256": "d7f309e2324f48f62f9f7f90c2e82f31354e414a20e3ddb0da724b60adeff7a3" - }, - "scripts/castiron/test_custom_code_status_publication.py": { - "mode": "100644", - "sha256": "7398877d4ec209d0cd40e0d83f8e4482b6de5bef95f1711c9905d6a97a3791d3" - }, - "scripts/castiron/test_custom_code_strict.py": { - "mode": "100644", - "sha256": "20684147ec37f8233798a46aef8b618a2bbf2c8b507605a959d644fce163d73d" - }, "src/client.ts": { "mode": "100644", "sha256": "2c261bd7faffbd847f45acaa5f0dbd433764b638431ad05d4e841cae6061b0f5" diff --git a/scripts/castiron/ruff.toml b/scripts/castiron/ruff.toml deleted file mode 100644 index 1e8a301260..0000000000 --- a/scripts/castiron/ruff.toml +++ /dev/null @@ -1,3 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -line-length = 100 -target-version = "py310" diff --git a/scripts/castiron/test_custom_code_budget.py b/scripts/castiron/test_custom_code_budget.py deleted file mode 100644 index 1290c57161..0000000000 --- a/scripts/castiron/test_custom_code_budget.py +++ /dev/null @@ -1,430 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regression tests for the custom-code budget. -from __future__ import annotations - -import json -import subprocess -import sys -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_budget as budget -from custom_code_test_support import GitTestCase - - -class GoRatchetTests(GitTestCase): - def prepare(self, *, inherited_ci: bool = False) -> str: - if inherited_ci: - self.write(".github/actions/check/main.go", "package main\n") - self.baseline() - self.policy() - return self.commit("activate Go ratchet") - - def policy(self, *, active: bool = True, limit: int = 2000) -> None: - value: dict[str, Any] = {"schema_version": 1, "max_custom_patch_lines": limit} - if active: - value["ratchet"] = "go" - self.write(budget.POLICY, json.dumps(value) + "\n") - - def evaluate(self, base: str, head: str, **kwargs: Any) -> dict[str, Any]: - return budget.evaluate(self.repo, base, head, public=False, **kwargs)[0] - - def test_handwritten_helpers_tests_comments_and_nested_modules_count(self) -> None: - base = self.prepare() - for path in ( - "helper.go", - "helper_test.go", - "nested/module/helper.go", - ".github/workflows/check.go", - ): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", base) - self.write(path, "// comment\n\npackage example\n// final line") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["ratchet"]["delta"], 4) - self.assertEqual( - result["ratchet"]["after"]["handwritten_files"], [{"path": path, "lines": 4}] - ) - - def test_inherited_private_checkpoint_ci_is_handwritten(self) -> None: - base = self.prepare(inherited_ci=True) - self.write(".github/actions/check/main.go", "package main\n// added\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["total"], 0) - self.assertEqual(result["ratchet"]["before"]["handwritten_lines"], 1) - self.assertEqual(result["ratchet"]["delta"], 1) - self.assertEqual( - result["ratchet"]["after"]["handwritten_files"], - [{"path": ".github/actions/check/main.go", "lines": 2}], - ) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_patch_reduction_can_pay_for_handwritten_lines(self) -> None: - self.prepare() - self.write("generated.py", "generated\ncustom\ncustom\n") - base = self.commit() - self.write("generated.py", "generated\n") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["ratchet"]["delta"], -1) - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_stale_branch_cannot_reclaim_landed_reduction(self) -> None: - self.prepare() - self.write("helper.go", "package example\n// old allowance\n") - old = self.commit() - self.git("checkout", "-q", "-b", "stale") - head = self.commit() - self.git("checkout", "-q", "main") - self.write("helper.go", "package example\n") - base = self.commit() - self.assertEqual(self.git("merge-base", base, head), old) - result = self.evaluate(base, head) - self.assertEqual(result["ratchet"]["before"]["revision"], base) - self.assertEqual(result["ratchet"]["delta"], 1) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_complete_merge_queue_candidate_is_counted(self) -> None: - base = self.prepare() - constituent = self.commit("empty constituent") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit("combined queue"), pull_heads=[constituent]) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_ratchet_cannot_be_disabled(self) -> None: - base = self.prepare() - self.policy(active=False) - result = self.evaluate(base, self.commit()) - for name in ("budget", "isolation"): - self.assertEqual(result["checks"][name]["state"], "failure") - self.assertIn("cannot be disabled", result["checks"][name]["description"]) - - def test_old_ceiling_increase_cannot_restore_ratchet_headroom(self) -> None: - self.prepare() - self.policy(limit=3000) - base = self.commit("separately reviewed ceiling increase") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["limit"], 3000) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_fixed_patch_ceiling_remains_independent(self) -> None: - self.prepare() - self.policy(limit=1) - self.write("helper.go", "package example\n// two\n// three\n") - base = self.commit() - (self.repo / "helper.go").unlink() - self.write("generated.py", "generated\ncustom\ncustom\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["ratchet"]["delta"], -1) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_nontext_and_symlink_go_fail_closed(self) -> None: - base = self.prepare() - for content in (b"package example\0", b"\xff", None): - with self.subTest(content=content): - self.git("checkout", "--detach", "-q", base) - path = self.repo / "helper.go" - if content is None: - path.symlink_to("generated.py") - else: - path.write_bytes(content) - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_live_target_snapshot_hash_is_verified(self) -> None: - self.prepare() - self.git("checkout", "-q", "-b", "candidate") - head = self.commit() - self.git("checkout", "-q", "main") - stats = self.repo / ".castiron.stats.yml" - stats.write_text(stats.read_text().replace("codegen_hash: ", "codegen_hash: bad")) - result = self.evaluate(self.commit(), head) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_activation_must_be_policy_only(self) -> None: - self.prepare() - self.policy(active=False) - base = self.commit() - self.policy() - head = self.commit() - result = self.evaluate(base, head) - self.assertTrue(all(check["state"] == "success" for check in result["checks"].values())) - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - -class BudgetTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.generated, _ = self.baseline() - self.policy(10) - self.base = self.commit("human-owned budget") - - def policy(self, limit: int) -> None: - self.write( - budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": limit}) + "\n" - ) - - def evaluate(self, head: str, base: str | None = None, **kwargs: Any) -> dict[str, Any]: - return budget.evaluate(self.repo, base or self.base, head, public=False, **kwargs)[0] - - def test_additions_and_deletions_do_not_cancel(self) -> None: - self.write("generated.py", "replacement\n") - result = self.evaluate(self.commit()) - self.assertEqual((result["additions"], result["deletions"], result["total"]), (1, 1, 2)) - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_below_equal_and_above_limit(self) -> None: - for additions in (9, 10, 11): - with self.subTest(additions=additions): - self.write("generated.py", "generated\n" + "custom\n" * additions) - result = self.evaluate(self.commit()) - self.assertEqual(result["total"], additions) - self.assertEqual( - result["checks"]["budget"]["state"], "failure" if additions > 10 else "success" - ) - - def test_whole_generated_file_deletion_is_counted(self) -> None: - (self.repo / "generated.py").unlink() - result = self.evaluate(self.commit()) - self.assertEqual((result["additions"], result["deletions"]), (0, 1)) - self.assertEqual(result["mixed_files"], 1) - - def test_restoring_generated_content_removes_customization(self) -> None: - self.write("generated.py", "generated\ncustom\n") - customized = self.commit() - self.write("generated.py", "generated\n") - result = self.evaluate(self.commit(), base=customized) - self.assertEqual(result["total"], 0) - self.assertEqual(result["mixed_files"], 0) - - def test_handwritten_only_files_keep_existing_report_scope(self) -> None: - self.write("handwritten.py", "custom\n" * 100) - self.assertEqual(self.evaluate(self.commit())["total"], 0) - - def test_increase_is_isolated_but_does_not_apply_to_itself(self) -> None: - self.policy(20) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["limit"], 10) - self.assertEqual(result["checked_limit"], 10) - - def test_entire_pr_must_be_budget_only_not_just_latest_commit(self) -> None: - self.write("generated.py", "generated\n" + "custom\n" * 11) - self.commit("SDK change first") - self.policy(100) - result = self.evaluate(self.commit("budget-only last commit")) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertIn("separate, budget-only PR", result["checks"]["isolation"]["description"]) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - - def test_new_base_budget_is_used_for_stale_pr_branch(self) -> None: - self.git("checkout", "-q", "-b", "sdk", self.base) - self.write("generated.py", "generated\n" + "custom\n" * 11) - head = self.commit() - self.git("checkout", "-q", "main") - self.policy(20) - new_base = self.commit("separate approved increase") - result = self.evaluate(head, base=new_base) - self.assertEqual(result["limit"], 20) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_next_uses_main_policy_and_target_diff_independently(self) -> None: - self.git("checkout", "-q", "-b", "next", self.base) - self.policy(100) - target = self.commit("next carries a different policy") - self.write("generated.py", "generated\n" + "custom\n" * 11) - head = self.commit("next change") - result = self.evaluate(head, base=target, policy_sha=self.base) - self.assertEqual(result["base_sha"], target) - self.assertEqual(result["policy_sha"], self.base) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - self.git("checkout", "-q", "main") - self.policy(20) - policy = self.commit("approved main policy") - result = self.evaluate(head, base=target, policy_sha=policy) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.policy(1) - policy = self.commit("main lowers policy after PR passed") - result = self.evaluate(head, base=target, policy_sha=policy, pull_heads=[head]) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 1) - - def test_decrease_must_fit_current_usage(self) -> None: - self.write("generated.py", "generated\ncustom\ncustom\n") - base = self.commit() - self.policy(1) - result = self.evaluate(self.commit(), base=base) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["checked_limit"], 1) - - def test_missing_base_policy_fails_closed(self) -> None: - (self.repo / budget.POLICY).unlink() - base = self.commit() - self.policy(100) - result = self.evaluate(self.commit(), base=base) - self.assertTrue(all(c["state"] == "failure" for c in result["checks"].values())) - - def test_policy_deletion_rename_symlink_and_mode_change_fail(self) -> None: - for change in ("delete", "rename", "symlink", "executable"): - with self.subTest(change=change): - self.git("checkout", "--detach", "-q", self.base) - path = self.repo / budget.POLICY - if change == "delete": - path.unlink() - elif change == "rename": - path.rename(self.repo / "renamed.json") - elif change == "symlink": - path.unlink() - path.symlink_to("generated.py") - else: - path.chmod(0o755) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - - def test_invalid_policy_values_fail(self) -> None: - invalid = [ - "[]", - "{}", - "not json", - '{"schema_version":1,"max_custom_patch_lines":true}', - '{"schema_version":true,"max_custom_patch_lines":10}', - '{"schema_version":2,"max_custom_patch_lines":10}', - '{"schema_version":1,"max_custom_patch_lines":-1}', - '{"schema_version":1,"max_custom_patch_lines":10.5}', - '{"schema_version":1,"max_custom_patch_lines":null}', - '{"schema_version":1,"max_custom_patch_lines":10,"mode":"report-only"}', - '{"schema_version":1,"max_custom_patch_lines":10,"max_custom_patch_lines":999}', - " " * 4097, - ] - for contents in invalid: - with self.subTest(contents=contents[:100]): - self.write(budget.POLICY, contents) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - - def test_bad_snapshot_and_binary_change_fail_budget(self) -> None: - self.write("generated.py", "binary\0content") - result = self.evaluate(self.commit()) - self.assertIn("non-text", result["checks"]["budget"]["description"]) - self.write("generated.py", "generated\n") - path = self.repo / ".castiron.stats.yml" - path.write_text( - path.read_text().replace( - budget.report.hash_codegen_commit(self.repo, self.generated), "f" * 64 - ) - ) - result = self.evaluate(self.commit()) - self.assertIn("codegen_hash mismatch", result["checks"]["budget"]["description"]) - - def test_budget_uses_existing_reporter_with_strict_verification(self) -> None: - with mock.patch.object( - budget.report, "build_report", wraps=budget.report.build_report - ) as measured: - self.evaluate(self.base) - self.assertEqual( - measured.call_args.kwargs, - {"public": False, "fetch": False, "require_head_hash": True, "target_branch": "main"}, - ) - - def test_queue_isolates_prs_but_uses_main_budget_for_combined_tree(self) -> None: - self.git("checkout", "-q", "-b", "policy", self.base) - self.policy(100) - policy_head = self.commit() - self.git("checkout", "-q", "-b", "sdk", self.base) - self.write("generated.py", "generated\n" + "custom\n" * 11) - sdk_head = self.commit() - self.git("checkout", "-q", "-b", "queue", self.base) - self.git("merge", "--no-ff", "-m", "queue policy", policy_head) - self.git("merge", "--no-ff", "-m", "queue SDK", sdk_head) - result = self.evaluate(self.git("rev-parse", "HEAD"), pull_heads=[policy_head, sdk_head]) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - - def test_queue_without_verified_members_fails(self) -> None: - self.assertEqual( - self.evaluate(self.base, pull_heads=[])["checks"]["isolation"]["state"], "failure" - ) - - def test_cli_executes_trusted_reporter_not_inspected_repo(self) -> None: - self.write( - "scripts/castiron/custom_code_report.py", 'raise RuntimeError("PR CODE EXECUTED")\n' - ) - self.write("sitecustomize.py", 'raise RuntimeError("PR IMPORTED")\n') - head = self.commit() - out = self.repo / "result" - command = [ - sys.executable, - "-I", - str(Path(budget.__file__).resolve()), - "check", - "--repo", - str(self.repo), - "--base", - self.base, - "--head", - head, - "--out", - str(out), - ] - completed = subprocess.run( - command, cwd=self.repo, capture_output=True, text=True, check=False - ) - self.assertEqual(completed.returncode, 0, completed.stderr + completed.stdout) - self.assertEqual(json.loads((out / "budget.json").read_text())["total"], 0) - - def test_summary_has_counts_revisions_and_no_longer_generated(self) -> None: - self.write("generated.py", "replacement\n") - result = self.evaluate(self.commit()) - out = self.repo / "report-output" - budget.write_result(out, result) - summary = (out / "summary.md").read_text() - self.assertIn("+1 / -1 = 2", summary) - self.assertIn(self.generated, summary) - self.assertIn(self.base, summary) - self.assertIn("human approving review", summary) - - -class InventoryTests(unittest.TestCase): - def test_literal_inventory_is_data_not_executable_code(self) -> None: - source = ( - b"raise AssertionError('must not execute')\n" - b"GENERATED_GITHUB_PATHS = frozenset({b'.github/workflows/generated.yml'})\n" - ) - self.assertEqual( - budget.checkpoint_github_inventory(source), - frozenset({b".github/workflows/generated.yml"}), - ) - self.assertEqual( - budget.checkpoint_github_inventory(b"GENERATED_GITHUB_PATHS = frozenset({})"), - frozenset(), - ) - - def test_inventory_rejects_missing_ambiguous_and_nonliteral_values(self) -> None: - for source in ( - b"pass", - b"GENERATED_GITHUB_PATHS = frozenset({})\nGENERATED_GITHUB_PATHS = frozenset({})", - b"GENERATED_GITHUB_PATHS = __import__('os').environ", - b"GENERATED_GITHUB_PATHS = frozenset(__import__('os').environ)", - b"GENERATED_GITHUB_PATHS = frozenset({'text'})", - b"GENERATED_GITHUB_PATHS = frozenset({b'not-github'})", - ): - with self.subTest(source=source), self.assertRaises(ValueError): - budget.checkpoint_github_inventory(source) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_github.py b/scripts/castiron/test_custom_code_github.py deleted file mode 100644 index 8cf302c659..0000000000 --- a/scripts/castiron/test_custom_code_github.py +++ /dev/null @@ -1,1048 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Trusted GitHub evaluation with real local Git objects and an offline API.""" - -from __future__ import annotations - -import json -import os -import sys -import tempfile -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_budget as budget -import custom_code_report as report -from custom_code_test_support import GENERATION, GitTestCase, source_head, source_run - -REPOSITORY = "openai/example" - - -class GitHubAPI: - """Mutable remote state; unexpected requests fail instead of reaching GitHub.""" - - def __init__(self, base: str, head: str, event: str = "pull_request") -> None: - self.main = base - self.next = base - self.run = source_run(head, event) - self.pull: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"sha": base, "ref": "main", "repo": {"full_name": REPOSITORY}}, - } - self.associations = [{"number": 3}] - self.branches = [{"number": 3}] - self.rules = [{"type": "merge_queue"}] - self.bodies: list[str] = [] - - def __call__(self, method: str, path: str, payload: dict[str, Any] | None = None) -> Any: - root = f"repos/{REPOSITORY}" - if method == "POST" and path == f"{root}/issues/3/comments" and payload: - self.bodies.append(payload["body"]) - return {"html_url": "published"} - responses = { - root: {"default_branch": "main", "private": False}, - f"{root}/actions/runs/123": self.run, - f"{root}/pulls/3": self.pull, - f"{root}/pulls/4": self.pull, - f"{root}/git/ref/heads/main": {"object": {"sha": self.main}}, - f"{root}/rules/branches/main?per_page=100&page=1": self.rules[:100], - f"{root}/rules/branches/main?per_page=100&page=2": self.rules[100:200], - f"{root}/git/ref/heads/next": {"object": {"sha": self.next}}, - f"{root}/rules/branches/next?per_page=100&page=1": self.rules[:100], - f"{root}/rules/branches/next?per_page=100&page=2": self.rules[100:200], - f"{root}/commits/{self.run['head_sha']}/pulls?per_page=100": self.associations, - f"{root}/pulls?state=open&head=contributor%3Asdk&per_page=100&page=1": self.branches, - f"{root}/issues/3/comments?per_page=100&page=1": [], - } - if method != "GET" or path not in responses: - raise AssertionError(f"Unexpected GitHub request: {method} {path}") - return responses[path] - - -class GitHubReportTests(GitTestCase): - def test_same_head_targets_preserve_required_main_discovery(self) -> None: - head = "b" * 40 - run = source_run(head, "pull_request") - run["pull_requests"] = [] - main: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - next_pull: dict[str, Any] = {**main, "base": {**main["base"], "ref": "next"}} - for numbers in ([3, 4], [4, 3]): - - def api_response(_method: str, path: str, numbers: list[int] = numbers) -> Any: - if "/commits/" in path: - return [{"number": number} for number in numbers] - return {"3": main, "4": next_pull}[path.rsplit("/", 1)[1]] - - with ( - self.subTest(numbers=numbers), - mock.patch.object(report, "api", side_effect=api_response), - ): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - run["pull_requests"] = [{"number": 4}] - self.assertEqual(report.current_pull(REPOSITORY, run), (4, "next")) - run["pull_requests"] = [] - next_pull["base"]["ref"] = "main" - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.current_pull(REPOSITORY, run) - next_pull["base"]["ref"] = "next" - - def test_trusted_report_recomputes_pr_output_in_a_bare_repository(self) -> None: - generated, _ = self.baseline() - content_hash = report.hash_codegen_commit(self.repo, generated) - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - GENERATION, - content_hash, - "codegen/public-test", - None, - ) - self.git("branch", "codegen/public-test", snapshot) - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write(".castiron.stats.yml", stats + f"public_codegen_sha: {snapshot}\n") - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - base = self.commit() - legitimate, _ = report.build_report(self.repo, base, base, require_head_hash=True) - self.write("generated.py", "generated\n# custom\n") - self.write( - "scripts/castiron/custom_code_report.py", - "raise RuntimeError('PR code ran')\n", - ) - self.write("report.json", json.dumps(legitimate)) - head = self.commit() - self.write( - ".castiron.stats.yml", - (self.repo / ".castiron.stats.yml").read_text().replace(content_hash, "0" * 64), - ) - broken = self.commit() - remote = self.repo / "public.git" - self.git("clone", "--bare", str(self.repo), str(remote)) - real_git = report.git - - def local_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[:3] == ("remote", "add", "origin"): - self.assertEqual(args[3], f"https://github.com/{REPOSITORY}.git") - args = (*args[:3], str(remote)) - self.assertNotIn("checkout", args) - return real_git(repo, *args, input_bytes=input_bytes) - - for label, revision, total in ( - ("baseline", base, 0), - ("custom", head, 1), - ("fork", head, 1), - ("bad-hash", broken, None), - ("next", head, 1), - ("next-fork", head, 1), - ("next-bad-hash", broken, None), - ): - with self.subTest(label=label): - result: dict[str, Any] | None = None - api = GitHubAPI( - "d" * 40, revision - ) # PR metadata already differs from captured main. - if label.startswith("next"): - api.pull["base"]["ref"] = "next" - api.next = base - if "fork" in label: - api.run["pull_requests"] = [] - api.associations = [] - objects, out = self.repo / f"{label}.git", self.repo / f"{label}-report" - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(budget.report, "api", side_effect=api), - mock.patch.object(report, "git", side_effect=local_git), - ): - report.trusted_report(objects, REPOSITORY, 123, 1, out, base=base) - self.assertEqual( - real_git(objects, "rev-parse", "--is-bare-repository"), b"true\n" - ) - self.assertFalse((objects / "scripts").exists()) - self.assertEqual(api.bodies, []) - measured = json.loads((out / "report.json").read_text()) - self.assertEqual( - (measured["target_base_sha"], measured["head_sha"]), (base, revision) - ) - # Advance main before both budget evaluation and publication. - api.main = api.next = api.pull["base"]["sha"] = "e" * 40 - if label.startswith("next"): - # Advisory reporting alone cannot authorize next without - # the trusted selector's protected-base pin receipt. - with self.assertRaisesRegex(ValueError, "selection"): - budget.github_evaluate( - objects, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": api.run}, - base, - out, - ) - else: - result, _ = budget.github_evaluate( - objects, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": api.run}, - base, - out, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": base, - "head_sha": revision, - "policy_sha": base, - }, - ) - self.assertEqual((result["base_sha"], result["head_sha"]), (base, revision)) - report.publish_comment( - measured, REPOSITORY, 3, 123, 1, artifact_run_id=9, artifact_run_attempt=3 - ) - self.assertEqual(len(api.bodies), 1) - body = api.bodies[0] - self.assertIn("castiron:run:v1:123:1", body) - self.assertIn(base, body) - self.assertIn("/actions/runs/9", body) - if label.startswith("next"): - self.assertIn( - "Report unavailable" if total is None else "Generated baselines verified", - body, - ) - continue - assert result is not None - if total is None: - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn("could not verify", result["checks"]["budget"]["description"]) - self.assertIn("codegen_hash mismatch", measured["error"]) - self.assertIn("Report unavailable", body) - self.assertNotIn("Generated baselines verified", body) - else: - self.assertEqual(result["total"], total) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.assertIn("Generated baselines verified", body) - self.assertIn("--name castiron-custom-code-9-3", body) - if total: - self.assertIn("1 newly customized", body) - self.assertIn("generated.py", body) - self.assertIn(b"+# custom", (out / "custom-code.patch").read_bytes()) - self.assertNotIn("No new custom-code files detected", body) - else: - self.assertIn("No new custom-code files detected", body) - - def test_fork_association_fallback_paginates_candidates(self) -> None: - run: dict[str, Any] = { - "head_sha": "a" * 40, - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "fix/branch&other=value", - } - first = [{"number": number} for number in range(1, 101)] - with mock.patch.object(report, "api", side_effect=[[], first, [{"number": 101}]]) as api: - self.assertEqual(len(report.associated_pulls("openai/example", run)), 101) - self.assertEqual( - api.call_args.args, - ( - "GET", - "repos/openai/example/pulls?state=open&head=contributor%3Afix%2Fbranch%26other%3Dvalue&per_page=100&page=2", - ), - ) - - def test_trusted_report_rejects_invalid_or_stale_association_before_fetch(self) -> None: - cases: dict[str, tuple[dict[str, Any], dict[str, Any], bool]] = { - "wrong workflow": ({"path": "other.yml"}, {}, True), - "unfinished run": ({"status": "in_progress"}, {}, True), - "old attempt": ({"run_attempt": 2}, {}, False), - "closed PR": ({}, {"state": "closed"}, False), - "changed head": ({}, {"head": {"sha": "c" * 40}}, False), - "other branch": ( - {}, - {"base": {"ref": "other", "repo": {"full_name": REPOSITORY}}}, - False, - ), - "other repository": ( - {}, - {"base": {"ref": "main", "repo": {"full_name": "other/repo"}}}, - False, - ), - "multiple PRs": ({"pull_requests": [{"number": 3}, {"number": 4}]}, {}, True), - } - for fallback in (False, True): - for label, (run, pull, raises) in cases.items(): - with self.subTest(label=label, fallback=fallback): - api = GitHubAPI("b" * 40, "a" * 40) - api.run.update(run) - api.pull.update(pull) - if fallback: - api.branches = api.run["pull_requests"] - api.run["pull_requests"] = [] - api.associations = [] - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(report, "git") as git, - ): - if raises: - with self.assertRaises(report.ReportError): - report.trusted_report( - self.repo / "objects", - REPOSITORY, - 123, - 1, - self.repo / "out", - base="b" * 40, - ) - else: - report.trusted_report( - self.repo / "objects", - REPOSITORY, - 123, - 1, - self.repo / "out", - base="b" * 40, - ) - git.assert_not_called() - self.assertFalse((self.repo / "out").exists()) - api = GitHubAPI("b" * 40, "a" * 40) - api.run["pull_requests"] = api.associations = api.branches = [] - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(report, "git") as git, - ): - report.trusted_report( - self.repo / "objects", REPOSITORY, 123, 1, self.repo / "out", base="b" * 40 - ) - git.assert_not_called() - - def test_queue_membership_uses_synthetic_commit_not_original_pr_ancestry(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - base = self.commit("budget") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": source_run(queue_head, "merge_group"), - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - source_run(queue_head, "merge_group"), - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=[(pr_head, queue_head)]), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - base, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": base, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_next_queue_bootstrap_isolated_per_verified_constituent(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - policy = {"schema_version": 1, "max_custom_patch_lines": 10} - self.write(budget.POLICY, json.dumps(policy)) - base = self.commit("unpinned next budget") - self.git("checkout", "-q", "-b", "initialize", base) - self.write( - budget.POLICY, - json.dumps({**policy, "next_policy": {"repository": REPOSITORY, "main_sha": base}}), - ) - pin_head = self.commit("policy-only initialization") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - sdk_head = self.commit("unrelated SDK PR") - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "initialize") - pin_queue = self.commit("synthetic policy queue commit") - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic combined queue commit") - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = {"repository": {"full_name": REPOSITORY}, "workflow_run": run} - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - for entries, state in ( - ([(pin_head, pin_queue), (sdk_head, queue_head)], "success"), - ([(queue_head, queue_head)], "failure"), - ): - with ( - self.subTest(entries=entries), - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": base}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=entries), - mock.patch.object(budget.report, "git", side_effect=local_fetch), - ): - result, _ = budget.github_evaluate( - Path(temp) / "queue.git", - REPOSITORY, - event, - base, - selection={ - "schema_version": 1, - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": base, - "bootstrap": True, - }, - ) - self.assertEqual(result["checks"]["isolation"]["state"], state) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.assertEqual(result["limit"], 10) - self.assertEqual(result["total"], 1) - - def test_next_queue_uses_reviewed_main_policy_after_main_advances(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - approved_policy = self.commit("budget") - self.write( - budget.POLICY, - json.dumps( - { - "schema_version": 1, - "max_custom_patch_lines": 10, - "next_policy": {"repository": "openai/example", "main_sha": approved_policy}, - } - ), - ) - base = self.commit("reviewed next policy") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - self.git("checkout", "-q", "main") - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 0})) - policy = self.commit("main tightens policy independently of next") - self.assertNotEqual(self.git("merge-base", policy, queue_head), policy) - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": run, - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": policy}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=[(pr_head, queue_head)]), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - approved_policy, - selection={ - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": approved_policy, - "bootstrap": False, - "schema_version": 1, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["limit"], 10) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_sha"], approved_policy) - self.assertEqual(result["target_branch"], "next") - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_next_queue_bootstrap_with_separate_sdk_pr(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - approved_policy = self.commit("budget") - self.write( - budget.POLICY, - json.dumps( - { - "schema_version": 1, - "max_custom_patch_lines": 10, - "next_policy": {"repository": "openai/example", "main_sha": approved_policy}, - } - ), - ) - pin_head = self.commit("isolated initialization PR") - base = approved_policy - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - self.commit("SDK queue entry") - self.git("merge", "--squash", pin_head) - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - policy = approved_policy - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": run, - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": policy}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object( - budget, - "queued_entries", - return_value=[(pin_head, queue_head), (pr_head, queue_head)], - ), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - approved_policy, - selection={ - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": approved_policy, - "bootstrap": True, - "schema_version": 1, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["limit"], 10) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_sha"], approved_policy) - self.assertEqual(result["target_branch"], "next") - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_same_commit_discovery_is_bound_to_source_repository_and_branch( - self, - ) -> None: - run = source_run("b" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head(run["head_sha"]), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - for foreign_head in ( - {**own["head"], "ref": "other"}, - {**own["head"], "repo": {"id": 11, "full_name": "victim/example"}}, - {**own["head"], "repo": None}, - ): - victim = {**own, "head": foreign_head} - - def discover(_method: str, path: str, victim: dict[str, Any] = victim) -> Any: - if "/commits/" in path: - return [{"number": 3}, {"number": 4}] - return {"3": own, "4": victim}[path.rsplit("/", 1)[1]] - - with ( - self.subTest(head=foreign_head), - mock.patch.object( - report, - "api", - side_effect=discover, - ), - ): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - - def test_foreign_only_associations_retry_source_branch(self) -> None: - run = source_run("b" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head("b" * 40), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - other = {**own, "head": {**own["head"], "ref": "other"}} - branches = [{"number": 3}] - - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [{"number": 4}] - if "/pulls?" in path: - return branches - return {"3": own, "4": other, "5": own}[path.rsplit("/", 1)[1]] - - with mock.patch.object(report, "api", side_effect=api): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - run["pull_requests"] = [{"number": 4}] - self.assertIsNone(report.current_pull(REPOSITORY, run)) - run["pull_requests"] = [] - branches.append({"number": 5}) - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.current_pull(REPOSITORY, run) - - -class GitHubBudgetTests(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.repo = self.root / "objects.git" - self.base, self.head = "a" * 40, "b" * 40 - self.api = GitHubAPI(self.base, self.head) - patch = mock.patch.object(budget.report, "api", side_effect=self.api) - patch.start() - self.addCleanup(patch.stop) - - def evaluate(self, *, trusted_report: Path | None = None) -> tuple[dict[str, Any], bytes]: - event = { - "repository": {"full_name": REPOSITORY}, - "workflow_run": source_run(self.head, self.api.run["event"]), - } - return budget.github_evaluate( - self.repo, - REPOSITORY, - event, - self.base, - trusted_report, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": self.base, - "head_sha": self.head, - "policy_sha": self.base, - }, - ) - - def test_main_rejects_missing_or_retargeted_selection(self) -> None: - for branch in (None, "next"): - selection = ( - None - if branch is None - else { - "schema_version": 1, - "target_branch": branch, - "base_sha": self.base, - "head_sha": self.head, - "policy_sha": self.base, - } - ) - with ( - self.subTest(branch=branch), - self.assertRaisesRegex(ValueError, "policy selection"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - selection=selection, - ) - self.assertFalse(self.repo.exists()) - - def test_merge_queue_rule_is_required_before_passing(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.rules = [{"type": "required_status_checks"}] - with self.assertRaisesRegex(ValueError, "must require a merge queue"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_merge_queue_rule_on_later_page_is_honored(self) -> None: - self.api.rules = [{"type": "required_status_checks"}] * 100 + [{"type": "merge_queue"}] - with ( - mock.patch.object(budget.subprocess, "run"), - mock.patch.object(budget.report, "git"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - self.evaluate() - evaluate.assert_called_once() - - def test_merge_queue_rule_absent_after_full_page_is_rejected(self) -> None: - self.api.rules = [{"type": "required_status_checks"}] * 100 - with self.assertRaisesRegex(ValueError, "must require a merge queue"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_wrong_or_superseded_source_runs_fail_before_fetching(self) -> None: - for changes in ( - {"run_attempt": 2}, - {"status": "in_progress"}, - {"path": "other.yml"}, - {"head_sha": "c" * 40}, - {"repository": {"full_name": "wrong/repo"}}, - ): - with self.subTest(changes=changes): - self.api.run = {**source_run(self.head), **changes} - with self.assertRaisesRegex(ValueError, "source workflow run"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_reuses_only_matching_main_job_report(self) -> None: - self.api.main = self.api.pull["base"]["sha"] = "d" * 40 - self.repo.mkdir() - (self.root / "custom-code.patch").write_bytes(b"verified patch") - for stale in (None, "base", "head"): - with self.subTest(stale=stale): - measured = { - "target_base_sha": "c" * 40 if stale == "base" else self.base, - "head_sha": "c" * 40 if stale == "head" else self.head, - "target_branch": "main", - "policy_sha": self.base, - } - (self.root / "report.json").write_text(json.dumps(measured)) - with ( - mock.patch.object(budget.report, "git", return_value=b"true\n"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - if stale: - with self.assertRaisesRegex( - ValueError, - "trusted policy selection" - if stale == "base" - else "trusted report does not match", - ): - self.evaluate(trusted_report=self.root) - evaluate.assert_not_called() - else: - self.evaluate(trusted_report=self.root) - self.assertEqual(evaluate.call_args.args, (self.repo, self.base, self.head)) - self.assertEqual( - evaluate.call_args.kwargs["measurement"], (measured, b"verified patch") - ) - - def test_next_reuses_target_snapshot_but_uses_selected_main_policy(self) -> None: - self.api.pull["base"]["ref"] = "next" - target = "c" * 40 - self.api.next = target - self.repo.mkdir() - measured = { - "target_base_sha": target, - "target_branch": "next", - "policy_sha": self.base, - "head_sha": self.head, - } - (self.root / "report.json").write_text(json.dumps(measured)) - (self.root / "custom-code.patch").write_bytes(b"verified patch") - with ( - mock.patch.object(budget.report, "git", return_value=b"true\n"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - mock.patch.object( - budget, - "read_policy", - return_value={"next_policy": {"repository": REPOSITORY, "main_sha": self.base}}, - ), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - self.root, - selection={ - "target_branch": "next", - "base_sha": target, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - "schema_version": 1, - }, - ) - self.assertEqual(evaluate.call_args.args, (self.repo, target, self.head)) - self.assertEqual(evaluate.call_args.kwargs["policy_sha"], self.base) - - def test_next_report_reuse_rejects_retargeting_and_policy_mismatch(self) -> None: - self.api.pull["base"]["ref"] = "next" - self.repo.mkdir() - self.api.next = "d" * 40 - for field, value in ( - ("target_branch", "main"), - ("policy_sha", "c" * 40), - ("head_sha", "c" * 40), - ): - measured = { - "target_base_sha": "d" * 40, - "head_sha": self.head, - "target_branch": "next", - "policy_sha": self.base, - field: value, - } - (self.root / "report.json").write_text(json.dumps(measured)) - with ( - self.subTest(field=field), - self.assertRaisesRegex(ValueError, "trusted report does not match"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - self.root, - selection={ - "target_branch": "next", - "base_sha": self.api.next, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - "schema_version": 1, - }, - ) - - def test_next_queue_uses_target_rules_and_requires_policy_selection(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.run["head_branch"] = "gh-readonly-queue/next/pr-3-test" - self.api.rules = [] - with self.assertRaisesRegex(ValueError, "next must require a merge queue"): - self.evaluate() - self.api.rules = [{"type": "merge_queue"}] - self.api.main = "c" * 40 - with self.assertRaisesRegex(ValueError, "policy selection"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_next_selection_receipt_rejects_missing_unknown_or_boolean_version(self) -> None: - self.api.pull["base"]["ref"] = "next" - self.api.next = "d" * 40 - for version in (None, 2, True): - with ( - self.subTest(version=version), - self.assertRaisesRegex(ValueError, "policy selection"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - selection={ - "schema_version": version, - "target_branch": "next", - "base_sha": self.api.next, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - }, - ) - self.assertFalse(self.repo.exists()) - - def test_queue_cannot_reuse_pr_measurement(self) -> None: - self.api.run = source_run(self.head, "merge_group") - with self.assertRaisesRegex(ValueError, "only PR runs can reuse"): - self.evaluate(trusted_report=self.root) - self.assertFalse(self.repo.exists()) - - def test_evaluation_error_emits_failure_outputs_and_diagnostic_summary(self) -> None: - event = self.root / "event.json" - event.write_text( - json.dumps({"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}) - ) - output, result = self.root / "outputs", self.root / "result" - self.api.rules = [] # Real validation failure, before any Git objects are fetched. - args = [ - "budget", - "github", - "--repository", - REPOSITORY, - "--event-path", - str(event), - "--trusted-sha", - self.base, - "--repo", - str(self.repo), - "--out", - str(result), - ] - with ( - mock.patch.object(sys, "argv", args), - mock.patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), - ): - self.assertEqual(budget.main(), 1) - self.assertIn("isolation=failure\nbudget=failure\n", output.read_text()) - self.assertIn("must require a merge queue", (result / "summary.md").read_text()) - - def test_queue_pagination(self) -> None: - pages = [ - { - "nodes": [ - { - "pullRequest": {"number": 1, "headRefOid": "a" * 40}, - "headCommit": {"oid": "c" * 40}, - } - ], - "pageInfo": {"hasNextPage": True, "endCursor": "cursor"}, - }, - { - "nodes": [ - { - "pullRequest": {"number": 2, "headRefOid": "b" * 40}, - "headCommit": {"oid": "d" * 40}, - } - ], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - with mock.patch.object( - budget.report, - "api", - side_effect=[{"data": {"repository": {"mergeQueue": {"entries": p}}}} for p in pages], - ) as api: - self.assertEqual( - budget.queued_entries("openai/example", "main"), - [("a" * 40, "c" * 40), ("b" * 40, "d" * 40)], - ) - self.assertEqual(api.call_args_list[1].args[2]["variables"]["cursor"], "cursor") - - def test_queue_context_refuses_stale_checkout_before_fetching(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.main = "c" * 40 - with self.assertRaisesRegex(ValueError, "trusted checkout is stale"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_github_uses_fresh_bare_data_repo_and_checks_current_head(self) -> None: - self.api.run["pull_requests"] = self.api.associations = [] - self.api.pull["base"]["sha"] = "c" * 40 - with ( - mock.patch.object(budget.report, "git") as git, - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - self.evaluate() - self.assertTrue((self.repo / "HEAD").exists()) - self.assertFalse((self.repo / "src").exists()) - self.assertIn( - mock.call( - self.repo, "fetch", "--quiet", "--no-tags", "origin", self.base, self.head - ), - git.call_args_list, - ) - evaluate.assert_called_once_with( - self.repo, - self.base, - self.head, - public=True, - fetch=True, - pull_heads=None, - measurement=None, - policy_sha=self.base, - target_branch="main", - main_snapshot=None, - repository=REPOSITORY, - ) - - def test_stale_pull_and_wrong_target_fail_before_objects_created(self) -> None: - original = self.api.pull - self.api.run["pull_requests"] = self.api.associations = [] - for changes in ( - {"head": {"sha": "c" * 40}}, - {"base": {**original["base"], "repo": {"full_name": "wrong/repo"}}}, - {"base": {**original["base"], "ref": "other"}}, - {"state": "closed"}, - ): - with self.subTest(changes=changes): - self.api.pull = {**original, **changes} - with self.assertRaisesRegex(ValueError, "exactly one current PR"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_policy.py b/scripts/castiron/test_custom_code_policy.py deleted file mode 100644 index c68bb7072e..0000000000 --- a/scripts/castiron/test_custom_code_policy.py +++ /dev/null @@ -1,988 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Current-main policy selection and review authorization regressions.""" - -from __future__ import annotations - -import io -import json -import sys -import tempfile -import unittest -import unittest.mock as mock # noqa: PLR0402 - keep unittest imports consistent -from collections.abc import Callable -from pathlib import Path -from typing import Any - -import custom_code_budget as budget -import custom_code_policy as selector -from custom_code_test_support import GitTestCase - -REPO = "openai/example" - - -class ReviewedPinTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.git("remote", "add", "origin", f"https://github.com/{REPO}.git") - self.policy = {"schema_version": 1, "max_custom_patch_lines": 4000} - self.write_policy(self.policy) - checker = self.repo / "scripts/castiron/custom_code_budget.py" - checker.parent.mkdir(parents=True) - checker.write_text("NEXT_POLICY_VERSION = 1\n") - checker.with_name("custom_code_policy.py").write_text( - 'PIN_REVIEW_POLICY = "effective-policy-v1"\n' - ) - self.main = self.commit("main policy") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.pin = {"repository": REPO, "main_sha": self.main} - self.write_policy({**self.policy, "next_policy": self.pin}) - self.base = self.commit("reviewed next pin") - self.git("checkout", "-q", "-b", "next", self.base) - (self.repo / "sdk.txt").write_text("candidate SDK bytes") - self.head = self.commit("sdk change") - - def test_review_command_uses_git_policies_and_rejects_stale_main(self) -> None: - git: Callable[..., bytes] = selector.report.git - - def local_fetch(repo: Path, *args: str, **kwargs: Any) -> bytes: - if args[0] == "fetch": - return git( - repo, - "fetch", - "--quiet", - str(self.repo), - f"{self.main}:refs/remotes/origin/main", - self.base, - self.head, - ) - return git(repo, *args, **kwargs) - - with ( - tempfile.TemporaryDirectory() as directory, - mock.patch.object(selector.report, "git", side_effect=local_fetch), - ): - for stale in (False, True): - with ( - self.subTest(stale=stale), - mock.patch("sys.stdout", new_callable=io.StringIO) as output, - ): - args = [ - "review-required", - "--repo", - str(Path(directory) / str(stale)), - "--repository", - REPO, - "--base", - self.base, - "--head", - self.head, - "--main", - self.base if stale else self.main, - ] - if stale: - with self.assertRaisesRegex(ValueError, "authenticated main changed"): - selector.main(args) - self.assertEqual(output.getvalue(), "") - else: - self.assertEqual(selector.main(args), 0) - self.assertEqual(json.loads(output.getvalue()), {"requires_human": False}) - - def write_policy(self, value: dict[str, Any]) -> None: - (self.repo / ".castiron-ratchet.json").write_text(json.dumps(value)) - - def select( - self, base: str | None = None, head: str | None = None, main: str | None = None, **kw: Any - ) -> dict[str, Any]: - return selector.select_policy( - self.repo, - REPO, - "next", - base or self.base, - head or self.head, - main or self.main, - **kw, - ) - - def review_api(self, reviews: list[dict[str, Any]]) -> Callable[[str, str], Any]: - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [{"number": 3}] - if "/reviews?" in path: - return reviews - return { - "number": 3, - "state": "open", - "head": {"sha": self.head}, - "base": {"ref": "next", "repo": {"full_name": REPO}}, - "user": {"login": "author"}, - } - - return api - - def test_policy_review_fork_fallback_keeps_live_exact_head_checks(self) -> None: - original = self.review_api([self.approved()]) - current = original("GET", "repos/openai/example/pulls/3") - cases: list[tuple[dict[str, Any], bool]] = [ - ({}, True), - ({"state": "closed"}, False), - ({"head": {"sha": "f" * 40}}, False), - ({"base": {"ref": "main", "repo": {"full_name": REPO}}}, False), - ({"base": {"ref": "next", "repo": {"full_name": "other/repo"}}}, False), - ] - for change, accepted in cases: - with self.subTest(change=change): - - def api(method: str, path: str, change: dict[str, Any] = change) -> Any: - if "/commits/" in path: - return [] - if "/pulls?" in path: - if path.endswith("page=1"): - return [{"head": {"sha": "a" * 40}}] * 100 - return [current] - if path.endswith("/pulls/3"): - return {**current, **change} - return original(method, path) - - with mock.patch.object(selector.report, "api", side_effect=api): - if accepted: - self.assertEqual(selector.require_human_policy_review(REPO, self.head), 3) - else: - with self.assertRaisesRegex(ValueError, "one open next PR"): - selector.require_human_policy_review(REPO, self.head) - - def approved(self, **kwargs: Any) -> dict[str, Any]: - return { - "id": 10, - "state": "APPROVED", - "commit_id": self.head, - "user": {"type": "User", "login": "reviewer"}, - "author_association": "MEMBER", - "body": selector.HUMAN_REVIEW_RULES["attestation_prefix"] + self.head, - **kwargs, - } - - def test_fork_policy_review_uses_the_discovered_pull(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 5000}) - self.head = self.commit("fork increases policy budget") - review = self.review_api([self.approved()]) - run: dict[str, Any] = { - "head_sha": self.head, - "head_repository": { - "id": 17, - "full_name": "forker/example", - "owner": {"login": "forker"}, - }, - "head_branch": "policy", - "pull_requests": [], - } - - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [] - if "/pulls?" in path: - self.assertIn("head=forker%3Apolicy", path) - return [{"number": 3}] - result = review(method, path) - if path == f"repos/{REPO}/pulls/3": - result["head"].update(ref="policy", repo=run["head_repository"]) - return result - - with mock.patch.object(selector.report, "api", side_effect=api): - current = selector.report.current_pull(REPO, run) - assert current is not None - number, branch = current - self.assertEqual(branch, "next") - result = self.select(base=self.main, pull_numbers={self.head: number}) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - with self.assertRaisesRegex(ValueError, "one open next PR"): - selector.require_human_policy_review(REPO, self.main, number) - - def test_only_base_pin_selects_checker(self) -> None: - (self.repo / "scripts/castiron/custom_code_budget.py").write_text( - 'raise RuntimeError("candidate code must never execute")\n' - ) - self.head = self.commit("malicious candidate code") - self.assertEqual(self.select()["policy_sha"], self.main) - self.assertEqual(self.select()["schema_version"], 1) - - def test_old_reviewed_pin_survives_main_advancing(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "main-new.txt").write_text("later main") - later = self.commit("main later") - self.git("update-ref", "refs/remotes/origin/main", later) - self.assertEqual(self.select(main=later)["policy_sha"], self.main) - - def test_wrong_repository_and_unadopted_pin_rejected(self) -> None: - for pin in ( - {"repository": "other/repo", "main_sha": self.main}, - {"repository": REPO, "main_sha": self.head}, - ): - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": pin}) - base = self.commit("bad pin") - with self.assertRaises((ValueError, selector.report.ReportError)): - self.select(base=base, head=base) - - def test_side_branch_merge_does_not_adopt_policy(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "side").write_text("side") - side = self.commit("side policy") - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "parent").write_text("main") - self.commit("main") - self.git("merge", "--no-ff", "-qm", "merge side", side) - main = self.git("rev-parse", "HEAD") - self.git("update-ref", "refs/remotes/origin/main", main) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": side}}) - base = self.commit("next pins side") - with self.assertRaisesRegex(ValueError, "first-parent"): - self.select(base=base, head=base, main=main) - - def test_absent_pin_ordinary_change_fails(self) -> None: - with self.assertRaisesRegex(ValueError, "initialization"): - self.select(base=self.main, head=self.head) - - def test_absent_pin_preserves_value_error(self) -> None: - with self.assertRaisesRegex(ValueError, "trusted next base has no reviewed policy pin"): - selector.next_policy_sha(self.repo, REPO, self.main, self.main) - - def test_malformed_base_pin_cannot_bootstrap(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy({**self.policy, "next_policy": {}}) - base = self.commit("malformed") - with self.assertRaisesRegex(ValueError, "exactly"): - self.select(base=base, head=self.base) - - def test_bootstrap_requires_exact_policy_only_with_native_review(self) -> None: - self.head = self.base - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=self.main) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["human_review_rules"], selector.HUMAN_REVIEW_RULES) - self.assertEqual(result["policy_sha"], self.main) - review.assert_not_called() - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=self.main, head=self.git("rev-parse", "next")) - - def test_queue_bootstrap_isolated_per_constituent(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "unrelated.txt").write_text("independent SDK change") - sdk_head = self.commit("unrelated PR on unpinned next") - self.git("merge", "--no-ff", "-qm", "queue initialization and SDK", self.base) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(base=self.main, head=group, pull_heads=[self.base, sdk_head]) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - for heads in (None, [group]): - with self.subTest(heads=heads), self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=self.main, head=group, pull_heads=heads) - with self.assertRaisesRegex(ValueError, "verified constituent"): - self.select(base=self.main, head=group, pull_heads=[]) - with self.assertRaisesRegex(ValueError, "verified policy-only constituent"): - self.select(base=self.main, head=group, pull_heads=[sdk_head]) - - def test_queue_pin_update_isolated_per_constituent(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "main-new.txt").write_text("later main") - later = self.commit("main later") - self.git("update-ref", "refs/remotes/origin/main", later) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - pin_head = self.commit("isolated pin update") - self.git("merge", "--no-ff", "-qm", "queue group", self.head) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(head=group, main=later, pull_heads=[pin_head, self.head]) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - # A combined SDK/pin change in one PR is still forbidden. - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(head=group, main=later) - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(head=group, main=later, pull_heads=[group]) - - def advanced_pin_base(self) -> tuple[str, str]: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "later-main.txt").write_text("later main") - later = self.commit("later approved main") - self.git("update-ref", "refs/remotes/origin/main", later) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - return later, self.commit("reviewed pin update on next") - - def test_stale_sdk_branch_uses_current_protected_pin(self) -> None: - later, base = self.advanced_pin_base() - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=base, main=later) - self.assertEqual(result["policy_sha"], later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - - def test_queue_stale_sdk_constituent_inherits_protected_pin(self) -> None: - later, base = self.advanced_pin_base() - self.git("merge", "--no-ff", "-qm", "queue SDK branch", self.head) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=base, head=group, main=later, pull_heads=[self.head]) - self.assertEqual(result["policy_sha"], later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - - def test_stale_actual_pin_edit_still_requires_isolation(self) -> None: - later, base = self.advanced_pin_base() - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - edited = self.commit("PR independently edits pin") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(base=base, head=edited, main=later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - (self.repo / "mixed-sdk.txt").write_text("SDK change") - mixed = self.commit("mix policy and SDK") - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed, main=later) - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed, main=later, pull_heads=[mixed]) - - def test_stale_sdk_budget_after_target_decrease_needs_no_review(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 3000}) - base = self.commit("lower protected next budget") - self.git("merge", "--no-ff", "-qm", "queue stale SDK branch", self.head) - group = self.git("rev-parse", "HEAD") - for head, heads in ((self.head, None), (group, [self.head])): - with ( - self.subTest(head=head), - mock.patch.object(selector, "require_human_policy_review") as review, - ): - result = self.select(base=base, head=head, pull_heads=heads) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(budget.read_budget(self.repo, base), 3000) - review.assert_not_called() - - def test_actual_budget_increase_is_reviewed_even_if_target_increased_further(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 6000}) - base = self.commit("reviewed target budget increase") - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 5000}) - edited = self.commit("PR actually raises budget") - for heads in (None, [edited]): - with ( - mock.patch.object( - selector, - "require_human_policy_review", - side_effect=ValueError("human required"), - ), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=base, head=edited, pull_heads=heads) - (self.repo / "mixed-sdk.txt").write_text("SDK change") - mixed = self.commit("mix SDK and actual budget increase") - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed) - - def test_selector_outputs_transport_review_receipt_as_json(self) -> None: - event = self.repo / "event.json" - event.write_text("{}") - out, output = self.repo / "selection.json", self.repo / "outputs" - selected = { - "schema_version": 1, - "policy_reviews": [{"number": 3, "head_sha": self.head}], - "human_review_rules": selector.HUMAN_REVIEW_RULES, - "bootstrap": False, - } - args = [ - "budget", - "select-policy", - "--repo", - str(self.repo / "objects.git"), - "--repository", - REPO, - "--event-path", - str(event), - "--trusted-sha", - self.main, - "--out", - str(out), - ] - with ( - mock.patch.object(sys, "argv", args), - mock.patch.dict(selector.os.environ, {"GITHUB_OUTPUT": str(output)}), - mock.patch.object(selector, "select_github_policy", return_value=selected), - ): - self.assertEqual(selector.main(), 0) - values = dict(line.split("=", 1) for line in output.read_text().splitlines()) - self.assertEqual(json.loads(values["policy_reviews"]), selected["policy_reviews"]) - self.assertEqual(json.loads(values["human_review_rules"]), selector.HUMAN_REVIEW_RULES) - self.assertFalse(json.loads(values["bootstrap"])) - self.assertEqual(json.loads(out.read_text())["schema_version"], 1) - - def test_follow_up_comments_preserve_policy_approval(self) -> None: - approval = self.approved() - for state in ("COMMENTED", "PENDING"): - follow_up = self.approved(id=11, state=state, body="A non-blocking follow-up") - for reviews in ([approval, follow_up], [follow_up, approval]): - with ( - self.subTest(state=state, reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - ): - self.assertEqual(selector.require_human_policy_review(REPO, self.head), 3) - - def test_comments_do_not_restore_revoked_policy_approval(self) -> None: - comment = self.approved(id=12, state="COMMENTED", body="A follow-up comment") - cases = [ - [self.approved(), self.approved(id=11, state="CHANGES_REQUESTED"), comment], - [self.approved(state="DISMISSED"), comment], - [self.approved(body="Attestation removed"), comment], - [comment], - ] - for reviews in cases: - with ( - self.subTest(reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - selector.require_human_policy_review(REPO, self.head) - - def test_forged_bot_stale_dismissed_untrusted_reviews_rejected(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - { - **self.policy, - "next_policy": self.pin, - "max_custom_patch_lines": self.policy["max_custom_patch_lines"] + 1, - } - ) - self.head = self.commit("bootstrap pin with budget increase") - cases = [ - [], - [self.approved(body="")], - [self.approved(body="Looks good to me")], - [self.approved(body=selector.HUMAN_REVIEW_RULES["attestation_prefix"] + "f" * 40)], - [self.approved(state="COMMENTED", body="APPROVED by a human")], - [self.approved(user={"type": "Bot", "login": "reviewer[bot]"})], - [self.approved(commit_id="a" * 40)], - [self.approved(author_association="NONE")], - [ - self.approved( - user={"type": "User", "login": "dpiet-oai"}, - body="", - ) - ], - [ - self.approved( - user={"type": "User", "login": "markstuart-oai"}, - body="Reviewed `f4c5de3e28ea4c21c2a3d230d06341c3c25a0004`. No actionable findings in the two workflow changes. Adding `next` retains the existing CodeQL and exported-API jobs, uses the actual PR or merge-group comparison base, and leaves merge-group triggers and main-only release/credential guards intact.\n\nSource-only review; I did not execute candidate code or workflows. Current-head CodeQL and exported-API checks passed. Some hosted ecosystem and Node-version tests are still pending. Branch protection settings and the author's branch-creation claims were not independently verified; required CI and review gates still apply.", - ) - ], - [self.approved(body="Automated source review.")], - [self.approved(body="This does not substitute for human approval.")], - [self.approved(body="This does not constitute human approval.")], - [self.approved(body="This is not a human approving review.")], - [self.approved(), self.approved(id=11, state="DISMISSED")], - ] - for reviews in cases: - with ( - self.subTest(reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=self.main) - - def test_bootstrap_budget_increase_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - { - **self.policy, - "next_policy": self.pin, - "max_custom_patch_lines": self.policy["max_custom_patch_lines"] + 1, - } - ) - self.head = self.commit("bootstrap pin with budget increase") - for heads in (None, [self.head]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=self.main, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, pull_heads=heads) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_authorless_review_does_not_block_valid_approval(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - policy = selector.read_policy(self.repo, self.base) - self.write_policy( - {**policy, "max_custom_patch_lines": policy["max_custom_patch_lines"] + 1} - ) - self.head = self.commit("increase budget") - reviews = [self.approved(id=9, user=None), self.approved()] - with mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)): - self.assertEqual( - self.select()["policy_reviews"], - [{"number": 3, "head_sha": self.head}], - ) - - def test_budget_increase_without_pin_change_requires_human(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - policy = selector.read_policy(self.repo, self.base) - self.write_policy( - {**policy, "max_custom_patch_lines": policy["max_custom_patch_lines"] + 1} - ) - self.head = self.commit("increase budget without changing pin") - for reviews in ([], [self.approved(body="Automated source review.")]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select() - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - self.assertEqual(self.select()["policy_sha"], self.main) - - def test_changed_authenticated_main_ref_rejected(self) -> None: - self.git("update-ref", "refs/remotes/origin/main", self.head) - with self.assertRaisesRegex(ValueError, "changed"): - self.select() - - def test_go_ratchet_activation_requires_human_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**selector.read_policy(self.repo, self.base), "ratchet": "go"}) - self.head = self.commit("activate Go ratchet without changing pin or ceiling") - for pull_heads in (None, [self.head]): - for reviews in ([], [self.approved(body="Automated source review.")]): - with ( - self.subTest(pull_heads=pull_heads, reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(pull_heads=pull_heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - selected = self.select(pull_heads=pull_heads) - self.assertEqual(selected["policy_sha"], self.main) - self.assertEqual(selected["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_wrong_origin_rejected(self) -> None: - self.git("remote", "set-url", "origin", "https://github.com/attacker/example.git") - with self.assertRaisesRegex(ValueError, "origin"): - self.select() - - def test_same_repository_github_url_spellings_are_accepted(self) -> None: - for origin in ( - f"https://github.com/{REPO}.git", - f"https://github.com/{REPO}", - f"git@github.com:{REPO}.git", - ): - with self.subTest(origin=origin): - self.git("remote", "set-url", "origin", origin) - self.assertEqual(self.select()["policy_sha"], self.main) - - def test_policy_update_uses_old_base_checker_and_requires_human(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy({**self.policy, "max_custom_patch_lines": 5000}) - newer = self.commit("new reviewed main policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}}) - self.head = self.commit("request new pin") - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - self.assertEqual(self.select(main=newer)["policy_sha"], self.main) - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(main=newer) - - def test_pin_effective_policy_review_for_pr_and_queue(self) -> None: - cases = ( - ({**self.policy, "max_custom_patch_lines": 4000}, False), - ({**self.policy, "max_custom_patch_lines": 3999}, False), - ({**self.policy, "max_custom_patch_lines": 5000}, True), - ({**self.policy, "ratchet": "go"}, True), - ({"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"}, True), - ) - for proposed, requires_human in cases: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy(proposed) - newer = self.commit("adopt proposed effective policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - for bootstrap in (False, True): - base = self.main if bootstrap else self.base - self.git("checkout", "-q", "--detach", base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}} - ) - self.head = self.commit("request pin without changing local budget") - for heads in (None, [self.head]): - with self.subTest(proposed=proposed, bootstrap=bootstrap, heads=heads): - needs_proof = requires_human and not bootstrap - with mock.patch.object( - selector, - "require_human_policy_review", - side_effect=ValueError("human review missing"), - ) as review: - if needs_proof: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=base, main=newer, pull_heads=heads) - review.assert_called_once_with(REPO, self.head, None) - else: - result = self.select(base=base, main=newer, pull_heads=heads) - review.assert_not_called() - self.assertEqual(result["policy_reviews"], []) - if needs_proof: - with mock.patch.object( - selector.report, - "api", - side_effect=self.review_api([self.approved()]), - ): - result = self.select(base=base, main=newer, pull_heads=heads) - self.assertEqual( - result["policy_reviews"], [{"number": 3, "head_sha": self.head}] - ) - - def test_legacy_checker_pin_review_receipt_for_pr_and_queue(self) -> None: - declarations = ( - None, - 'PIN_REVIEW_POLICY = "unknown"\n', - 'PIN_REVIEW_POLICY = "effective-policy-v1"\n' * 2, - ) - for declaration in declarations: - self.git("checkout", "-q", "--detach", self.main) - checker = self.repo / "scripts/castiron/custom_code_policy.py" - if declaration is None: - checker.unlink() - else: - checker.write_text(declaration) - legacy = self.commit("legacy review contract") - checker.write_text('PIN_REVIEW_POLICY = "effective-policy-v1"\n') - newer = self.commit("modern review contract") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", legacy) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": legacy}} - ) - base = self.commit("protected legacy checker") - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}} - ) - self.head = self.commit("forward same-budget pin") - for heads in (None, [self.head]): - with self.subTest(declaration=declaration, heads=heads): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=base, main=newer, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=base, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], legacy) - self.assertEqual( - result["policy_reviews"], [{"number": 3, "head_sha": self.head}] - ) - - def test_rollback_pin_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("reviewed checker fix") - newer = self.commit("newer main checker with unchanged policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}}) - advanced_base = self.commit("adopt newer checker") - self.write_policy({**self.policy, "next_policy": self.pin}) - self.head = self.commit("roll back checker without changing budget") - for heads in (None, [self.head]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=advanced_base, main=newer, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=advanced_base, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], newer) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_stale_pin_compares_current_protected_policy_for_pr_and_queue(self) -> None: - for protected_policy in ( - self.policy, - {**self.policy, "max_custom_patch_lines": 3999}, - {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"}, - ): - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("intermediate checker") - intermediate = self.commit("intermediate main checker") - self.write_policy(protected_policy) - (self.repo / "checker-fix.txt").write_text("protected checker") - current = self.commit("current main checker") - (self.repo / "checker-fix.txt").write_text("forward checker") - forward = self.commit("forward main checker") - self.git("update-ref", "refs/remotes/origin/main", forward) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": current}} - ) - protected_base = self.commit("advance protected next pin") - for proposed in (intermediate, current, forward): - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": proposed}} - ) - self.head = self.commit("stale policy-only PR") - for heads in (None, [self.head]): - with self.subTest(policy=protected_policy, proposed=proposed, heads=heads): - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([]) - ): - if proposed == intermediate: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=protected_base, main=forward, pull_heads=heads) - else: - result = self.select( - base=protected_base, main=forward, pull_heads=heads - ) - self.assertEqual(result["policy_reviews"], []) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select( - base=protected_base, main=forward, pull_heads=heads - ) - self.assertEqual(result["policy_sha"], current) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] - if proposed == intermediate - else [], - ) - - def test_initial_older_pin_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("reviewed bootstrap checker fix") - newer = self.commit("newer main checker with unchanged policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - for pin_sha in (self.main, newer): - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": pin_sha}} - ) - self.head = self.commit("initial checker pin") - for heads in (None, [self.head]): - with mock.patch.object(selector.report, "api", side_effect=self.review_api([])): - if pin_sha == self.main: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=self.main, main=newer, pull_heads=heads) - else: - result = self.select(base=self.main, main=newer, pull_heads=heads) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["policy_sha"], newer) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], newer) - self.assertTrue(result["bootstrap"]) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] if pin_sha == self.main else [], - ) - - def test_bootstrap_uses_active_main_policy_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - ) - current = self.commit("main activates strict zero policy") - self.git("update-ref", "refs/remotes/origin/main", current) - for proposed in (self.main, current): - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": proposed}} - ) - self.head = self.commit("initial pin with unchanged legacy local ceiling") - for heads in (None, [self.head]): - with self.subTest(proposed=proposed, heads=heads): - with mock.patch.object(selector.report, "api", side_effect=self.review_api([])): - if proposed == self.main: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=self.main, main=current, pull_heads=heads) - else: - result = self.select(base=self.main, main=current, pull_heads=heads) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["policy_sha"], current) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, main=current, pull_heads=heads) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_sha"], current) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] if proposed == self.main else [], - ) - - def test_incapable_policy_is_not_executed(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "scripts/castiron/custom_code_budget.py").write_text( - "raise RuntimeError('must not execute')\n" - ) - incapable = self.commit("old checker without pin protocol") - self.git("update-ref", "refs/remotes/origin/main", incapable) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": incapable}} - ) - base = self.commit("pin incapable checker") - with self.assertRaisesRegex(ValueError, "does not support"): - self.select(base=base, head=base, main=incapable) - - def test_main_selection_unchanged(self) -> None: - result = selector.select_policy(self.repo, REPO, "main", self.main, self.head, self.main) - self.assertEqual(result["policy_sha"], self.main) - self.assertFalse(result["bootstrap"]) - - def prepare_snapshot(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.base = self.main # unpinned legacy next - self.strict = { - "schema_version": 2, - "max_custom_patch_lines": 0, - "generated_files": "strict", - } - self.write_policy(self.strict) - self.write("sdk.txt", "approved main output\n") - self.write( - ".castiron.stats.yml", - "schema_version: 1\ngeneration_id: 550e8400-e29b-41d4-a716-446655440000\ncodegen_sha: " - + "a" * 40 - + "\npublic_codegen_sha: " - + "a" * 40 - + "\n", - ) - self.main = self.commit("strict main snapshot") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.write_policy( - {**self.strict, "next_policy": {"repository": REPO, "main_sha": self.main}} - ) - tree_commit = self.commit("adopt exact main snapshot") - self.head = self.git( - "commit-tree", - f"{tree_commit}^{{tree}}", - "-p", - self.base, - "-p", - self.main, - "-m", - "canonical snapshot", - ) - self.git("checkout", "--detach", "-q", self.head) - - def test_exact_main_snapshot_pr_and_queue(self) -> None: - self.prepare_snapshot() - for queue in (False, True): - with ( - self.subTest(queue=queue), - mock.patch.object( - selector.report, "api", side_effect=AssertionError("no human proof needed") - ), - ): - receipt = self.select(pull_heads=[self.head] if queue else None) - self.assertEqual(receipt["main_snapshot"], self.main) - self.assertEqual(receipt["policy_sha"], self.main) - self.assertEqual(receipt["policy_reviews"], []) - - def test_snapshot_rejects_additional_bytes_modes_or_metadata(self) -> None: - self.prepare_snapshot() - valid = self.head - for kind in ("content", "mode", "extra", "stats", "symlink"): - for queue in (False, True): - with self.subTest(kind=kind, queue=queue): - self.git("reset", "--hard", valid) - if kind == "content": - self.write("sdk.txt", "unreviewed output\n") - elif kind == "mode": - (self.repo / "sdk.txt").chmod(0o755) - elif kind == "extra": - self.write("extra.txt", "extra") - elif kind == "stats": - self.write(".castiron.stats.yml", "schema_version: 1\n") - else: - (self.repo / "sdk.txt").unlink() - (self.repo / "sdk.txt").symlink_to(".castiron.stats.yml") - self.head = self.commit("invalid adoption") - with self.assertRaises(ValueError): - self.select(pull_heads=[self.head] if queue else None) - - def test_snapshot_rejects_main_movement_and_nonancestor_target(self) -> None: - self.prepare_snapshot() - old_main = self.main - candidate = self.head - self.git("checkout", "-q", "--detach", old_main) - self.write("sdk.txt", "new approved main\n") - self.main = self.commit("main advances") - self.git("update-ref", "refs/remotes/origin/main", self.main) - with self.assertRaises(ValueError): - self.select(head=candidate) - self.main = old_main - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.git("checkout", "-q", "--detach", self.base) - self.write("private-only.txt", "unmerged next history") - self.base = self.commit("divergent next") - with self.assertRaises(ValueError): - self.select(head=candidate) - - def test_snapshot_rejects_private_history_but_accepts_verified_queue_wrapper(self) -> None: - self.prepare_snapshot() - constituent = self.head - wrapper = self.git( - "commit-tree", - f"{constituent}^{{tree}}", - "-p", - self.base, - "-p", - constituent, - "-m", - "queue wrapper", - ) - self.assertEqual( - self.select(head=wrapper, pull_heads=[constituent])["main_snapshot"], self.main - ) - with self.assertRaises(ValueError): - self.select(head=wrapper) # A wrapper is not an independently verified PR head. - self.write("private.txt", "private-only historical bytes") - self.commit("private historical commit") - (self.repo / "private.txt").unlink() - reverted = self.commit("revert private historical bytes") - self.assertEqual( - self.git("rev-parse", f"{reverted}^{{tree}}"), - self.git("rev-parse", f"{constituent}^{{tree}}"), - ) - for heads in (None, [reverted]): - with self.subTest(queue=heads is not None), self.assertRaises(ValueError): - self.select(head=reverted, pull_heads=heads) - - def test_snapshot_rejects_multiple_queue_constituents(self) -> None: - self.prepare_snapshot() - with self.assertRaisesRegex(ValueError, "one exact"): - self.select(pull_heads=[self.head, self.head]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_policy_source.py b/scripts/castiron/test_custom_code_policy_source.py deleted file mode 100644 index 1b60637c45..0000000000 --- a/scripts/castiron/test_custom_code_policy_source.py +++ /dev/null @@ -1,198 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Current-main policy selection and review authorization regressions.""" - -from __future__ import annotations - -import copy -import json -import sys -import tempfile -import unittest -import unittest.mock as mock # noqa: PLR0402 - keep unittest imports consistent -from pathlib import Path -from typing import Any - -import custom_code_budget as budget -import custom_code_policy as selector - -REPO = "openai/example" - - -POLICY_SOURCE_ROOT = "repos/" + REPO -CANONICAL_POLICY_WORKFLOW = ".github/workflows/castiron-custom-code.yml" -CI_POLICY_WORKFLOW = ".github/workflows/ci.yml" - - -class PolicySourceWorkflowTests(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.repo = Path(self.temp.name) / "objects" - self.workflow_run: dict[str, Any] = { - "id": 42, - "workflow_id": 456, - "repository": {"full_name": REPO}, - "head_sha": "a" * 40, - "status": "completed", - "event": "pull_request", - "run_attempt": 3, - "path": CANONICAL_POLICY_WORKFLOW + "@refs/pull/12/merge", - } - self.event: dict[str, Any] = { - "repository": {"full_name": REPO}, - "workflow_run": copy.deepcopy(self.workflow_run), - } - self.branch = "next" - self.select = self.start_patch( - mock.patch.object(selector, "select_policy", return_value={"selected": True}) - ) - self.start_patch(mock.patch.object(selector.report, "api", side_effect=self.api)) - self.start_patch( - mock.patch.object(selector.report, "current_pull", side_effect=self.current_pull) - ) - self.start_patch( - mock.patch.object( - selector.report, - "git", - side_effect=self.git_result, - ) - ) - - def current_pull(self, *args: Any) -> tuple[int, str]: - return 12, self.branch - - def git_result(self, repo: Path, *args: str) -> bytes: - return b"b" * 40 if args[0] == "rev-parse" else b"" - - def start_patch(self, patcher: Any) -> Any: - result = patcher.start() - self.addCleanup(patcher.stop) - return result - - def api(self, method: str, path: str, *args: Any) -> Any: - return { - POLICY_SOURCE_ROOT: {"default_branch": "main"}, - POLICY_SOURCE_ROOT + "/actions/runs/42": self.workflow_run, - POLICY_SOURCE_ROOT + "/actions/workflows/castiron-custom-code.yml": { - "id": 456, - "path": CANONICAL_POLICY_WORKFLOW, - }, - POLICY_SOURCE_ROOT + "/actions/workflows/ci.yml": { - "id": 456, - "path": CI_POLICY_WORKFLOW, - }, - POLICY_SOURCE_ROOT + "/git/ref/heads/main": {"object": {"sha": "c" * 40}}, - POLICY_SOURCE_ROOT + "/git/ref/heads/next": {"object": {"sha": "b" * 40}}, - }[path] - - def call(self, **kwargs: Any) -> dict[str, Any]: - return selector.select_github_policy(self.repo, REPO, self.event, "c" * 40, **kwargs) - - def use_ci(self) -> None: - self.workflow_run["path"] = CI_POLICY_WORKFLOW + "@refs/pull/12/merge" - - def test_lookalike_workflow_cannot_select_policy(self) -> None: - self.workflow_run["path"] = CANONICAL_POLICY_WORKFLOW + "@evil.yml" - self.workflow_run["workflow_id"] = 789 - with self.assertRaisesRegex(ValueError, "canonical workflow"): - self.call() - self.select.assert_not_called() - - def test_default_canonical_unchanged(self) -> None: - self.branch = "main" - self.assertEqual(self.call(), {"selected": True}) - - def test_explicit_ci_uses_same_base_pin_selector(self) -> None: - self.use_ci() - self.assertEqual(self.call(source_workflow=CI_POLICY_WORKFLOW), {"selected": True}) - self.select.assert_called_once_with( - self.repo, - REPO, - "next", - "b" * 40, - "a" * 40, - "c" * 40, - pull_heads=None, - pull_numbers={"a" * 40: 12}, - ) - - def test_candidate_event_cannot_select_ci_adapter(self) -> None: - self.use_ci() - self.event["source_workflow"] = CI_POLICY_WORKFLOW - with self.assertRaisesRegex(ValueError, "source run"): - self.call() - self.select.assert_not_called() - - def test_unknown_workflow_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "source workflow"): - self.call(source_workflow=".github/workflows/evil.yml") - self.select.assert_not_called() - - def test_ci_adapter_cannot_retarget_main(self) -> None: - self.use_ci() - self.branch = "main" - with self.assertRaisesRegex(ValueError, "restricted to next"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.select.assert_not_called() - - def test_wrong_actual_workflow(self) -> None: - with self.assertRaisesRegex(ValueError, "source run"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.select.assert_not_called() - - def test_stale_or_mismatched_authenticated_identity(self) -> None: - self.use_ci() - for field, value in [ - ("head_sha", "d" * 40), - ("run_attempt", 4), - ("status", "in_progress"), - ("event", "push"), - ("repository", {"full_name": "attacker/example"}), - ]: - with self.subTest(field=field): - old = self.workflow_run[field] - self.workflow_run[field] = value - with self.assertRaisesRegex(ValueError, "source run"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.workflow_run[field] = old - self.select.assert_not_called() - - -class SelectorInterfaceTests(unittest.TestCase): - def test_review_rules_cli_exports_the_canonical_predicate(self) -> None: - import subprocess - - assert budget.__file__ is not None - completed = subprocess.run( - [sys.executable, "-I", budget.__file__, "review-rules"], - capture_output=True, - text=True, - check=True, - ) - self.assertEqual(json.loads(completed.stdout), selector.HUMAN_REVIEW_RULES) - - def test_budget_cli_forwards_to_adjacent_selector(self) -> None: - args = ["select-policy", "--repo", "objects", "--repository", REPO] - with ( - mock.patch.object(sys, "argv", ["budget", *args]), - mock.patch.object(budget.policy, "main", return_value=0) as select, - ): - self.assertEqual(budget.main(), 0) - select.assert_called_once_with(args) - - def test_isolated_cli_loads_only_adjacent_modules(self) -> None: - import subprocess - - for module in (budget, selector): - assert module.__file__ is not None - completed = subprocess.run( - [sys.executable, "-I", module.__file__, "select-policy", "--help"], - capture_output=True, - text=True, - check=True, - ) - self.assertIn("--source-workflow", completed.stdout) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_publication.py b/scripts/castiron/test_custom_code_publication.py deleted file mode 100644 index 205fab5077..0000000000 --- a/scripts/castiron/test_custom_code_publication.py +++ /dev/null @@ -1,600 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Run the trusted publishers against offline GitHub state.""" - -from __future__ import annotations - -import hashlib -import json -import shutil -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_report as report -from custom_code_test_support import ( - WORKFLOW, - GitTestCase, - run_publisher, - run_workflow_script, - source_head, - source_run, -) - - -class WorkflowTests(unittest.TestCase): - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_workflow_path_filters_accept_refs_without_accepting_other_files(self) -> None: - compute = WORKFLOW.read_text().split("\n compute:\n", 1)[1] - condition = next( - line.removeprefix(" if: ") - for line in compute.splitlines() - if line.startswith(" if: ") - ) - expected_path = ".github/workflows/castiron-custom-code.yml" - cases = [ - (expected_path, True), - (expected_path + "@main", True), - (expected_path + "@refs/heads/next", True), - (expected_path + "@refs/pull/42/merge", True), - (".github/workflows/other.yml@main", False), - (expected_path + ".bak@main", False), - (expected_path + "/other@main", False), - ("other/" + expected_path + "@main", False), - ("", False), - ] - head = "a" * 40 - pull = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - for path, accepted in cases: - for event in ("pull_request", "merge_group", "push"): - with self.subTest(path=path, event=event): - run = {**source_run(head), "id": 20, "path": path, "event": event} - payload = { - "context": { - "payload": {"workflow_run": run}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "current": pull, - } - # Evaluate the emitted job condition with its Actions builtins. - script = ( - "const fromJSON = JSON.parse;\n" - "const contains = (items, value) => items.includes(value);\n" - "const startsWith = (value, prefix) => value.startsWith(prefix);\n" - "github.event = context.payload;\n" - f"if ({condition}) await github.rest.issues.createComment({{}});" - ) - self.assertEqual( - bool(run_workflow_script(script, payload)), - accepted and event in ("pull_request", "merge_group"), - ) - self.assertEqual( - bool(run_publisher("Publish a trusted failure status", payload)), - accepted and event == "pull_request", - ) - - def test_candidate_tests_cannot_modify_credentialed_reporter_runner(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - tests, reporter = producer.split("\n report:\n", 1) - tests = tests.split("\n candidate-tests:\n", 1)[1] - self.assertIn("runs-on: ubuntu-latest", tests) - self.assertIn("persist-credentials: false", tests) - self.assertIn("python3 -m unittest discover", tests) - self.assertNotIn("GH_TOKEN", tests) - self.assertNotIn("upload-artifact@", tests) - self.assertNotIn("unittest", reporter) - self.assertIn("GH_TOKEN:", reporter) - self.assertNotIn("needs: candidate-tests", reporter) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_trusted_failure_publisher_updates_one_current_comment(self) -> None: - head = "a" * 40 - run = {**source_run(head), "id": 20} - current = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - previous = { - "id": 42, - "user": {"type": "Bot", "login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - } - cases: dict[str, tuple[list[dict[str, Any]], dict[str, Any], str | None]] = { - "create": ([], current, "create"), - "update": ([previous], current, "update"), - "changed head": ([previous], {**current, "head": {"sha": "c" * 40}}, None), - "newer comment": ( - [{**previous, "body": report.MARKER + "\n"}], - current, - None, - ), - } - for fork in (False, True): - for name, (comments, pull, expected) in cases.items(): - with self.subTest(name=name, fork=fork): - event: dict[str, Any] = {**run, "pull_requests": []} if fork else run - writes = run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": event}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "current": pull, - "comments": comments, - "fallback_pulls": [{"number": 3}], - }, - ) - self.assertEqual( - [w["operation"] for w in writes], [expected] if expected else [] - ) - if expected: - self.assertIn("Report unavailable", writes[0]["body"]) - self.assertIn("castiron:run:v1:20:1", writes[0]["body"]) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_failure_comments_require_one_supported_target(self) -> None: - head = "a" * 40 - - def pull(number: int, branch: str = "main", repo: str = "openai/example") -> dict[str, Any]: - return { - "number": number, - "state": "open", - "head": source_head(head), - "base": {"ref": branch, "repo": {"full_name": repo}}, - } - - cases: list[tuple[str, list[dict[str, Any]], list[int]]] = [ - ("retargeted", [pull(3, "release")], []), - ("other repository", [pull(3, repo="other/example")], []), - ("ambiguous main", [pull(3), pull(4)], []), - ("ambiguous next", [pull(3, "next"), pull(4, "next")], []), - ("prefer main", [pull(3, "next"), pull(4)], [4]), - ("one next", [pull(3, "next")], [3]), - ("ignore unsupported", [pull(3, "release"), pull(4)], [4]), - ] - for name, pulls, expected in cases: - for fallback in (False, True): - with self.subTest(name=name, fallback=fallback): - run = { - **source_run(head), - "id": 20, - "pull_requests": [] if fallback else pulls, - } - writes = run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": run}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "pulls_by_number": {str(pr["number"]): pr for pr in pulls}, - "fallback_pulls": pulls, - }, - ) - self.assertEqual([write["issue_number"] for write in writes], expected) - - def test_workflow_reports_all_branches_without_write_credentials(self) -> None: - workflows = Path(__file__).resolve().parents[2] / ".github/workflows" - producer = (workflows / "castiron-custom-code.yml").read_text() - publisher = (workflows / "castiron-custom-code-comment.yml").read_text() - self.assertIn("pull_request:", producer) - self.assertNotIn("CASTIRON_CUSTOM_CODE_BRANCHES", producer + publisher) - self.assertNotIn("pull-requests: write", producer) - self.assertNotIn("head.repo.full_name ==", producer) - self.assertIn("workflow_run:", publisher) - concurrency = publisher.split("\nconcurrency:\n", 1)[1].split("\njobs:\n", 1)[0] - self.assertIn("queue: max", concurrency) - self.assertIn("cancel-in-progress: false", concurrency) - self.assertIn("ref: ${{ github.workflow_sha }}", publisher) - self.assertNotIn("ref: ${{ github.event.pull_request.head.sha }}", publisher) - self.assertIn("persist-credentials: false", publisher) - self.assertIn("--report", publisher) - compute, comment = publisher.split("\n comment:\n", 1) - self.assertNotIn("pull-requests: write", compute) - self.assertIn("pull-requests: read", compute) - self.assertIn(" trusted-report ", compute) - self.assertIn("ref: main", compute) - self.assertIn('--base "$(git -C .castiron-policy-checker rev-parse HEAD)"', compute) - self.assertIn("trusted_sha=$(git -C .castiron-policy-checker rev-parse HEAD)", compute) - self.assertIn("merge_group:", producer) - self.assertNotIn("download-artifact@", compute) - self.assertNotIn("unittest", compute) - self.assertIn("needs: compute", comment) - self.assertIn("artifact-ids: ${{ needs.compute.outputs.artifact-id }}", comment) - self.assertNotIn("run-id: ${{ github.event.workflow_run.id }}", comment) - self.assertNotIn("git fetch", comment) - self.assertIn("--artifact-run-id", comment) - digest = hashlib.sha256( - (workflows.parents[1] / "scripts/castiron/custom_code_report.py").read_bytes() - ).hexdigest() - self.assertIn(f"REPORTER_SHA256: {digest}", producer) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_review_signals_cannot_replace_pending_candidate_validation(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - group = producer.split("concurrency:\n", 1)[1].splitlines()[0].removeprefix(" group: ") - expression = group.replace("${{", "${").replace("}}", "}") - groups: list[str] = [] - for event in ("pull_request", "pull_request_review", "pull_request"): - payload = {"context": {"payload": {"pull_request": {"number": 42}}}} - script = ( - "github.event = context.payload;\n" - f"github.event_name = {json.dumps(event)};\n" - f"await github.rest.issues.createComment({{body: `{expression}`}});" - ) - groups.append(run_workflow_script(script, payload)[0]["body"]) - self.assertNotEqual(groups[0], groups[1]) - self.assertEqual(groups[0], groups[2]) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_failure_publisher_binds_one_source_before_writing(self) -> None: - head = "a" * 40 - run: dict[str, Any] = {**source_run(head), "id": 20, "pull_requests": []} - own: dict[str, Any] = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - previous = { - "id": 42, - "user": {"type": "Bot", "login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - } - - def publish(**overrides: Any) -> list[dict[str, Any]]: - return run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": run}, - "runId": 30, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "run": run, - "current": own, - "comments": [previous], - "comments_by_number": {3: [previous], 4: [{**previous, "id": 43}]}, - "associated_pulls": [{"number": 3}, {"number": 4}], - **overrides, - }, - ) - - for foreign_head in ( - {**own["head"], "ref": "victim"}, - {**own["head"], "repo": {"id": 11, "full_name": "victim/example"}}, - ): - victim = {**own, "number": 4, "head": foreign_head} - with self.subTest(head=foreign_head): - writes = publish(pulls_by_number={3: own, 4: victim}) - self.assertEqual([write["comment_id"] for write in writes], [42]) - writes = publish( - associated_pulls=[{"number": 4}], - fallback_pulls=[{"number": 3}], - pulls_by_number={3: own, 4: victim}, - ) - self.assertEqual([write["comment_id"] for write in writes], [42]) - self.assertEqual( - publish( - run={**run, "pull_requests": [{"number": 4}]}, - fallback_pulls=[{"number": 3}], - pulls_by_number={3: own, 4: victim}, - ), - [], - ) - # Even two equally authorized PRs are ambiguous: no write to either. - self.assertEqual(publish(), []) - for changed_run in ( - {**run, "run_attempt": 2}, - {**run, "status": "in_progress"}, - {**run, "head_repository": None}, - {**run, "repository": {"full_name": "other/repo"}}, - ): - with self.subTest(run=changed_run): - self.assertEqual(publish(run=changed_run), []) - changed = {**own, "head": {**own["head"], "ref": "other"}} - self.assertEqual( - publish(associated_pulls=[{"number": 3}], current_sequence=[own, changed]), [] - ) - - -class CommentTests(GitTestCase): - def test_comment_only_rerun_links_to_the_compute_artifact_attempt(self) -> None: - workflow = ( - Path(__file__).resolve().parents[2] - / ".github/workflows/castiron-custom-code-comment.yml" - ).read_text() - compute, comment = workflow.split("\n comment:\n", 1) - - def field(section: str, prefix: str) -> str: - return next( - line.removeprefix(prefix) - for line in section.splitlines() - if line.startswith(prefix) - ) - - def resolve(value: str, context: dict[str, str]) -> str: - for key, replacement in context.items(): - value = value.replace("${{ " + key + " }}", replacement) - self.assertNotIn("${{", value) - return value - - # A successful compute job's outputs survive a comment-only rerun. - compute_context = {"github.run_id": "9", "github.run_attempt": "3"} - uploaded_name = resolve(field(compute, " name: "), compute_context) - saved_attempt = resolve(field(compute, " artifact-run-attempt: "), compute_context) - comment_context = { - "github.run_id": "9", - "github.run_attempt": "4", - "needs.compute.outputs.artifact-run-attempt": saved_attempt, - } - artifact_attempt = resolve( - field(comment, " ARTIFACT_RUN_ATTEMPT: "), comment_context - ) - self.assertEqual(uploaded_name, "castiron-custom-code-9-3") - self.assertEqual(artifact_attempt, "3") - - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - pull: dict[str, Any] = { - "state": "open", - "head": source_head(base), - "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 1, - "pull_requests": [{"number": 1}], - } - with mock.patch.object( - report, "api", side_effect=[pull, run, pull, [], pull, {"html_url": "published"}] - ) as api: - self.assertEqual( - report.publish_comment( - result, - "openai/example", - 1, - 2, - 1, - artifact_run_id=9, - artifact_run_attempt=int(artifact_attempt), - ), - "published", - ) - body = api.call_args.args[2]["body"] - self.assertIn(f"--name {uploaded_name}", body) - self.assertNotIn("--name castiron-custom-code-9-4", body) - self.assertIn("castiron:run:v1:2:1", body) - - def test_comment_updates_existing_bot_comment_and_skips_stale(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - calls: list[tuple[str, str, object]] = [] - - def fake_api(method: str, path: str, payload: object = None) -> object: - calls.append((method, path, payload)) - if "/pulls/" in path: - return { - "state": "open", - "head": source_head(base), - "base": { - "sha": base, - "ref": "main", - "repo": {"full_name": "openai/example"}, - }, - } - if "/actions/runs/" in path: - return { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 1, - "pull_requests": [{"number": 1}], - } - if "/comments?" in path: - return [ - {"id": 7, "user": {"login": "someone"}, "body": report.MARKER}, - { - "id": 8, - "user": {"login": "github-actions[bot]"}, - "body": report.MARKER, - "html_url": "existing", - }, - ] - return {"html_url": "updated"} - - with mock.patch.object(report, "api", side_effect=fake_api): - self.assertEqual(report.publish_comment(result, "openai/example", 1, 2, 1), "updated") - self.assertEqual(calls[-1][:2], ("PATCH", "repos/openai/example/issues/comments/8")) - calls.clear() - result["head_sha"] = "f" * 40 - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 1), - "Skipped stale report", - ) - self.assertEqual(len(calls), 1) - - def test_comment_rejects_older_runs_attempts_and_wrong_pr(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - pull: dict[str, Any] = { - "state": "open", - "head": source_head(base), - "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 2, - "pull_requests": [{"number": 1}], - "head_repository": source_run(base)["head_repository"], - "head_branch": "sdk", - } - comment = { - "id": 8, - "user": {"login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - "html_url": "existing", - } - with mock.patch.object(report, "api", side_effect=[pull, run]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 1), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 2) - with mock.patch.object(report, "api", side_effect=[pull, run, pull, [comment]]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 4) - with ( - mock.patch.object( - report, "api", side_effect=[pull, {**run, "pull_requests": []}, [], [], []] - ), - self.assertRaisesRegex(report.ReportError, "does not match report PR"), - ): - report.publish_comment(result, "openai/example", 1, 2, 2) - with ( - mock.patch.object(report, "api", side_effect=[pull, {**run, "path": "other.yml"}]), - self.assertRaisesRegex(report.ReportError, "does not match report PR"), - ): - report.publish_comment(result, "openai/example", 1, 2, 2) - with mock.patch.object( - report, - "api", - side_effect=[pull, {**run, "pull_requests": []}, [{"number": 1}], pull, [comment]], - ) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertIn(f"/commits/{base}/pulls", api.call_args_list[2].args[1]) - changed = {**pull, "head": source_head("f" * 40)} - with mock.patch.object(report, "api", side_effect=[pull, run, pull, [], changed]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 5) - for target in ( - {**pull["base"], "ref": "other"}, - {**pull["base"], "repo": {"full_name": "other/repo"}}, - ): - with mock.patch.object( - report, "api", side_effect=[pull, run, pull, [], {**pull, "base": target}] - ): - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - - def test_comment_uses_canonical_branch_fallback(self) -> None: - run = source_run("a" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head("a" * 40), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - foreign = {**own, "head": {**own["head"], "ref": "other"}} - branches = [{"number": 3}] - writes: list[str] = [] - measured = { - "schema_version": 1, - "status": "error", - "head_sha": "a" * 40, - "target_base_sha": "b" * 40, - "error": "synthetic failure", - } - - def api(method: str, path: str, payload: Any = None) -> Any: - if method == "POST": - writes.append(path) - return {"html_url": "published"} - if "/actions/runs/" in path: - return run - if "/commits/" in path: - return [{"number": 4}] - if "/pulls?" in path: - return branches - if "/comments?" in path: - return [] - return {"3": own, "4": foreign, "5": own}[path.rsplit("/", 1)[1]] - - with mock.patch.object(report, "api", side_effect=api): - self.assertEqual(report.current_pull("openai/example", run), (3, "main")) - self.assertEqual( - report.publish_comment(measured, "openai/example", 3, 123, 1), "published" - ) - self.assertEqual(writes, ["repos/openai/example/issues/3/comments"]) - writes.clear() - run["pull_requests"] = [{"number": 4}] - with self.assertRaisesRegex(report.ReportError, "does not match report PR"): - report.publish_comment(measured, "openai/example", 3, 123, 1) - run["pull_requests"] = [] - branches.append({"number": 5}) - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.publish_comment(measured, "openai/example", 3, 123, 1) - self.assertEqual(writes, []) - - def test_successful_comment_rejects_another_source_before_and_after_pagination( - self, - ) -> None: - head = "a" * 40 - run = source_run(head) - own: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - measured = { - "schema_version": 1, - "status": "error", - "head_sha": head, - "target_base_sha": "b" * 40, - "error": "synthetic failure", - } - for foreign_head in ( - {**own["head"], "ref": "other"}, - {**own["head"], "repo": {**own["head"]["repo"], "id": 11}}, - {**own["head"], "repo": {**own["head"]["repo"], "full_name": "other/repo"}}, - {**own["head"], "repo": None}, - ): - foreign = {**own, "head": foreign_head} - cases: tuple[list[Any], ...] = ([foreign, run], [own, run, own, [], foreign]) - for states in cases: - with ( - self.subTest(head=foreign_head, reads=len(states)), - mock.patch.object(report, "api", side_effect=states) as api, - ): - self.assertEqual( - report.publish_comment(measured, "openai/example", 3, 123, 1), - "Skipped stale report", - ) - self.assertTrue(all(call.args[0] == "GET" for call in api.call_args_list)) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_report.py b/scripts/castiron/test_custom_code_report.py deleted file mode 100644 index a2c9f41909..0000000000 --- a/scripts/castiron/test_custom_code_report.py +++ /dev/null @@ -1,1591 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -from __future__ import annotations - -import base64 -import hashlib -import io -import json -import os -import struct -import subprocess -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_report as report -from custom_code_test_support import GENERATION, GitTestCase - - -class CustomCodeTests(GitTestCase): - def test_large_blob_batches_preserve_every_object_byte(self) -> None: - commands: list[bytes] = [] - expected: dict[str, bytes] = {} - entries: dict[bytes, report.Entry] = {} - for index in range(2048): - content = f"synthetic object {index}\n".encode() + b"x" * 1024 - oid = hashlib.sha1( - f"blob {len(content)}\0".encode() + content, usedforsecurity=False - ).hexdigest() - expected[oid] = content - entries[f"file-{index}".encode()] = report.Entry(b"100644", oid) - commands.extend([b"blob\n", f"data {len(content)}\n".encode(), content, b"\n"]) - report.git(self.repo, "fast-import", "--quiet", input_bytes=b"".join(commands)) - actual = report.read_blobs(self.repo, entries) - self.assertEqual(actual.keys(), expected.keys()) - for oid, content in expected.items(): - self.assertEqual(actual[oid], content) - - def test_git_preserves_authentication_without_inherited_repository_routing(self) -> None: - environment = { - "GIT_CONFIG_GLOBAL": "/ordinary/gitconfig", - "GIT_ASKPASS": "/ordinary/askpass", - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": "credential.helper", - "GIT_CONFIG_VALUE_0": "example", - "GIT_DIR": "/wrong/repository", - "GIT_COMMON_DIR": "/wrong/common", - "GIT_INDEX_FILE": "/wrong/index", - } - with ( - mock.patch.dict(os.environ, environment), - mock.patch.object( - report.subprocess, - "run", - return_value=subprocess.CompletedProcess([], 0, stdout=b"ok"), - ) as run, - ): - self.assertEqual(report.git(self.repo, "status"), b"ok") - actual = run.call_args.kwargs["env"] - for name, value in environment.items(): - if name in {"GIT_DIR", "GIT_COMMON_DIR", "GIT_INDEX_FILE"}: - self.assertNotIn(name, actual) - else: - self.assertEqual(actual[name], value) - self.assertEqual(actual["GIT_NO_REPLACE_OBJECTS"], "1") - - @unittest.skipUnless(os.environ.get("CASTIRON_TEST_BIN"), "Castiron compiler contract test") - def test_rust_and_python_hashes_match(self) -> None: - self.write("a", "first\n") - self.write("nested/b", "second\n") - self.write(".github/workflows/ignored.yml", "ignored\n") - for change in ("initial", "mode", "bytes"): - if change == "mode": - (self.repo / "a").chmod(0o755) - elif change == "bytes": - self.write("nested/b", "changed\n") - commit = self.commit(change) - rust = subprocess.check_output( - [ - os.environ["CASTIRON_TEST_BIN"], - "sdk", - "codegen-hash", - "--repo", - str(self.repo), - "--commit", - commit, - ], - text=True, - ).strip() - self.assertEqual(rust, report.hash_codegen_commit(self.repo, commit)) - - def test_reporting_script_can_be_a_mixed_file(self) -> None: - path = "scripts/castiron/custom_code_report.py" - self.write(path, "# generated reporter\n") - _, base = self.baseline() - self.write(path, "# generated reporter\n# local customization\n") - head = self.commit() - result, _ = report.build_report(self.repo, base, head, require_head_hash=True) - changed = next(file for file in result["files"] if file["path"] == path) - self.assertEqual(changed["category"], "newly_customized") - - def test_hash_vector_and_exclusions(self) -> None: - self.write("a", "hello\n") - first = self.commit() - encoded = ( - report.DOMAIN - + struct.pack(">Q", 1) - + struct.pack(">Q", 1) - + b"a100644" - + struct.pack(">Q", 6) - + b"hello\n" - ) - expected = hashlib.sha256(encoded).hexdigest() - self.assertEqual(report.hash_codegen_commit(self.repo, first), expected) - self.write(".github/workflows/test.yml", "ignored\n") - self.write(".github/actions/test/action.yml", "ignored\n") - second = self.commit() - self.assertEqual(report.hash_codegen_commit(self.repo, second), expected) - self.assertEqual(set(report.tree_entries(self.repo, second)), {b"a"}) - self.assertEqual( - set(report.tree_entries(self.repo, second, include_workflows=True)), - {b"a", b".github/workflows/test.yml", b".github/actions/test/action.yml"}, - ) - os.chmod(self.repo / "a", 0o755) - self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) - self.write(".github/CODEOWNERS", "not ignored\n") - self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) - - def test_classifications(self) -> None: - a, b, c = (report.Entry(b"100644", value * 40) for value in "abc") - cases = [ - ((a, a, b, b), None), - ((a, a, a, b), "newly_customized"), - ((a, b, a, a), "removed"), - ((a, b, a, c), "existing_changed"), - ((a, b, a, b), "unchanged"), - ((a, b, c, b), "baseline_changed"), - ((a, b, None, b), "no_longer_generated"), - ((a, a, None, a), "no_longer_generated"), - ((a, a, None, None), None), - ((None, b, a, b), "newly_generation_owned"), - ((None, a, a, a), "newly_generation_owned"), - ((a, a, b, a), "baseline_changed"), - ((a, b, b, b), "baseline_changed"), - ((a, a, a, None), "newly_customized"), - ] - for values, expected in cases: - with self.subTest(expected=expected): - self.assertEqual(report.classify(*values), expected) - - def test_report_ignores_handwritten_files_and_shows_custom_patch(self) -> None: - generated, base = self.baseline() - self.write("generated.py", "generated\n# custom\n") - self.write("handwritten.py", "handwritten\n") - head = self.commit() - result, patch = report.build_report(self.repo, base, head, require_head_hash=True) - self.assertEqual(result["counts"]["newly_customized"], 1) - self.assertEqual(result["files"][0]["path"], "generated.py") - self.assertIn(b"+# custom", patch) - self.assertNotIn(b"handwritten", patch) - self.assertEqual(result["after"]["commit"], generated) - body = report.render_report(result, repository="openai/example", run_id=42, run_attempt=2) - self.assertIn("Inspect the custom-code diff", body) - self.assertIn("gh run download 42 --repo openai/example", body) - self.assertIn("--name castiron-custom-code-42-2", body) - self.assertIn(f"--head {head}", body) - self.assertIn("current full custom patch", body) - - def test_bad_hash_and_lost_checkpoint_fail(self) -> None: - generated, base = self.baseline() - stats = self.repo / ".castiron.stats.yml" - stats.write_text( - stats.read_text().replace(report.hash_codegen_commit(self.repo, generated), "f" * 64) - ) - with self.assertRaisesRegex(report.ReportError, "codegen_hash mismatch"): - report.build_report(self.repo, base, self.commit()) - self.git("update-ref", "-d", "refs/remotes/origin/codegen/test") - with self.assertRaises(report.ReportError): - report.resolve_baseline(self.repo, base, fetch=False, require_hash=True) - - def test_stats_symlink_is_rejected(self) -> None: - self.baseline() - path = self.repo / ".castiron.stats.yml" - original = path.read_text() - path.unlink() - path.symlink_to(original) - with self.assertRaisesRegex(report.ReportError, "regular Git file"): - report.read_stats(self.repo, self.commit()) - - def test_next_checkpoint_uses_its_namespaced_ref(self) -> None: - generated, base = self.baseline() - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - ref = "refs/remotes/origin/codegen/castiron/next/test" - self.git("update-ref", ref, checkpoint) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - result = report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - self.assertEqual(result["commit"], checkpoint) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("update-ref", "-d", ref) - with self.assertRaises(report.ReportError): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - - def test_next_checkpoint_cannot_authorize_main_reports(self) -> None: - import custom_code_budget as budget - - generated, base = self.baseline() - self.write( - ".castiron-ratchet.json", '{"schema_version": 1, "max_custom_patch_lines": 10000}\n' - ) - base = self.commit("policy") - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", checkpoint) - self.git("checkout", "--detach", base) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("next checkpoint") - for target, expected in (("main", "error"), ("next", "ok")): - with self.subTest(target=target): - result = report.write_report( - self.repo, - base, - head, - self.repo / "report-output", - fetch=False, - require_head_hash=True, - public=False, - target_branch=target, - ) - self.assertEqual(result["status"], expected) - measured, _ = budget.evaluate( - self.repo, - base, - head, - public=False, - target_branch=target, - ) - self.assertEqual(measured["target_branch"], target) - self.assertEqual( - measured["checks"]["budget"]["state"], - "success" if target == "next" else "failure", - ) - if target == "next": - output = self.repo / "budget-output" - budget.write_result(output, measured) - self.assertEqual( - json.loads((output / "budget.json").read_text())["target_branch"], "next" - ) - self.assertIn("Target next:", (output / "summary.md").read_text()) - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{checkpoint}^{{tree}}"), - GENERATION, - stats["codegen_hash"], - "codegen/castiron/next/test", - None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", snapshot) - stats["public_codegen_sha"] = snapshot - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - public_head = self.commit("public next snapshot") - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.build_report(self.repo, base, public_head, public=True) - accepted, _ = report.build_report( - self.repo, base, public_head, public=True, target_branch="next" - ) - self.assertEqual(accepted["status"], "ok") - - def test_public_next_namespace_root_remains_reserved(self) -> None: - generated, _ = self.baseline() - branch = "codegen/castiron/next" - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - GENERATION, - report.hash_codegen_commit(self.repo, generated), - branch, - None, - ) - metadata = report.public_metadata(self.repo, snapshot) - self.assertEqual(metadata["Target"], "next") - for target in ("main", "next"): - with self.subTest(target=target): - metadata["Target"] = target - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="next", fetch=False - ) - - def test_prepare_public_next_root_and_descendant(self) -> None: - generated, base = self.baseline() - self.git("branch", "codegen/test", generated) - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("branch", "codegen/castiron/next/test", checkpoint) - self.git("checkout", "--detach", base) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("next checkpoint") - self.git("remote", "add", "origin", str(self.repo)) - public = self.repo / "public" - self.git("clone", "--no-local", str(self.repo), str(public)) - report.git(public, "checkout", "--detach", base) - public_remote = self.repo / "public.git" - report.git(public, "init", "--bare", str(public_remote)) - report.git(public, "remote", "set-url", "origin", str(public_remote)) - report.git(public, "push", "origin", f"{base}:refs/heads/main") - for branch in ("castiron/next", "castiron/next/test"): - with self.subTest(branch=branch): - snapshot = report.prepare_public_snapshot( - self.repo, base, head, public, base, branch - ) - self.assertEqual(snapshot["branch"], f"codegen/{branch}") - public_metadata = report.public_metadata(public, snapshot["commit"]) - self.assertEqual(public_metadata["Target"], "next") - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - public, - snapshot["commit"], - public_metadata, - target_branch="main", - fetch=False, - ) - with self.assertRaises(report.ReportError): - report.prepare_public_snapshot(self.repo, base, head, public, base, "castiron/nextish") - - def test_legacy_public_snapshot_requires_trusted_main_adoption(self) -> None: - generated, base = self.baseline() - branch = "codegen/castiron/promotions/pr-42" - content_hash = report.hash_codegen_commit(self.repo, generated) - tree = self.git("rev-parse", f"{generated}^{{tree}}") - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - legacy = self.git("commit-tree", tree, "-m", message) - self.git("update-ref", f"refs/remotes/origin/{branch}", legacy) - self.git("update-ref", "refs/remotes/origin/main", base) - stats = report.read_stats(self.repo, base) | {"public_codegen_sha": legacy} - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("untrusted legacy snapshot adoption") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - report.resolve_public_baseline( - self.repo, head, fetch=False, require_hash=True, target_branch="next" - ) - # A side branch can add then remove the reference before merging. None - # of its intermediate stats were ever adopted on main's first-parent line. - base_tree = self.git("rev-parse", f"{base}^{{tree}}") - removed = self.git("commit-tree", base_tree, "-p", head, "-m", "remove reference") - self.write( - ".castiron.stats.yml", - self.git("show", f"{base}:.castiron.stats.yml") + "\n# main metadata retained\n", - ) - self.git("add", ".castiron.stats.yml") - merged = self.git( - "commit-tree", - self.git("write-tree"), - "-p", - base, - "-p", - removed, - "-m", - "merge clean tip with main metadata update", - ) - self.git("update-ref", "refs/remotes/origin/main", merged) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - # A merge that actually adopts the snapshot on main is sufficient. - adopted = self.git( - "commit-tree", - self.git("rev-parse", f"{head}^{{tree}}"), - "-p", - merged, - "-p", - head, - "-m", - "adopt snapshot on main", - ) - self.git("update-ref", "refs/remotes/origin/main", adopted) - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - for spelling in (f'"{legacy}"', f" {legacy}"): - with self.subTest(spelling=spelling): - self.write( - ".castiron.stats.yml", - "".join( - f"{key}: {spelling if key == 'public_codegen_sha' else value}\n" - for key, value in stats.items() - ), - ) - self.git("add", ".castiron.stats.yml") - formatted = self.git( - "commit-tree", self.git("write-tree"), "-p", base, "-m", "formatted adoption" - ) - self.git("update-ref", "refs/remotes/origin/main", formatted) - report.resolve_public_baseline(self.repo, formatted, fetch=False, require_hash=True) - - # Replacing a comment with the field preserves the SHA occurrence count. - original_stats = self.git("show", f"{base}:.castiron.stats.yml") + "\n" - self.write(".castiron.stats.yml", original_stats + f"# candidate: {legacy}\n") - self.git("add", ".castiron.stats.yml") - mentioned = self.git( - "commit-tree", self.git("write-tree"), "-p", base, "-m", "mention candidate" - ) - self.git("update-ref", "refs/remotes/origin/main", mentioned) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, legacy, fetch=False)) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - self.write(".castiron.stats.yml", original_stats + f"public_codegen_sha: {legacy}\n") - self.git("add", ".castiron.stats.yml") - adopted = self.git( - "commit-tree", self.git("write-tree"), "-p", mentioned, "-m", "adopt candidate" - ) - self.git("update-ref", "refs/remotes/origin/main", adopted) - report.resolve_public_baseline(self.repo, adopted, fetch=False, require_hash=True) - - # Explicit next metadata cannot be grandfathered, even with an old locator. - explicit_next = self.git( - "commit-tree", tree, "-m", message + "Castiron-Public-Codegen-Target: next\n" - ) - self.git("update-ref", f"refs/remotes/origin/{branch}", explicit_next) - stats["public_codegen_sha"] = explicit_next - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("explicit next target") - self.git("update-ref", "refs/remotes/origin/main", head) - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - - def test_legacy_bootstrap_baseline_uses_verified_main_lineage(self) -> None: - generated, base = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - branch = "codegen/castiron/promotions/pr-1" - - def legacy_snapshot(generation: str, parent: str | None = None) -> str: - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - return self.git("commit-tree", tree, *(["-p", parent] if parent else []), "-m", message) - - bootstrap = legacy_snapshot(GENERATION) - generation = "650e8400-e29b-41d4-a716-446655440000" - adopted = legacy_snapshot(generation, bootstrap) - self.git("update-ref", f"refs/remotes/origin/{branch}", adopted) - stats = report.read_stats(self.repo, base) | { - "generation_id": generation, - "public_codegen_sha": adopted, - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("adopt first public checkpoint") - self.git("update-ref", "refs/remotes/origin/main", head) - result, _ = report.build_report(self.repo, base, head, public=True, require_head_hash=True) - self.assertEqual(result["before"]["commit"], bootstrap) - self.assertEqual(result["after"]["commit"], adopted) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, bootstrap, fetch=False)) - - # An unadopted legacy head cannot authorize its own bootstrap lineage. - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline( - self.repo, base, fetch=False, require_hash=False, lineage=bootstrap - ) - - def test_public_next_namespace_root_is_reserved(self) -> None: - generated, _ = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - for branch, target in ( - ("codegen/castiron/next", "next"), - ("codegen/castiron/next/child", "next"), - ("codegen/castiron/next-door", "main"), - ): - with self.subTest(branch=branch): - snapshot = report.create_public_snapshot( - self.repo, tree, GENERATION, content_hash, branch, None - ) - metadata = report.public_metadata(self.repo, snapshot) - self.assertEqual(metadata["Target"], target) - # A misleading explicit target cannot authorize the reserved root. - metadata["Target"] = "main" - if target == "next": - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - else: - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - - def test_legacy_main_adoption_accepts_stats_formatting(self) -> None: - generated, base = self.baseline() - branch = "codegen/castiron/promotions/pr-42" - content_hash = report.hash_codegen_commit(self.repo, generated) - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - snapshot = self.git( - "commit-tree", self.git("rev-parse", f"{generated}^{{tree}}"), "-m", message - ) - self.git("update-ref", f"refs/remotes/origin/{branch}", snapshot) - for value in (f"'{snapshot}'", f'"{snapshot}"', f" {snapshot}"): - with self.subTest(value=value): - self.git("checkout", "--detach", "-q", base) - stats = self.git("show", f"{base}:.castiron.stats.yml") - self.write(".castiron.stats.yml", stats + f"\n# pending snapshot: {snapshot}\n") - commented = self.commit("mention snapshot without adopting it") - self.git("update-ref", "refs/remotes/origin/main", commented) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, snapshot, fetch=False)) - self.write(".castiron.stats.yml", stats + f"\npublic_codegen_sha: {value}\n") - adopted = self.commit("adopt formatted snapshot") - self.git("update-ref", "refs/remotes/origin/main", adopted) - resolved = report.resolve_public_baseline( - self.repo, adopted, fetch=False, require_hash=True - ) - self.assertEqual(resolved["commit"], snapshot) - - def test_legacy_bootstrap_parent_uses_main_adopted_lineage(self) -> None: - generated, base = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - child_generation = "550e8400-e29b-41d4-a716-446655440001" - for parent_branch, parent_target, allowed in ( - ("codegen/castiron/bootstrap", "", True), - ("codegen/castiron/bootstrap", "next", False), - ("codegen/castiron/next", "", False), - ("codegen/castiron/next/bootstrap", "", False), - ): - with self.subTest(branch=parent_branch, target=parent_target): - for reserved in self.git( - "for-each-ref", - "--format=%(refname)", - "refs/remotes/origin/codegen/castiron/next", - ).splitlines(): - self.git("update-ref", "-d", reserved) - self.git("checkout", "--detach", "-q", base) - self.git("update-ref", "refs/remotes/origin/main", base) - parent_message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {parent_branch}\n" - ) - if parent_target: - parent_message += f"Castiron-Public-Codegen-Target: {parent_target}\n" - parent = self.git("commit-tree", tree, "-m", parent_message) - self.git("update-ref", f"refs/remotes/origin/{parent_branch}", parent) - child_branch = "codegen/castiron/promotions/test" - child_message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {child_generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {child_branch}\n" - ) - child = self.git("commit-tree", tree, "-p", parent, "-m", child_message) - self.git("update-ref", f"refs/remotes/origin/{child_branch}", child) - stats = report.read_stats(self.repo, base) | { - "generation_id": child_generation, - "public_codegen_sha": child, - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("adopt only the legacy descendant") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.build_report(self.repo, base, head, public=True, require_head_hash=True) - self.git("update-ref", "refs/remotes/origin/main", head) - if allowed: - result, _ = report.build_report( - self.repo, base, head, public=True, require_head_hash=True - ) - self.assertEqual(result["status"], "ok") - self.assertEqual( - report.resolve_public_baseline( - self.repo, base, fetch=False, require_hash=False, lineage=child - )["commit"], - parent, - ) - # Adoption of a descendant never authorizes a direct parent reference. - direct = report.read_stats(self.repo, base) | {"public_codegen_sha": parent} - self.write( - ".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in direct.items()) - ) - direct_head = self.commit("reference an unadopted parent directly") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline( - self.repo, direct_head, fetch=False, require_hash=True, lineage=child - ) - else: - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.build_report( - self.repo, base, head, public=True, require_head_hash=True - ) - - def test_legacy_checkpoint_cannot_use_next_namespace(self) -> None: - generated, base = self.baseline() - self.git("checkout", "--detach", generated) - metadata = {"generation_id": GENERATION, "source_branch": "castiron/next/test"} - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", checkpoint) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - for target in ("main", "next"): - with ( - self.subTest(target=target), - self.assertRaisesRegex(report.ReportError, "reserved next"), - ): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch=target - ) - - def test_invalid_checkpoint_envelopes_are_rejected(self) -> None: - _, base = self.baseline() - record = {"generation_id": GENERATION, "source_branch": "test"} - envelope = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": record, - } - for metadata in ( - envelope | {"schema_version": "unknown"}, - envelope | {"release_target": "main"}, - envelope | {"extra": True}, - envelope | {"record": record | {"release_target": "next"}}, - record | {"release_target": "next"}, - ): - with self.subTest(metadata=metadata): - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - stats = report.read_stats(self.repo, base) | {"codegen_sha": checkpoint} - with self.assertRaisesRegex(report.ReportError, "envelope"): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - - def test_public_snapshot_rejects_private_governance_and_unpublished_paths(self) -> None: - entry = report.Entry(b"100644", "a" * 40) - blocked = [ - b".castiron/private.json", - b".github/CODEOWNERS", - b".gitmodules", - b"CODEOWNERS", - b"SECURITY.md", - b"docs/SECURITY.md", - ] - for path in blocked: - with ( - self.subTest(path=path), - mock.patch.object( - report, "tree_entries", side_effect=[{path: entry}, {path: entry}] - ), - mock.patch.object(report, "read_blobs") as read, - ): - with self.assertRaisesRegex(report.ReportError, "private or governance"): - report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) - read.assert_not_called() - with ( - mock.patch.object(report, "tree_entries", side_effect=[{b"unreleased.py": entry}, {}]), - mock.patch.object(report, "read_blobs") as read, - ): - with self.assertRaisesRegex( - report.ReportError, "absent from the approved public SDK tree" - ): - report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) - read.assert_not_called() - - def test_zero_new_files_summary_reports_existing_changes(self) -> None: - _, clean = self.baseline() - self.write("generated.py", "generated\n# existing customization\n") - base = self.commit() - self.write("generated.py", "generated\n# existing customization updated\n") - result, _ = report.build_report(self.repo, base, self.commit()) - body = report.render_report(result) - self.assertIn("✅ No new custom-code files detected.", body) - self.assertIn("1 mixed file remains; 1 existing customization changed.", body) - self.assertNotIn("0 newly customized", body) - empty, _ = report.build_report(self.repo, clean, clean) - self.assertIn( - "0 mixed files remain; 0 existing customizations changed.", report.render_report(empty) - ) - - def test_new_mixed_file_headline_includes_changed_generated_baselines(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - for category in ("baseline_changed", "newly_generation_owned"): - result["files"] = [ - { - "path": "generated.py", - "category": category, - "custom_before": False, - "custom_after": True, - "added": "1", - "removed": "1", - } - ] - body = report.render_report(result) - self.assertNotIn("No new custom-code files", body) - self.assertIn("1 newly customized", body) - - def test_removals_include_changed_baselines_but_not_handwritten_only_files(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - result["files"] = [ - { - "path": category, - "category": category, - "custom_before": True, - "custom_after": False, - "added": "0", - "removed": "0", - } - for category in ("baseline_changed", "no_longer_generated") - ] - self.assertIn("1 customizations removed", report.render_report(result)) - result["files"].append( - { - "path": "new.py", - "category": "newly_customized", - "custom_before": False, - "custom_after": True, - "added": "1", - "removed": "0", - } - ) - self.assertIn("1 customizations removed", report.render_report(result)) - - def test_pending_legacy_promotion_is_upgraded_without_authorizing_legacy_reports(self) -> None: - generated, source_base = self.baseline() - self.git("branch", "codegen/test", generated) - private_remote = self.repo / "private.git" - self.git("clone", "--bare", str(self.repo), str(private_remote)) - self.git("remote", "add", "origin", str(private_remote)) - public = self.repo / "public" - public.mkdir() - report.git(public, "init", "-q", "-b", "main") - report.git(public, "config", "user.name", "Public test") - report.git(public, "config", "user.email", "public@example.test") - (public / "generated.py").write_text("generated\n") - (public / ".castiron.stats.yml").write_text((self.repo / ".castiron.stats.yml").read_text()) - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "public baseline") - initial = report.git(public, "rev-parse", "HEAD").decode().strip() - branch = "castiron/promotions/legacy-retry" - codegen_branch = f"codegen/{branch}" - - def legacy_snapshot( - tree: str, generation: str, content_hash: str, parent: str | None - ) -> str: - message = ( - "Castiron generated snapshot\n\nCastiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {codegen_branch}\n" - ) - parents = ["-p", parent] if parent is not None else [] - return report.git(public, "commit-tree", tree, *parents, "-m", message).decode().strip() - - base_tree = report.copy_generated_tree(self.repo, generated, public, initial) - legacy_base = legacy_snapshot( - base_tree, GENERATION, report.hash_codegen_commit(self.repo, generated), None - ) - with (public / ".castiron.stats.yml").open("a") as stats: - stats.write(f"public_codegen_sha: {legacy_base}\n") - report.git(public, "add", ".castiron.stats.yml") - report.git(public, "commit", "-qm", "adopt legacy baseline") - public_base = report.git(public, "rev-parse", "HEAD").decode().strip() - public_remote = self.repo / "public.git" - report.git(public, "clone", "--bare", str(public), str(public_remote)) - report.git(public, "remote", "add", "origin", str(public_remote)) - report.git(public, "push", "origin", f"{legacy_base}:refs/heads/{codegen_branch}") - - generation = "650e8400-e29b-41d4-a716-446655440000" - self.git("checkout", "--detach", generated) - self.write("generated.py", "regenerated\n") - self.git("add", "generated.py") - metadata = base64.b64encode( - json.dumps({"generation_id": generation, "source_branch": "test"}).encode() - ).decode() - self.git("commit", "-qm", f"updated codegen\n\nGeneration metadata: {metadata}") - updated = self.git("rev-parse", "HEAD") - self.git("push", "origin", f"{updated}:refs/heads/codegen/test") - content_hash = report.hash_codegen_commit(self.repo, updated) - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {generation}\ncodegen_sha: {updated}\ncodegen_hash: {content_hash}\n", - ) - self.git("add", ".castiron.stats.yml") - self.git("commit", "-qm", "updated integrated SDK") - source_head = self.git("rev-parse", "HEAD") - tree = report.copy_generated_tree(self.repo, updated, public, public_base) - pending = legacy_snapshot(tree, generation, content_hash, legacy_base) - report.git(public, "push", "origin", f"{pending}:refs/heads/{codegen_branch}") - self.assertTrue(report.legacy_snapshot_on_main(public, legacy_base, fetch=True)) - self.assertFalse(report.legacy_snapshot_on_main(public, pending, fetch=False)) - (public / "generated.py").write_text("regenerated\n") - report.git(public, "add", "generated.py") - - message = report.git(public, "show", "-s", "--format=%B", pending).decode() - for invalid_message, error in ( - (message + "Castiron-Public-Codegen-Target: next\n", "next checkpoint"), - (message.replace(content_hash, "f" * 64), "invalid provenance"), - ( - message.replace(codegen_branch, "codegen/another-promotion"), - "no verified main adoption", - ), - ): - with self.subTest(error=error): - invalid = ( - report.git( - public, "commit-tree", tree, "-p", legacy_base, "-m", invalid_message - ) - .decode() - .strip() - ) - report.git( - public, "push", "--force", "origin", f"{invalid}:refs/heads/{codegen_branch}" - ) - with self.assertRaisesRegex(report.ReportError, error): - report.prepare_public_snapshot( - self.repo, source_base, source_head, public, public_base, branch - ) - report.git(public, "push", "--force", "origin", f"{pending}:refs/heads/{codegen_branch}") - upgraded = report.prepare_public_snapshot( - self.repo, source_base, source_head, public, public_base, branch - ) - self.assertNotEqual(upgraded["commit"], pending) - self.assertEqual(upgraded["base_commit"], legacy_base) - self.assertEqual(upgraded["previous"], pending) - self.assertEqual( - report.git(public, "rev-parse", upgraded["commit"] + "^1").decode().strip(), pending - ) - self.assertEqual(report.public_metadata(public, upgraded["commit"])["Target"], "main") - self.assertEqual(report.hash_codegen_commit(public, upgraded["commit"]), content_hash) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.validate_public_target( - public, - pending, - report.public_metadata(public, pending), - target_branch="main", - fetch=False, - ) - report.git(public, "push", "origin", f"{upgraded['commit']}:refs/heads/{codegen_branch}") - report.git(public, "commit", "-qm", "upgraded public SDK") - head = report.git(public, "rev-parse", "HEAD").decode().strip() - result, _ = report.build_report( - public, public_base, head, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(result["after"]["generation_id"], generation) - - def test_public_snapshot_has_no_private_history_and_reports_without_private_remote( - self, - ) -> None: - self.write("private-only.txt", "must never be published\n") - private_ancestor = self.commit("private history") - (self.repo / "private-only.txt").unlink() - generated, base = self.baseline() - self.git("branch", "codegen/test", generated) - private_remote = self.repo / "private.git" - self.git("clone", "--bare", str(self.repo), str(private_remote)) - self.git("remote", "add", "origin", str(private_remote)) - - public = self.repo / "public" - public.mkdir() - report.git(public, "init", "-q", "-b", "main") - report.git(public, "config", "user.name", "Public test") - report.git(public, "config", "user.email", "public@example.test") - (public / "generated.py").write_text("generated\n") - (public / ".castiron.stats.yml").write_text( - (self.repo / ".castiron.stats.yml").read_text().split("codegen_hash:")[0] - ) - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "already public") - public_base = report.git(public, "rev-parse", "HEAD").decode().strip() - public_remote = self.repo / "public.git" - report.git(public, "clone", "--bare", str(public), str(public_remote)) - report.git(public, "remote", "add", "origin", str(public_remote)) - snapshot = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(snapshot["base_commit"], snapshot["commit"]) - self.assertNotEqual(snapshot["commit"], generated) - self.assertEqual(snapshot["base_commit"], snapshot["commit"]) - self.assertEqual(report.git(public, "rev-list", "--count", snapshot["commit"]), b"1\n") - self.assertEqual(report.hash_codegen_commit(public, snapshot["commit"]), snapshot["hash"]) - with self.assertRaises(report.ReportError): - report.git(public, "cat-file", "-e", private_ancestor) - self.assertNotIn( - b"private-only.txt", report.git(public, "ls-tree", "-r", snapshot["commit"]) - ) - report.git( - public, "push", "origin", f"{snapshot['commit']}:refs/heads/{snapshot['branch']}" - ) - (public / "generated.py").write_text("generated\n# custom\n") - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "public customization") - public_head = report.git(public, "rev-parse", "HEAD").decode().strip() - result, patch = report.build_report( - public, public_base, public_head, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(result["counts"]["newly_customized"], 1) - self.assertIn(b"+# custom", patch) - self.assertIn(" --public", report.render_report(result)) - self.assertFalse(result["before"]["hash_recorded"]) - again = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(again["base_commit"], snapshot["base_commit"]) - self.assertEqual(again["commit"], snapshot["commit"]) - self.assertEqual(again["base_commit"], snapshot["base_commit"]) - tree = report.git(public, "rev-parse", f"{snapshot['commit']}^{{tree}}").decode().strip() - message = report.git(public, "show", "-s", "--format=%B", snapshot["commit"]) - chained = ( - report.git( - public, - "commit-tree", - tree, - "-p", - snapshot["commit"], - "-F", - "-", - input_bytes=message, - ) - .decode() - .strip() - ) - report.git( - public, "push", "--force", "origin", f"{chained}:refs/heads/{snapshot['branch']}" - ) - reused = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(reused["commit"], chained) - - # A fresh generation may introduce files. Its public parent must be a - # previously public snapshot, never the private checkpoint parent. - next_generation = "650e8400-e29b-41d4-a716-446655440000" - self.git("checkout", "--detach", generated) - self.write("generated.py", "regenerated\n") - self.write("new.py", "new generated file\n") - self.git("add", "--", "generated.py", "new.py") - metadata = base64.b64encode( - json.dumps( - { - "generation_id": next_generation, - "source_branch": "test", - } - ).encode() - ).decode() - self.git("commit", "-qm", f"next codegen\n\nGeneration metadata: {metadata}") - next_codegen = self.git("rev-parse", "HEAD") - self.git("push", "origin", f"{next_codegen}:refs/heads/codegen/test") - self.write( - ".castiron.stats.yml", - ( - f"schema_version: 1\ngeneration_id: {next_generation}\n" - f"codegen_sha: {next_codegen}\n" - f"codegen_hash: {report.hash_codegen_commit(self.repo, next_codegen)}\n" - ), - ) - self.git("add", "--", ".castiron.stats.yml") - self.git("commit", "-qm", "next integrated SDK") - next_head = self.git("rev-parse", "HEAD") - # An unreviewed path in private HEAD is not enough to publish it. - with self.assertRaisesRegex(report.ReportError, "approved public SDK tree"): - report.prepare_public_snapshot( - self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" - ) - # Model normal promotion applying its reviewed patch to the public index. - (public / "generated.py").write_text("regenerated\n") - (public / "new.py").write_text("new generated file\n") - report.git(public, "add", "--", "generated.py", "new.py") - advanced = report.prepare_public_snapshot( - self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual( - report.git(public, "rev-parse", advanced["commit"] + "^1").decode().strip(), chained - ) - self.assertEqual(report.hash_codegen_commit(public, advanced["commit"]), advanced["hash"]) - with self.assertRaises(report.ReportError): - report.git(public, "cat-file", "-e", next_codegen) - report.git( - public, "push", "origin", f"{advanced['commit']}:refs/heads/{advanced['branch']}" - ) - (public / "generated.py").write_text("regenerated\n") - (public / "new.py").write_text("new generated file\n") - report.git(public, "add", "--", ".castiron.stats.yml", "generated.py", "new.py") - report.git(public, "commit", "-qm", "next public SDK") - next_public = report.git(public, "rev-parse", "HEAD").decode().strip() - compared, _ = report.build_report( - public, public_base, next_public, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(compared["before"]["generation_id"], GENERATION) - self.assertEqual(compared["after"]["generation_id"], next_generation) - - # Once A lands, handwritten-only promotion B may reuse A's exact - # snapshot. Its original branch is a locator, not exclusive ownership. - report.git(public, "push", "origin", f"{next_public}:refs/heads/main") - self.write("generated.py", "regenerated\n# handwritten change\n") - self.git("add", "--", "generated.py") - self.git("commit", "-qm", "handwritten change") - custom_head = self.git("rev-parse", "HEAD") - (public / "generated.py").write_text("regenerated\n# handwritten change\n") - report.git(public, "add", "--", "generated.py") - other_branch = "castiron/promotions/pr-2" - other = report.prepare_public_snapshot( - self.repo, next_head, custom_head, public, next_public, other_branch - ) - self.assertEqual(other["commit"], advanced["commit"]) - self.assertEqual( - report.public_metadata(public, other["commit"])["Branch"], advanced["branch"] - ) - report.git(public, "push", "origin", f"{other['commit']}:refs/heads/{other['branch']}") - retried = report.prepare_public_snapshot( - self.repo, next_head, custom_head, public, next_public, other_branch - ) - self.assertEqual(retried["commit"], other["commit"]) - - # A merge-queue commit also resolves the baseline from its own stats, - # independently of the name or parents of the integrated SDK commit. - merged_tree = report.git(public, "write-tree").decode().strip() - pr_head = ( - report.git( - public, - "commit-tree", - merged_tree, - "-p", - next_public, - "-m", - "public handwritten change", - ) - .decode() - .strip() - ) - merged = ( - report.git( - public, - "commit-tree", - merged_tree, - "-p", - next_public, - "-p", - pr_head, - "-m", - "merge queue", - ) - .decode() - .strip() - ) - merged_report, _ = report.build_report( - public, next_public, merged, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(merged_report["after"]["commit"], advanced["commit"]) - self.assertEqual(merged_report["counts"]["newly_customized"], 1) - - -ADOPTION_GENERATION = "550e8400-e29b-41d4-a716-446655440001" - - -class FirstAdoptionTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.repository = "openai/openai-java" - self.write(".stats.yml", "configured_endpoints: 278\n") - self.root = self.commit("legacy root") - self.git("remote", "add", "origin", "https://github.com/openai/openai-java.git") - self.policy: dict[str, Any] = {"schema_version": 1, "max_custom_patch_lines": 3000} - self.write(".castiron-ratchet.json", json.dumps(self.policy) + "\n") - self.g0 = self.generated("source\n", GENERATION, False) - self.public0 = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{self.g0}^{{tree}}"), - GENERATION, - report.hash_codegen_commit(self.repo, self.g0), - "codegen/main", - None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/main", self.public0) - self.write("generated.py", "source\n") - self.stats(self.g0, self.public0, GENERATION) - self.main = self.commit("approved main") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.git("checkout", "--detach", self.root) - self.pin: dict[str, Any] = { - **self.policy, - "next_policy": {"repository": self.repository, "main_sha": self.main}, - } - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("separate approved next policy") - self.git("update-ref", "refs/remotes/origin/next", self.base) - - def generated(self, text: str, generation: str, next_target: bool) -> str: - blob = ( - report.git(self.repo, "hash-object", "-w", "--stdin", input_bytes=text.encode()) - .decode() - .strip() - ) - tree = ( - report.git( - self.repo, - "mktree", - input_bytes=f"100644 blob {blob}\tgenerated.py\n".encode(), - ) - .decode() - .strip() - ) - record: dict[str, Any] = { - "generation_id": generation, - "source_branch": "test", - "target": self.repository.split("/")[1], - "language": self.repository.split("-")[-1], - } - if next_target: - record["public_baseline_commit"] = self.base - record = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": record, - } - encoded = base64.b64encode(json.dumps(record).encode()).decode() - commit = ( - report.git( - self.repo, - "commit-tree", - tree, - input_bytes=f"codegen\n\nGeneration metadata: {encoded}\n".encode(), - ) - .decode() - .strip() - ) - namespace = "castiron/next/" if next_target else "" - self.git("update-ref", f"refs/remotes/origin/codegen/{namespace}test", commit) - return commit - - def stats(self, private: str, public: str, generation: str) -> None: - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {generation}\ncodegen_sha: {private}\npublic_codegen_sha: {public}\ncodegen_hash: {report.hash_codegen_commit(self.repo, private)}\nconfig_hash: source-config\nopenapi_spec_hash: source-spec\n", - ) - - def seed(self) -> str: - # Synthetic two-parent private seed; production worker initialization is - # covered separately in the SDK worker candidate-flow suite. - self.git("read-tree", "--reset", "-u", self.main) - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.git("add", "-A") - return self.git( - "commit-tree", - self.git("write-tree"), - "-p", - self.base, - "-p", - self.main, - "-m", - "approved main adoption fixture", - ) - - def head(self, *, public_receipt: bool = True, customize: bool = False) -> str: - seed = self.seed() - self.git("checkout", "--detach", seed) - self.g1 = self.generated("next generated\n", ADOPTION_GENERATION, True) - snap = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{self.g1}^{{tree}}"), - ADOPTION_GENERATION, - report.hash_codegen_commit(self.repo, self.g1), - "codegen/castiron/next/new", - self.public0, - (self.base, self.main) if public_receipt else None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/new", snap) - self.write( - "generated.py", - "next generated\n" + ("# customization\n" if customize else ""), - ) - self.stats(self.g1, snap, ADOPTION_GENERATION) - return self.commit("next candidate") - - def test_java_public_and_private_report_have_no_fake_before_generation(self) -> None: - head = self.head(customize=True) - for public in (False, True): - result, patch = report.build_report( - self.repo, - self.base, - head, - public=public, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - self.assertEqual(result["files"][0]["added"], "1") - self.assertIn(b"# customization", patch) - self.assertIn("Legacy before-generation unavailable", report.render_report(result)) - - def test_cli_uses_same_protected_pin_path(self) -> None: - self.repository = "openai/openai-cli" - self.git("remote", "set-url", "origin", "https://github.com/openai/openai-cli.git") - self.pin["next_policy"]["repository"] = self.repository - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("CLI approved policy fixture") - head = self.head() - result, _ = report.build_report( - self.repo, - self.base, - head, - public=False, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - - def test_promotion_base_cli_recognizes_exact_next_branch(self) -> None: - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - source, destination = args[-1].lstrip("+").split(":", 1) - return original( - repo, - "update-ref", - destination, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return original(repo, *args, input_bytes=input_bytes) - - for base, branch, expected in ( - (self.base, "castiron/next", self.main), - (self.base, "castiron/next/child", self.main), - (self.base, "castiron/nextish", None), - (self.base, "castiron/next-door", None), - (self.base, "castiron/promotions/main", None), - (self.main, "castiron/promotions/main", self.main), - ): - with self.subTest(base=base, branch=branch): - output, errors = io.StringIO(), io.StringIO() - with ( - mock.patch.object(report, "git", side_effect=offline_git), - mock.patch.object( - report.sys, - "argv", - [ - "custom_code_report.py", - "promotion-base", - "--repo", - str(self.repo), - "--base", - base, - "--branch", - branch, - ], - ), - mock.patch.object(report.sys, "stdout", output), - mock.patch.object(report.sys, "stderr", errors), - ): - result = report.main() - if expected is None: - self.assertEqual(result, 1) - self.assertEqual(output.getvalue(), "") - self.assertIn("first adoption is next-only", errors.getvalue()) - else: - self.assertEqual(result, 0, errors.getvalue()) - self.assertEqual(output.getvalue(), expected + "\n") - - def test_promotion_prepares_new_snapshot_from_real_main_snapshot(self) -> None: - head = self.head() - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - for arg in args: - if arg.startswith("+refs/") and ":" in arg: - source, destination = arg.lstrip("+").split(":", 1) - original( - repo, - "update-ref", - destination, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return b"" - if args[0] == "ls-remote": - return b"" - return original(repo, *args, input_bytes=input_bytes) - - with mock.patch.object(report, "git", side_effect=offline_git): - prepared = report.prepare_public_snapshot( - self.repo, self.base, head, self.repo, self.base, "castiron/next/fresh" - ) - metadata = report.public_metadata(self.repo, prepared["commit"]) - self.assertEqual(metadata["Target"], "next") - self.assertEqual(metadata["Adoption-Base"], self.base) - self.assertEqual(metadata["Adoption-Source"], self.main) - self.assertEqual(prepared["base_commit"], self.public0) - self.assertEqual(self.git("rev-parse", prepared["commit"] + "^"), self.public0) - - def test_missing_receipt_rejected(self) -> None: - head = self.head(public_receipt=False) - with self.assertRaisesRegex(report.ReportError, "authorized first adoption"): - report.build_report(self.repo, self.base, head, public=True, target_branch="next") - - def test_wrong_sdk_record_rejected(self) -> None: - head = self.head() - self.repository = "openai/openai-cli" - wrong = self.generated("next generated\n", ADOPTION_GENERATION, True) - self.stats( - wrong, report.read_stats(self.repo, head)["public_codegen_sha"], ADOPTION_GENERATION - ) - head = self.commit("wrong SDK binding") - with self.assertRaisesRegex(report.ReportError, "SDK mismatch"): - report.build_report(self.repo, self.base, head, public=False, target_branch="next") - - def test_config_change_rejected(self) -> None: - self.head() - path = self.repo / ".castiron.stats.yml" - path.write_text(path.read_text().replace("source-config", "wrong-config")) - head = self.commit() - with self.assertRaisesRegex(report.ReportError, "authenticated main inputs"): - report.build_report(self.repo, self.base, head, public=False, target_branch="next") - - def test_existing_corrupt_deleted_stats_rejected(self) -> None: - self.write(".castiron.stats.yml", "corrupt: true\n") - corrupt = self.commit() - with self.assertRaisesRegex(report.ReportError, "existing/corrupt"): - report.first_adoption_source(self.repo, corrupt, target_branch="next", fetch=False) - self.git("rm", ".castiron.stats.yml") - deleted = self.commit() - with self.assertRaisesRegex(report.ReportError, "deleted"): - report.first_adoption_source(self.repo, deleted, target_branch="next", fetch=False) - - def test_candidate_cannot_select_source_or_remove_policy(self) -> None: - self.head() - self.write(".castiron-ratchet.json", json.dumps(self.policy) + "\n") - head = self.commit() - with self.assertRaisesRegex(report.ReportError, "protected target policy"): - report.build_report(self.repo, self.base, head, public=True, target_branch="next") - - def test_current_count_and_strict_zero_still_enforced(self) -> None: - import custom_code_budget as budget - - head = self.head(customize=True) - measurement = report.build_report( - self.repo, self.base, head, public=True, target_branch="next" - ) - evaluated, _ = budget.evaluate( - self.repo, - self.base, - head, - public=True, - policy_sha=self.main, - target_branch="next", - measurement=measurement, - ) - self.assertEqual(evaluated["total"], 1) - self.assertEqual(evaluated["checks"]["budget"]["state"], "success", evaluated) - self.write( - ".castiron-ratchet.json", - '{"schema_version":2,"max_custom_patch_lines":0,"generated_files":"strict"}\n', - ) - strict = self.commit() - evaluated, _ = budget.evaluate( - self.repo, - self.base, - head, - public=True, - policy_sha=strict, - target_branch="next", - measurement=measurement, - ) - self.assertEqual(evaluated["limit"], 0) - self.assertEqual(evaluated["checks"]["budget"]["state"], "failure") - self.assertIn("manifest", evaluated["checks"]["budget"]["description"]) - - def test_unapproved_source_pin_rejected(self) -> None: - self.pin["next_policy"]["main_sha"] = self.base - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - invalid = self.commit() - with self.assertRaisesRegex(report.ReportError, "authenticated main history"): - report.first_adoption_source(self.repo, invalid, target_branch="next", fetch=False) - - def test_advanced_target_requires_fresh_adoption(self) -> None: - head = self.head() - self.git("checkout", "--detach", self.base) - self.write("legacy-note.txt", "another approved target change\n") - advanced = self.commit() - with self.assertRaisesRegex(report.ReportError, "target advanced"): - report.build_report(self.repo, advanced, head, public=True, target_branch="next") - - def test_main_target_rejected(self) -> None: - with self.assertRaisesRegex(report.ReportError, "next-only"): - report.first_adoption_source(self.repo, self.base, target_branch="main", fetch=False) - - def test_normal_public_main_report_unchanged(self) -> None: - self.git("checkout", "--detach", self.main) - self.write("generated.py", "source\n# customization\n") - head = self.commit() - result, _ = report.build_report( - self.repo, self.main, head, public=True, target_branch="main" - ) - self.assertIsNotNone(result["before"]) - self.assertEqual(result["before_status"], "verified") - - def promoted_public_head(self) -> tuple[Path, str, dict[str, Any]]: - # Exercise the separate public checkout and the actual single-parent - # commit-tree shape in sdk/promote.rs, not the private merge seed. - import subprocess - - destination = self.repo / "public-checkout" - head = self.head(customize=True) - subprocess.run( - ["git", "clone", "-q", "--no-hardlinks", str(self.repo), str(destination)], check=True - ) - - def public_git(*args: str) -> str: - return report.git(destination, *args).decode().strip() - - public_git("config", "user.name", "Castiron test") - public_git("config", "user.email", "castiron@example.test") - public_git("remote", "set-url", "origin", "https://github.com/" + self.repository + ".git") - for ref, commit in [ - ("main", self.main), - ("next", self.base), - ("codegen/main", self.public0), - ]: - public_git("update-ref", "refs/remotes/origin/" + ref, commit) - public_git("checkout", "--detach", self.base) - # Normal promotion applies the reviewed private patch to this index. - public_git("read-tree", "--reset", "-u", head) - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - for arg in args: - if arg.startswith("+refs/") and ":" in arg: - source, target = arg.lstrip("+").split(":", 1) - original( - repo, - "update-ref", - target, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return b"" - if args[0] == "ls-remote": - return b"" - return original(repo, *args, input_bytes=input_bytes) - - with mock.patch.object(report, "git", side_effect=offline_git): - prepared = report.prepare_public_snapshot( - self.repo, self.base, head, destination, self.base, "castiron/next/fresh" - ) - public_git("update-ref", "refs/remotes/origin/" + prepared["branch"], prepared["commit"]) - promoted = public_git( - "commit-tree", - public_git("write-tree"), - "-p", - self.base, - "-m", - "normal public promotion", - ) - self.assertEqual( - public_git("rev-list", "--parents", "-n", "1", promoted), promoted + " " + self.base - ) - with self.assertRaises(report.ReportError): - public_git("merge-base", "--is-ancestor", self.main, promoted) - # Private authorization still retains both parents. - self.git("merge-base", "--is-ancestor", self.main, head) - self.git("merge-base", "--is-ancestor", self.base, head) - return destination, promoted, prepared - - def test_normal_single_parent_public_promotion_reports_first_adoption(self) -> None: - destination, promoted, _ = self.promoted_public_head() - result, patch = report.build_report( - destination, - self.base, - promoted, - public=True, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - self.assertIn(b"# customization", patch) - - def test_public_adoption_rejects_disconnected_snapshot_with_valid_receipt(self) -> None: - destination, promoted, prepared = self.promoted_public_head() - # The right receipt/hash alone does not authenticate approved-main lineage. - detached = report.create_public_snapshot( - destination, - report.git(destination, "rev-parse", prepared["commit"] + "^{tree}").decode().strip(), - ADOPTION_GENERATION, - prepared["hash"], - prepared["branch"], - None, - (self.base, self.main), - ) - report.git(destination, "update-ref", "refs/remotes/origin/" + prepared["branch"], detached) - report.git(destination, "checkout", "--detach", promoted) - stats = destination / ".castiron.stats.yml" - stats.write_text(stats.read_text().replace(prepared["commit"], detached)) - report.git(destination, "add", ".castiron.stats.yml") - invalid = ( - report.git( - destination, - "commit-tree", - report.git(destination, "write-tree").decode().strip(), - "-p", - self.base, - "-m", - "disconnected snapshot", - ) - .decode() - .strip() - ) - with self.assertRaises(report.ReportError): - report.build_report( - destination, - self.base, - invalid, - public=True, - target_branch="next", - require_head_hash=True, - ) - - def test_cli_normal_single_parent_public_promotion_reports_first_adoption(self) -> None: - self.repository = "openai/openai-cli" - self.git("remote", "set-url", "origin", "https://github.com/openai/openai-cli.git") - self.pin["next_policy"]["repository"] = self.repository - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("CLI approved next policy") - self.git("update-ref", "refs/remotes/origin/next", self.base) - destination, promoted, _ = self.promoted_public_head() - result, _ = report.build_report( - destination, - self.base, - promoted, - public=True, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_review_revocation.py b/scripts/castiron/test_custom_code_review_revocation.py deleted file mode 100644 index f7285795a8..0000000000 --- a/scripts/castiron/test_custom_code_review_revocation.py +++ /dev/null @@ -1,986 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Exercise review revocation and target freshness against offline GitHub state.""" - -from __future__ import annotations - -import json -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path -from typing import Any - -import custom_code_policy as policy -from custom_code_test_support import WORKFLOW, run_publisher, source_head, source_run - - -@unittest.skipUnless( - shutil.which("node"), "Node is needed to execute the review-revocation fixture" -) -class ReviewRevocationTests(unittest.TestCase): - def payload(self) -> dict[str, Any]: - head, base, queue = "a" * 40, "b" * 40, "c" * 40 - pull = { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "sha": base, "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(head), - "id": 20, - "actor": {"login": "outsider", "type": "User"}, - "event": "pull_request_review", - "status": "completed", - "pull_requests": [{"number": 3, "head": {"sha": head}}], - } - - def original(sha: str, event: str, identity: int) -> dict[str, Any]: - return {**run, "head_sha": sha, "id": identity, "event": event} - - return { - "run": run, - "policies_by_sha": { - base: {"schema_version": 1, "max_custom_patch_lines": 0}, - head: {"schema_version": 1, "max_custom_patch_lines": 1}, - }, - "previous_statuses": [ - { - "context": name, - "state": "success", - "creator": {"login": "github-actions[bot]"}, - "target_url": "https://github.com/openai/example/actions/runs/19", - } - for name in ( - "Castiron next / budget-only change", - "Castiron next / custom-code budget", - ) - ], - "current": pull, - "queue_nodes": [ - {"headCommit": {"oid": queue}, "pullRequest": {"number": 3, "headRefOid": head}} - ], - "runs_by_sha": { - head: [original(head, "pull_request", 12)], - queue: [original(queue, "merge_group", 14)], - }, - "context": { - "payload": {"workflow_run": dict(run)}, - "runId": 21, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - } - - def invoke(self, payload: dict[str, Any]) -> list[dict[str, Any]]: - return run_publisher("Re-evaluate current PR and queue after review changes", payload) - - def test_revokes_pr_and_queue_before_requesting_normal_re_evaluation(self) -> None: - result = self.invoke(self.payload()) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertTrue(all(item["context"].startswith("Castiron next /") for item in result[:4])) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - self.assertNotIn("success", [item.get("state") for item in result]) - - def test_retries_policy_failure_when_main_advances_before_revocation(self) -> None: - payload = self.payload() - payload["main_sha_sequence"] = ["d" * 40, "e" * 40] - payload["main_sha"] = "e" * 40 - payload["policy_error_sequence"] = [True, False] - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_uses_live_target_when_pull_base_snapshot_lags(self) -> None: - for requires_human in (False, True): - with self.subTest(requires_human=requires_human): - payload = self.payload() - payload["target_sha"] = "e" * 40 - payload["expected_policy_base"] = "e" * 40 - payload["requires_human"] = requires_human - result = self.invoke(payload) - if requires_human: - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - else: - self.assertEqual(result, []) - - def test_retries_when_live_target_advances_during_evaluation(self) -> None: - for requires_human in (False, True): - with self.subTest(requires_human=requires_human): - payload = self.payload() - payload["target_sha_sequence"] = ["d" * 40, "e" * 40] - payload["target_sha"] = "e" * 40 - payload["expected_policy_bases"] = ["d" * 40, "e" * 40] - payload["requires_human"] = requires_human - result = self.invoke(payload) - if requires_human: - self.assertEqual( - [item["run_id"] for item in result if "run_id" in item], [12, 14] - ) - self.assertTrue( - all(item.get("state") == "pending" for item in result if "state" in item) - ) - else: - self.assertEqual(result, []) - - def test_refreshes_executable_policy_when_main_changes_review_requirement(self) -> None: - payload = self.payload() - payload["main_sha_sequence"] = ["d" * 40, "e" * 40] - payload["main_sha"] = "e" * 40 - payload["requires_human_by_main"] = {"d" * 40: False, "e" * 40: True} - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_rejects_invalid_trusted_policy_downloads(self) -> None: - for override in [ - {"type": "symlink"}, - {"encoding": "none"}, - {"size": 1048577}, - {"size": 0}, - {"content": ""}, - ]: - with self.subTest(override=override): - payload = self.payload() - payload["policy_file_override"] = override - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def approval(self) -> dict[str, Any]: - return { - "fullDatabaseId": "101", - "state": "APPROVED", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + "a" * 40, - "author": {"login": "reviewer", "__typename": "User"}, - "authorAssociation": "MEMBER", - "commit": {"oid": "a" * 40}, - "updatedAt": "2026-10-22T01:00:00Z", - } - - def recovery(self) -> dict[str, Any]: - payload = self.payload() - payload["review_nodes"] = [self.approval()] - payload["run"]["actor"] = {"login": "reviewer", "type": "User"} - for status in payload["previous_statuses"]: - status["state"] = "failure" - return payload - - def history(self, payload: dict[str, Any], writes: list[dict[str, Any]]) -> None: - payload["statuses_by_sha"] = { - sha: [ - {**item, "creator": {"login": "github-actions[bot]"}} - for item in writes - if item.get("sha") == sha - ] - for sha in ("a" * 40, "c" * 40) - } - payload["runs_by_id"] = { - "21": { - "id": 21, - "workflow_id": 789, - "status": "completed", - "conclusion": "success", - "repository": {"full_name": "openai/example"}, - "event": "workflow_run", - } - } - - def test_unrelated_comment_cannot_invalidate_valid_approval_or_request_recovery(self) -> None: - for state in ("success", "failure", "pending"): - payload = self.recovery() - payload["run"]["actor"]["login"] = "outsider" - payload["review_nodes"].append( - { - **self.approval(), - "fullDatabaseId": "102", - "author": {"login": "outsider", "__typename": "User"}, - "state": "COMMENTED", - "body": "hello", - } - ) - for status in payload["previous_statuses"]: - status["state"] = state - payload["forbid_queue"] = True - self.assertEqual(self.invoke(payload), []) - - def test_later_comment_preserves_green_pr_and_queue_statuses(self) -> None: - for state in ("COMMENTED", "PENDING"): - for paginated in (False, True): - with self.subTest(state=state, paginated=paginated): - payload = self.recovery() - approval = self.approval() - comment = { - **approval, - "fullDatabaseId": "102", - "state": state, - "body": "Follow-up discussion", - } - payload["review_nodes"] = [approval, comment] - if paginated: - payload.pop("review_nodes") - payload["review_pages"] = [ - { - "expected_cursor": None, - "nodes": [comment], - "pageInfo": {"hasNextPage": True, "endCursor": "more"}, - }, - { - "expected_cursor": "more", - "nodes": [approval], - "pageInfo": {"hasNextPage": False}, - }, - ] - for status in payload["previous_statuses"]: - status["state"] = "success" - self.assertEqual(self.invoke(payload), []) - - def test_ineligible_review_cannot_request_recovery(self) -> None: - for overrides in ( - {"state": "COMMENTED"}, - {"body": ""}, - {"authorAssociation": "CONTRIBUTOR"}, - {"commit": {"oid": "b" * 40}}, - {"author": {"login": "author", "__typename": "User"}}, - {"author": {"login": "reviewer", "__typename": "Bot"}}, - ): - with self.subTest(overrides=overrides): - payload = self.recovery() - payload["review_nodes"][0].update(overrides) - self.assertEqual(self.invoke(payload), []) - - def test_repeated_invalid_signals_do_not_rerun_already_revoked_heads(self) -> None: - payload = self.payload() - writes = self.invoke(payload) - self.history(payload, writes) - self.assertEqual(self.invoke(payload), []) - for statuses in payload["statuses_by_sha"].values(): - for status in statuses: - status["state"] = "failure" - self.assertEqual(self.invoke(payload), []) - - def test_approval_recovery_coalesces_and_edited_restoration_recovers_again(self) -> None: - payload = self.recovery() - writes = self.invoke(payload) - self.assertEqual(len(writes), 6) - self.assertTrue(all(len(item["description"]) <= 140 for item in writes[:4])) - self.history(payload, writes) - self.assertEqual(self.invoke(payload), []) - payload["review_nodes"][0]["updatedAt"] = "2026-10-22T01:02:00Z" - restored = self.invoke(payload) - self.assertEqual(len(restored), 6) - self.assertNotEqual(writes[0]["description"], restored[0]["description"]) - - def test_new_revocation_supersedes_old_recovery_even_with_equal_edit_timestamp(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - for statuses in payload["statuses_by_sha"].values(): - statuses[:0] = [ - {**status, "state": "success", "description": "Successful evaluation"} - for status in statuses - ] - payload["review_nodes"][0]["body"] = "" - revoked = self.invoke(payload) - self.assertEqual(len(revoked), 6) - self.history(payload, revoked) - for sha, statuses in payload["statuses_by_sha"].items(): - statuses.extend( - {**item, "creator": {"login": "github-actions[bot]"}} - for item in approved - if item.get("sha") == sha - ) - payload["review_nodes"][0] = ( - self.approval() - ) # Same ID, body and second-precision update time. - self.assertEqual(len(self.invoke(payload)), 6) - - def test_full_review_id_handles_current_and_beyond_safe_integer_ids(self) -> None: - payload = self.recovery() - payload["review_nodes"][0].update(databaseId=None, fullDatabaseId="5614624241") - self.assertEqual(len(self.invoke(payload)), 6) - payload = self.payload() - payload["review_nodes"] = [ - {**self.approval(), "databaseId": None, "fullDatabaseId": "9007199254740992"}, - { - **self.approval(), - "databaseId": None, - "fullDatabaseId": "9007199254740993", - "state": "CHANGES_REQUESTED", - }, - ] - self.assertEqual(len(self.invoke(payload)), 6) - payload["review_nodes"][0]["fullDatabaseId"] = 9007199254740992 - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_publisher_failure_supersedes_recovery_without_observed_revocation(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - for statuses in payload["statuses_by_sha"].values(): - statuses[:0] = [ - {**status, "state": "failure", "description": "Approval missing during publication"} - for status in statuses - ] - # Publisher observed invalid body; review jobs only see its same-second restoration. - # No new review identity/timestamp and no review-handler revocation marker exists. - recovered = self.invoke(payload) - self.assertEqual(len(recovered), 6) - self.history(payload, recovered) - self.assertEqual(self.invoke(payload), []) - - def test_deleted_marker_writer_is_untrusted_but_service_errors_propagate(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - payload["run_errors"] = {"21": 404} - self.assertEqual(len(self.invoke(payload)), 6) - for status in (403, 429, 500): - payload["run_errors"]["21"] = status - with self.subTest(status=status), self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_failed_partial_dispatch_is_retried_before_signals_coalesce(self) -> None: - for authorized in (False, True): - for conclusion in ("failure", "cancelled", "timed_out"): - payload = self.recovery() if authorized else self.payload() - payload["fail_rerun"] = True - # The fixture persists actual pending writes before the simulated API error. - with tempfile.TemporaryDirectory() as directory: - capture = Path(directory) / "pending.json" - payload["capture_writes"] = str(capture) - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - pending = json.loads(capture.read_text()) - self.assertEqual(len(pending), 4) - self.history(payload, pending) - payload.pop("capture_writes") - payload.pop("fail_rerun") - payload["runs_by_id"]["21"]["conclusion"] = conclusion - payload["context"]["runId"] = 22 - retried = self.invoke(payload) - self.assertEqual([item["run_id"] for item in retried if "run_id" in item], [12, 14]) - self.history(payload, retried) - payload["runs_by_id"]["22"] = {**payload["runs_by_id"]["21"], "id": 22} - self.assertEqual(self.invoke(payload), []) - - def test_untrusted_marker_cannot_suppress_recovery(self) -> None: - for field, value in ( - ("workflow_id", 999), - ("event", "pull_request"), - ("repository", {"full_name": "attacker/example"}), - ): - payload = self.recovery() - self.history(payload, self.invoke(payload)) - payload["runs_by_id"]["21"][field] = value - self.assertEqual(len(self.invoke(payload)), 6) - - def test_dismissal_edit_away_and_blocking_review_revoke_even_for_other_signal_actor( - self, - ) -> None: - for changed in ( - {"state": "DISMISSED"}, - {"body": ""}, - {"state": "CHANGES_REQUESTED", "fullDatabaseId": "102"}, - ): - payload = self.payload() - payload["review_nodes"] = [{**self.approval(), **changed}] - self.assertEqual(len(self.invoke(payload)), 6) - payload = self.payload() - payload["review_pages"] = [ - { - "expected_cursor": None, - "nodes": [ - {**self.approval(), "fullDatabaseId": "102", "state": "CHANGES_REQUESTED"} - ], - "pageInfo": {"hasNextPage": True, "endCursor": "more"}, - }, - { - "expected_cursor": "more", - "nodes": [self.approval()], - "pageInfo": {"hasNextPage": False}, - }, - ] - self.assertEqual(len(self.invoke(payload)), 6) - - def test_missing_authoritative_edit_time_rejects_recovery(self) -> None: - payload = self.recovery() - payload["review_nodes"][0]["updatedAt"] = None - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_review_signal_requires_canonical_workflow_identity_before_writes(self) -> None: - for identity in (None, "456", 999): - for path in ( - ".github/workflows/castiron-custom-code.yml", - ".github/workflows/castiron-custom-code.yml@evil.yml", - ): - with self.subTest(identity=identity, path=path): - payload = self.payload() - payload["run"].update({"workflow_id": identity, "path": path}) - payload["context"]["payload"]["workflow_run"] = dict(payload["run"]) - self.assertEqual(self.invoke(payload), []) - - def test_rejects_forged_or_superseded_review_signals(self) -> None: - for field, value in [ - ("path", ".github/workflows/other.yml"), - ("run_attempt", 9), - ("head_sha", "d" * 40), - ("event", "pull_request"), - ]: - with self.subTest(field=field): - payload = self.payload() - payload["run"][field] = value - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - payload = self.payload() - payload["run"]["repository"] = {"full_name": "attacker/example"} - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_head_change_keeps_observed_statuses_revoked_without_stale_reruns(self) -> None: - payload = self.payload() - changed = {**payload["current"], "head": {"sha": "d" * 40}} - payload["current_sequence"] = [payload["current"], changed] - payload["current"] = changed - result = self.invoke(payload) - self.assertEqual(len(result), 4) - self.assertTrue(all(item["state"] == "pending" for item in result)) - self.assertNotIn("d" * 40, {item["sha"] for item in result}) - - def test_queue_change_retries_and_invalidates_new_snapshot(self) -> None: - payload = self.payload() - old = list(payload["queue_nodes"]) - payload["queue_nodes"].append( - { - "headCommit": {"oid": "d" * 40}, - "pullRequest": {"number": 4, "headRefOid": "e" * 40}, - } - ) - payload["runs_by_sha"]["d" * 40] = [ - { - **payload["runs_by_sha"]["c" * 40][0], - "head_sha": "d" * 40, - "id": 16, - } - ] - payload["queue_pages"] = [ - { - "expected_cursor": None, - "nodes": old, - "pageInfo": {"hasNextPage": False}, - } - ] - result = self.invoke(payload) - pending = [item for item in result if item.get("state") == "pending"] - self.assertEqual({item["sha"] for item in pending}, {"a" * 40, "c" * 40, "d" * 40}) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14, 16]) - self.assertTrue(all(item.get("state") == "pending" for item in result[:-3])) - - def test_empty_review_pr_list_uses_authenticated_fork_discovery(self) -> None: - for associated in ([], [{"number": 3}]): - payload = self.payload() - payload["run"].update( - { - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "sdk", - } - ) - payload["associated_pulls"] = associated - payload["fallback_pulls"] = [{"number": 3}] - self.assertEqual(len(self.invoke(payload)), 6) - payload["current"]["head"]["sha"] = "d" * 40 - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": []} - self.assertEqual(self.invoke(payload), []) - - def test_shared_head_revokes_the_run_source_pr(self) -> None: - payload = self.payload() - payload["run"]["pull_requests"] = [] - payload["run"]["head_repository"]["id"] = 101 - payload["current"]["head"].update( - ref="sdk", repo={"id": 101, "full_name": "contributor/example"} - ) - payload["associated_pulls"] = [{"number": number} for number in (3, 4, 5)] - payload["pulls_by_number"] = { - str(number): { - **payload["current"], - "number": number, - "head": {**payload["current"]["head"], **change}, - } - for number, change in ((4, {"ref": "other"}), (5, {"repo": {"id": 102}})) - } - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - - def test_branch_only_merge_signal_requires_current_merge_and_base(self) -> None: - payload = self.payload() - payload["run"]["pull_requests"] = [] - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["fallback_pulls"] = [{"number": 3}] - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - payload["merge_sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - payload["merge_sha"] = "e" * 40 - self.assertEqual(len(self.invoke(payload)), 6) - payload["current"]["base"]["sha"] = "d" * 40 - payload["comparisons"] = { - f"{'d' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - self.assertEqual(self.invoke(payload), []) - - def test_merge_sha_signal_survives_base_advance_and_rejects_wrong_parents(self) -> None: - payload = self.payload() - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["current"]["base"]["sha"] = "d" * 40 - payload["comparisons"] = { - f"{'d' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - self.assertEqual(len(self.invoke(payload)), 6) - payload["comparisons"][f"{'b' * 40}...{'d' * 40}"] = { - "merge_base_commit": {"sha": "f" * 40} - } - self.assertEqual(self.invoke(payload), []) - payload["merge_commit"]["parents"].reverse() - self.assertEqual(self.invoke(payload), []) - - def test_trusted_policy_decision_controls_revocation(self) -> None: - payload = self.payload() - payload["requires_human"] = False - self.assertEqual(self.invoke(payload), []) - payload["requires_human"] = True - self.assertEqual(len(self.invoke(payload)), 6) - - def test_policy_failure_or_malformed_decision_cannot_write_statuses(self) -> None: - for result in ({"policy_error": True}, {"requires_human": "false"}): - with self.subTest(result=result), tempfile.TemporaryDirectory() as directory: - payload = {**self.payload(), **result} - capture = Path(directory) / "writes.json" - payload["capture_writes"] = str(capture) - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - self.assertEqual(json.loads(capture.read_text()), []) - - def test_revoked_go_ratchet_activation_invalidates_pr_and_queue_statuses(self) -> None: - for changed in ({"state": "DISMISSED"}, {"body": ""}): - with self.subTest(changed=changed): - payload = self.payload() - payload["policies_by_sha"]["a" * 40] = { - **payload["policies_by_sha"]["b" * 40], - "ratchet": "go", - } - payload["review_nodes"] = [{**self.approval(), **changed}] - writes = self.invoke(payload) - self.assertCountEqual( - [(item["sha"], item["state"]) for item in writes if "state" in item], - [("a" * 40, "pending")] * 2 + [("c" * 40, "pending")] * 2, - ) - self.assertCountEqual( - [item["run_id"] for item in writes if "run_id" in item], [12, 14] - ) - - def test_closed_or_superseded_review_is_a_no_op(self) -> None: - for changed in ({"state": "closed"}, {"head": {"sha": "f" * 40}}): - payload = self.payload() - payload["current"].update(changed) - self.assertEqual(self.invoke(payload), []) - - def test_null_queue_association_does_not_block_revocation(self) -> None: - payload = self.payload() - payload["queue_nodes"].insert(0, {"headCommit": {"oid": "f" * 40}, "pullRequest": None}) - result = self.invoke(payload) - self.assertEqual(len(result), 6) - self.assertNotIn("f" * 40, {item.get("sha") for item in result}) - payload["queue_nodes"] = payload["queue_nodes"][:1] - self.assertEqual(len(self.invoke(payload)), 3) - - def test_rerun_requires_the_reviewed_pr_not_just_shared_head(self) -> None: - payload = self.payload() - correct = payload["runs_by_sha"]["a" * 40][0] - main = { - **payload["current"], - "number": 4, - "base": {**payload["current"]["base"], "ref": "main"}, - } - payload["pulls_by_number"] = {"4": main} - other: dict[str, Any] = { - **correct, - "id": 30, - "pull_requests": [{"number": 4, "head": {"sha": "a" * 40}}], - } - payload["runs_by_sha"]["a" * 40].append(other) - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - # Empty fork associations may rerun only if canonical discovery selects this next PR. - fork: dict[str, Any] = { - **correct, - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "sdk", - } - payload["fallback_pulls"] = [{"number": 3}] - payload["runs_by_sha"]["a" * 40] = [fork, other] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - association_cases: tuple[list[dict[str, Any]], ...] = ( - [], - [*correct["pull_requests"], *other["pull_requests"]], - ) - for associations in association_cases: - payload["associated_pulls"] = [{"number": 3}, {"number": 4}] - payload["runs_by_sha"]["a" * 40] = [{**fork, "pull_requests": associations}, other] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [14]) - self.assertTrue(all(item["state"] == "pending" for item in result[:4])) - - def test_unprepared_queue_entry_still_revokes_pr_statuses(self) -> None: - payload = self.payload() - payload["queue_nodes"][0]["headCommit"] = None - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending", None]) - self.assertEqual(result[-1]["run_id"], 12) - - def test_unrelated_review_does_not_restart_reporter(self) -> None: - payload = self.payload() - payload["files"] = [{"filename": "src/client.ts"}] - self.assertEqual(self.invoke(payload), []) - - def test_no_success_or_foreign_run_can_be_published(self) -> None: - payload = self.payload() - payload["runs_by_sha"]["a" * 40][0]["repository"] = {"full_name": "other/repo"} - result = self.invoke(payload) - self.assertEqual( - [item["run_id"] for item in result if item.get("operation") == "rerun"], [14] - ) - self.assertTrue(all(item.get("state") != "success" for item in result)) - - def main_advance_payload(self) -> dict[str, Any]: - payload = self.payload() - main = "d" * 40 - run = { - **source_run(main, "push"), - "head_branch": "main", - "head_repository": {"full_name": "openai/example"}, - } - payload["run"] = run - payload["main_sha"] = main - payload["context"]["payload"]["workflow_run"] = dict(run) - payload["runs_by_sha"]["c" * 40][0]["head_branch"] = "gh-readonly-queue/next/pr-3-example" - return payload - - def invoke_main_advance(self, payload: dict[str, Any]) -> list[dict[str, Any]]: - return run_publisher("Re-evaluate live next queue after main advances", payload) - - def test_main_advance_revokes_success_before_normal_queue_rerun(self) -> None: - result = self.invoke_main_advance(self.main_advance_payload()) - self.assertEqual([item.get("state") for item in result[:2]], ["pending", "pending"]) - self.assertEqual({item["sha"] for item in result[:2]}, {"c" * 40}) - self.assertTrue(all(item["context"].startswith("Castiron next /") for item in result[:2])) - self.assertEqual([item["run_id"] for item in result[2:]], [14]) - self.assertNotIn("success", [item.get("state") for item in result]) - - def test_old_main_push_does_not_invalidate_newer_qualified_candidates(self) -> None: - payload = self.main_advance_payload() - payload["main_sha"] = "e" * 40 - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_main_advance_rejects_foreign_or_non_main_signals(self) -> None: - for field, value in ( - ("head_repository", {"full_name": "other/repo"}), - ("repository", {"full_name": "other/repo"}), - ("head_branch", "next"), - ("event", "pull_request"), - ): - with self.subTest(field=field): - payload = self.main_advance_payload() - payload["run"][field] = value - with self.assertRaises(subprocess.CalledProcessError): - self.invoke_main_advance(payload) - payload = self.main_advance_payload() - payload["run"]["workflow_id"] = 999 - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_main_advance_invalidates_every_queue_page_before_dispatch(self) -> None: - payload = self.main_advance_payload() - payload["queue_pages"] = [ - { - "expected_cursor": None, - "nodes": [{"headCommit": None}, {"headCommit": {"oid": "c" * 40}}], - "pageInfo": {"hasNextPage": True, "endCursor": "second"}, - }, - { - "expected_cursor": "second", - "nodes": [{"headCommit": {"oid": "e" * 40}}, {"headCommit": None}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"c" * 40, "e" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [14]) - - def test_main_advance_still_rejects_malformed_non_null_heads(self) -> None: - for head in ({}, {"oid": None}, {"oid": "g" * 40}): - with self.subTest(head=head): - payload = self.main_advance_payload() - payload["queue_nodes"][0]["headCommit"] = head - with self.assertRaises(subprocess.CalledProcessError): - self.invoke_main_advance(payload) - - def test_main_advance_does_not_rerun_wrong_queue_or_foreign_workflows(self) -> None: - for field, value in ( - ("repository", {"full_name": "other/repo"}), - ("workflow_id", 999), - ("head_branch", "gh-readonly-queue/main/pr-3-example"), - ): - with self.subTest(field=field): - payload = self.main_advance_payload() - payload["runs_by_sha"]["c" * 40][0][field] = value - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending"]) - - def test_main_advance_leaves_newer_running_evaluation_to_publish(self) -> None: - payload = self.main_advance_payload() - runs = payload["runs_by_sha"]["c" * 40] - runs.append({**runs[0], "id": 15, "status": "in_progress"}) - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending"]) - - def test_main_advance_with_empty_queue_has_no_writes(self) -> None: - payload = self.main_advance_payload() - payload["queue_nodes"] = [] - payload["queue_absent"] = True - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_signal_is_unprivileged_and_writers_serialize(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - self.assertIn("pull_request_review:\n types: [submitted, edited, dismissed]", producer) - signal = producer.split(" queue-signal:\n", 1)[1].split(" candidate-tests:\n", 1)[0] - self.assertIn("permissions: {}", signal) - self.assertIn("push:\n branches: [main]", producer) - main_change = ( - WORKFLOW.read_text().split(" main-change:\n", 1)[1].split(" compute:\n", 1)[0] - ) - self.assertNotIn("checkout@", main_change) - self.assertNotIn("download-artifact", main_change) - self.assertNotIn("checkout@", signal) - handler = WORKFLOW.read_text() - revoke = handler.split(" review-change:\n", 1)[1].split(" compute:\n", 1)[0] - self.assertNotIn("checkout@", revoke) - self.assertNotIn("ref: ${{ github.event", revoke) - self.assertIn("path.join(root, 'custom_code_policy.py'), 'review-required'", revoke) - self.assertNotIn("download-artifact", revoke) - self.assertEqual( - handler.count("group: castiron-budget-writes-${{ github.repository_id }}"), 3 - ) - - def test_direct_head_review_discovery_rejects_shared_commit_from_other_sources( - self, - ) -> None: - for field, value in ( - ("id", 99), - ("full_name", "other/example"), - ("ref", "other-branch"), - ): - with self.subTest(field=field): - payload = self.payload() - payload["run"]["pull_requests"] = [] - head = source_head("a" * 40) - if field == "ref": - head[field] = value - else: - head["repo"][field] = value - other = {**payload["current"], "number": 4, "head": head} - payload["pulls_by_number"] = {"4": other} - for discovery in ("associated_pulls", "fallback_pulls"): - payload["associated_pulls"] = [] - payload[discovery] = [{"number": 4}, {"number": 3}] - result = self.invoke(payload) - self.assertEqual(len(result), 6) - self.assertEqual( - [item["run_id"] for item in result if "run_id" in item], [12, 14] - ) - - def test_foreign_only_associations_retry_revocation_and_reporter_discovery(self) -> None: - for explicit in (False, True): - with self.subTest(explicit=explicit): - payload = self.payload() - payload["run"]["pull_requests"] = [{"number": 4}] if explicit else [] - payload["runs_by_sha"]["a" * 40][0]["pull_requests"] = [] - payload["associated_pulls"] = [{"number": 4}] - payload["fallback_pulls"] = [{"number": 3}] - own = payload["current"] - foreign: dict[str, Any] = { - **own, - "number": 4, - "head": {**own["head"], "ref": "other"}, - } - payload["pulls_by_number"] = {"4": foreign} - payload["review_nodes"] = [{**self.approval(), "state": "DISMISSED"}] - writes = self.invoke(payload) - self.assertEqual( - [w["state"] for w in writes if "state" in w], - [] if explicit else ["pending"] * 4, - ) - self.assertEqual( - [w["run_id"] for w in writes if "run_id" in w], [] if explicit else [12, 14] - ) - # A fallback reached through unrelated commit associations is still branch-only: - # an old synthetic merge must not gain the association-only exemption. - payload["run"]["pull_requests"] = [] - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["merge_sha"] = "d" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - payload["pulls_by_number"]["4"]["head"]["sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - def test_merge_review_filters_foreign_sources_before_branch_fallback(self) -> None: - for field, value in (("ref", "other"), ("id", 99), ("full_name", "other/example")): - for explicit in (False, True): - with self.subTest(field=field, explicit=explicit): - payload = self.payload() - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["run"]["pull_requests"] = [{"number": 4}] if explicit else [] - payload["associated_pulls"] = [{"number": 4}] - payload["fallback_pulls"] = [{"number": 3}] - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - head = source_head("a" * 40) - if field == "ref": - head[field] = value - else: - head["repo"][field] = value - payload["pulls_by_number"] = { - "4": {**payload["current"], "number": 4, "head": head} - } - writes = self.invoke(payload) - self.assertEqual( - [w["state"] for w in writes if "state" in w], - [] if explicit else ["pending"] * 4, - ) - self.assertEqual( - [w["run_id"] for w in writes if "run_id" in w], - [] if explicit else [12, 14], - ) - # Filtering a foreign association must not exempt fallback from freshness. - payload["merge_sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - def test_review_requires_complete_source_identity(self) -> None: - for overrides in ( - {"head_repository": None}, - {"head_repository": {"id": 10}}, - {"head_repository": {"full_name": "contributor/example"}}, - {"head_branch": None}, - {"head_branch": "refs/pull/3/merge"}, - ): - for merge in (False, True): - with self.subTest(overrides=overrides, merge=merge): - payload = self.payload() - if merge: - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["merge_commit"] = { - "parents": [{"sha": "b" * 40}, {"sha": "a" * 40}] - } - payload["run"].update(overrides) - self.assertEqual(self.invoke(payload), []) - - def test_merge_sha_signal_requires_exact_live_merge_and_parents(self) -> None: - payload = self.payload() - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - self.assertEqual(len(self.invoke(payload)), 6) - payload["merge_commit"]["parents"].reverse() - self.assertEqual(self.invoke(payload), []) - - def test_historical_merge_revocation_after_base_advance(self) -> None: - for discovery in ("run", "commit", "branch"): - for revoked in ({"state": "DISMISSED"}, {"body": ""}): - with self.subTest(discovery=discovery, revoked=revoked): - payload = self.payload() - payload["review_nodes"] = [{**self.approval(), **revoked}] - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["current"]["base"]["sha"] = "f" * 40 - payload["comparisons"] = { - f"{'f' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - payload["merge_sha"] = "d" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - if discovery != "run": - payload["run"]["pull_requests"] = [] - payload["associated_pulls"] = ( - [{"number": 3}] if discovery == "commit" else [] - ) - payload["fallback_pulls"] = [{"number": 3}] - result = self.invoke(payload) - if discovery == "branch": - self.assertEqual(result, []) - else: - self.assertEqual( - [item.get("state") for item in result[:4]], ["pending"] * 4 - ) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_rerun_ignores_same_sha_main_from_another_source(self) -> None: - for changes in ( - {"ref": "other"}, - {"repo": {**source_head("a" * 40)["repo"], "id": 11}}, - {"repo": {**source_head("a" * 40)["repo"], "full_name": "other/repo"}}, - {"repo": None}, - ): - with self.subTest(changes=changes): - payload = self.payload() - reporter = payload["runs_by_sha"]["a" * 40][0] - reporter["pull_requests"] = [] - current = payload["current"] - main = { - **current, - "number": 4, - "head": {**current["head"], **changes}, - "base": {**current["base"], "ref": "main"}, - } - payload["pulls_by_number"] = {"4": main} - payload["associated_pulls"] = [{"number": 3}, {"number": 4}] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - - def test_rerun_rejects_missing_reporter_source_identity(self) -> None: - for changes in ( - {"head_repository": None}, - {"head_repository": {"full_name": "contributor/example"}}, - {"head_repository": {"id": 10}}, - {"head_branch": ""}, - ): - with self.subTest(changes=changes): - payload = self.payload() - payload["runs_by_sha"]["a" * 40][0].update(changes) - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [14]) - self.assertTrue(all(item["state"] == "pending" for item in result[:4])) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_status_publication.py b/scripts/castiron/test_custom_code_status_publication.py deleted file mode 100644 index 0aca402217..0000000000 --- a/scripts/castiron/test_custom_code_status_publication.py +++ /dev/null @@ -1,782 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Execute the trusted workflow against offline GitHub state.""" - -from __future__ import annotations - -import json -import shutil -import unittest -from typing import Any - -import custom_code_policy as policy -from custom_code_test_support import run_publisher, source_head, source_run - - -@unittest.skipUnless(shutil.which("node"), "Node is needed to execute the status-publisher fixture") -class StatusPublisherTests(unittest.TestCase): - def publish( - self, - *, - event_name: str = "pull_request", - head_changed: bool = False, - base_changed: bool = False, - no_result: bool = False, - failed_budget: bool = False, - fallback_pulls: list[dict[str, int]] | None = None, - associated_pulls: list[dict[str, int]] | None = None, - run_overrides: dict[str, Any] | None = None, - previous_statuses: list[dict[str, Any]] | None = None, - branch: str = "main", - evaluated_branch: str | None = None, - policy_changed: bool = False, - authenticated_main_sha: str | None = None, - bootstrap: str = "false", - main_snapshot: str = "", - policy_sha: str | None = None, - selected_policy_matches: bool = True, - queue_absent: bool = False, - queue_pages: list[dict[str, Any]] | None = None, - retarget_during_publication: bool = False, - source_changes_during_publication: dict[str, Any] | None = None, - same_head_next_pr: bool = False, - policy_reviews: list[dict[str, Any]] | None = None, - reviews: list[dict[str, Any]] | None = None, - review_nodes: list[dict[str, Any]] | None = None, - review_pages: list[dict[str, Any]] | None = None, - policy_pull: dict[str, Any] | None = None, - missing_review_contract: bool = False, - review_contract: dict[str, Any] | None = None, - ) -> list[dict[str, Any]]: - base, head = "a" * 40, "b" * 40 - run = source_run(head, event_name) - if event_name == "merge_group": - run["head_branch"] = f"gh-readonly-queue/{branch}/pr-7-example" - payload: dict[str, Any] = { - "reviews": reviews or [], - "review_nodes": review_nodes - if review_nodes is not None - else [ - { - "fullDatabaseId": str(review["id"]), - "state": review["state"], - "body": review.get("body"), - "commit": {"oid": review.get("commit_id")}, - "author": { - "login": review["user"]["login"], - "__typename": review["user"]["type"], - }, - "authorAssociation": review.get("author_association"), - } - for review in reviews or [] - ], - "review_pages": review_pages, - "main_sha": "d" * 40 - if policy_changed - else (policy_sha or ("c" * 40 if branch == "main" and base_changed else base)), - "queue_absent": queue_absent, - "queue_pages": queue_pages, - "fallback_pulls": fallback_pulls, - "associated_pulls": associated_pulls, - "previous_statuses": previous_statuses or [], - "context": { - "eventName": "workflow_run", - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - "runId": 123, - "payload": { - "workflow_run": run, - }, - }, - "run": {**run, **(run_overrides or {})}, - "current": { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head("c" * 40 if head_changed else head), - "base": { - "sha": "c" * 40 if base_changed else base, - "ref": branch, - "repo": {"full_name": "openai/example"}, - }, - }, - "env": { - "POLICY_REVIEWS": json.dumps(policy_reviews or []), - "HUMAN_REVIEW_RULES": "" - if missing_review_contract - else json.dumps( - policy.HUMAN_REVIEW_RULES if review_contract is None else review_contract - ), - "SELECTED_BRANCH": branch, - "SELECTED_BOOTSTRAP": bootstrap, - "MAIN_SNAPSHOT": main_snapshot, - "SELECTED_BASE_SHA": base, - "SELECTED_HEAD_SHA": head, - "SELECTED_POLICY_SHA": (policy_sha or base) - if selected_policy_matches - else "f" * 40, - "TARGET_BRANCH": "" if no_result else (evaluated_branch or branch), - "POLICY_SHA": "" if no_result else (policy_sha or base), - "BASE_SHA": "" if no_result else base, - "HEAD_SHA": "" if no_result else head, - "ISOLATION_RESULT": "success", - "BUDGET_RESULT": "failure" if failed_budget else "success", - "PUBLISH_ATTEMPT": "1", - }, - } - payload["env"]["AUTHENTICATED_MAIN_SHA"] = ( - payload["main_sha"] if authenticated_main_sha is None else authenticated_main_sha - ) - if policy_pull: - payload["pulls_by_number"] = {policy_pull["number"]: policy_pull} - if same_head_next_pr: - current = payload["current"] - payload["pulls_by_number"] = { - 3: current, - 4: {**current, "number": 4, "base": {**current["base"], "ref": "next"}}, - } - if retarget_during_publication: - current = payload["current"] - payload["current_sequence"] = [ - current, - {**current, "base": {**current["base"], "ref": "other"}}, - ] - if source_changes_during_publication is not None: - current = payload["current"] - payload["current_sequence"] = [ - current, - {**current, "head": {**current["head"], **source_changes_during_publication}}, - ] - return run_publisher("Publish exact-head statuses after checking freshness", payload) - - def test_live_human_review_predicate_matches_selector(self) -> None: - head = "b" * 40 - approved = { - "id": 10, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - cases = [ - [approved], - [], - [{**approved, "state": "DISMISSED"}], - [approved, {**approved, "id": 11, "state": "CHANGES_REQUESTED"}], - [approved, {**approved, "id": 11, "state": "COMMENTED"}], - [{**approved, "commit_id": "f" * 40}], - [{**approved, "user": {"login": "author", "type": "User"}}], - [{**approved, "user": {"login": "bot", "type": "Bot"}}], - [{**approved, "author_association": "NONE"}], - ] - for body in [ - "", - "Looks good to me", - policy.HUMAN_REVIEW_RULES["attestation_prefix"] + "f" * 40, - "> " + policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - "", - "Automated source review.", - "Source-only review; I did not execute candidate code or workflows.", - "This does not substitute for human approval.", - "This is not a human approving review.", - ]: - cases.append([{**approved, "body": body}]) - for reviews in cases: - with self.subTest(reviews=reviews): - expected = ( - "success" if policy.has_human_approval(reviews, "author", head) else "failure" - ) - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], [expected, expected]) - - def test_final_publisher_preserves_approval_across_comments(self) -> None: - head = "b" * 40 - approval = { - "id": 10, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - comment = {**approval, "id": 11, "state": "COMMENTED", "body": "A follow-up comment"} - cases = [ - ([approval, comment], "success"), - ([comment, approval], "success"), - ([comment], "failure"), - ([{**approval, "state": "DISMISSED"}, comment], "failure"), - ([{**approval, "body": "Attestation removed"}, comment], "failure"), - ( - [ - approval, - {**approval, "id": 12, "state": "CHANGES_REQUESTED"}, - {**comment, "id": 13}, - ], - "failure", - ), - ( - [ - {**approval, "state": "CHANGES_REQUESTED"}, - {**approval, "id": 12}, - {**comment, "id": 13}, - ], - "success", - ), - ] - for reviews, expected in cases: - with self.subTest(reviews=reviews): - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], [expected] * 2) - - def test_final_publisher_preserves_review_identity_precision(self) -> None: - head = "b" * 40 - approval = { - "id": 9007199254740992, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - replacement = {**approval, "id": 9007199254740993, "state": "CHANGES_REQUESTED"} - for reviews in ([approval, replacement], [replacement, approval]): - with self.subTest(reviews=reviews): - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], ["failure"] * 2) - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=[{**replacement, "state": "APPROVED"}, {**approval, "state": "DISMISSED"}], - ) - self.assertEqual([w["state"] for w in writes], ["success"] * 2) - - def test_final_publisher_review_pages_and_malformed_identity(self) -> None: - head = "b" * 40 - approval = { - "fullDatabaseId": "9007199254740992", - "state": "APPROVED", - "commit": {"oid": head}, - "author": {"login": "human", "__typename": "User"}, - "authorAssociation": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - - def publish(**kwargs: Any) -> list[str]: - return [ - w["state"] - for w in self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - **kwargs, - ) - ] - - self.assertEqual(publish(review_nodes=[approval]), ["success"] * 2) - for identity in (None, "", "1.5", "-1", "0", 9007199254740992): - with self.subTest(identity=identity): - self.assertEqual( - publish(review_nodes=[{**approval, "fullDatabaseId": identity}]), - ["failure"] * 2, - ) - pages: list[dict[str, Any]] = [ - { - "expected_cursor": None, - "nodes": [approval], - "pageInfo": {"hasNextPage": True, "endCursor": "page2"}, - }, - { - "expected_cursor": "page2", - "nodes": [{**approval, "fullDatabaseId": "9007199254740993", "state": "DISMISSED"}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - self.assertEqual(publish(review_pages=pages), ["failure"] * 2) - self.assertEqual( - publish( - review_pages=[{**pages[0], "pageInfo": {"hasNextPage": True, "endCursor": None}}] - ), - ["failure"] * 2, - ) - - def test_next_publication_binds_authenticated_main(self) -> None: - # The reviewed checker pin may remain old. The evaluation's main snapshot - # must still be current so an old publisher cannot undo invalidation. - for event in ("pull_request", "merge_group"): - for authenticated_main, expected in ( - ("a" * 40, "failure"), - ("d" * 40, "success"), - ("", "failure"), - ("invalid", "failure"), - ): - with self.subTest(event=event, authenticated_main=authenticated_main): - writes = self.publish( - event_name=event, - branch="next", - policy_changed=True, - authenticated_main_sha=authenticated_main, - ) - self.assertEqual([item["state"] for item in writes], [expected] * 2) - - def test_main_snapshot_publication_rechecks_live_main(self) -> None: - for event in ("pull_request", "merge_group"): - for moved in (False, True): - with self.subTest(event=event, moved=moved): - writes = self.publish( - event_name=event, - branch="next", - policy_sha="e" * 40, - main_snapshot="e" * 40, - policy_changed=moved, - ) - self.assertEqual( - [w["state"] for w in writes], - ["failure", "failure"] if moved else ["success", "success"], - ) - - def test_bootstrap_rechecks_authenticated_main(self) -> None: - for event in ("pull_request", "merge_group"): - for bootstrap, changed, expected in ( - ("true", False, "success"), - ("true", True, "failure"), - ("false", True, "success"), - ("", False, "failure"), - ("invalid", False, "failure"), - ): - with self.subTest(event=event, bootstrap=bootstrap, changed=changed): - writes = self.publish( - event_name=event, - branch="next", - bootstrap=bootstrap, - policy_changed=changed, - ) - self.assertEqual([item["state"] for item in writes], [expected] * 2) - - def test_freshness_is_rechecked_after_review_pagination(self) -> None: - head = "b" * 40 - approval = { - "fullDatabaseId": "1", - "state": "APPROVED", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - "author": {"login": "human", "__typename": "User"}, - "authorAssociation": "MEMBER", - "commit": {"oid": head}, - } - current: dict[str, Any] = { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "sha": "a" * 40, "repo": {"full_name": "openai/example"}}, - } - advanced = {**current, "base": {**current["base"], "sha": "c" * 40}} - changed_source = {**current, "head": {**current["head"], "ref": "other-source"}} - for event, mutation, expected in ( - ("pull_request", {}, ["success"] * 2), - ("merge_group", {}, ["success"] * 2), - ("pull_request", {"main_sha": "d" * 40}, ["failure"] * 2), - ("merge_group", {"main_sha": "d" * 40}, ["failure"] * 2), - ("pull_request", {"current": advanced}, ["failure"] * 2), - ("merge_group", {"current": advanced}, ["failure"] * 2), - ("merge_group", {"queue_absent": True}, ["failure"] * 2), - ("pull_request", {"current": changed_source}, []), - ): - with self.subTest(event=event, mutation=mutation): - result = self.publish( - event_name=event, - branch="next", - bootstrap="true", - policy_reviews=[{"number": 3, "head_sha": head}], - review_pages=[ - { - "expected_cursor": None, - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "last"}, - }, - { - "expected_cursor": "last", - "nodes": [approval], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - "after_read": mutation, - }, - ], - ) - self.assertEqual([item["state"] for item in result], expected) - - def test_queue_policy_approval_dismissal_after_selection_fails(self) -> None: - candidate = "e" * 40 - for queue_head in (source_head(candidate), {"sha": candidate}): - with self.subTest(queue_head=queue_head): - pull: dict[str, Any] = { - "number": 7, - "user": {"login": "author"}, - "state": "open", - "head": queue_head, - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - approved = { - "id": 1, - "state": "APPROVED", - "commit_id": candidate, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + candidate, - } - for state in ("APPROVED", "DISMISSED", "CHANGES_REQUESTED"): - with self.subTest(state=state): - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=pull, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[{**approved, "state": state}], - ) - self.assertEqual( - [w["state"] for w in writes], - ["success" if state == "APPROVED" else "failure"] * 2, - ) - # Ordinary automated approvals may keep the queue live, but cannot replace - # the independently required human policy approval. - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=pull, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[ - {**approved, "state": "DISMISSED"}, - { - **approved, - "id": 2, - "user": {"login": "automated", "type": "User"}, - "body": "", - }, - ], - ) - self.assertEqual([w["state"] for w in writes], ["failure", "failure"]) - - def test_queue_policy_constituent_changes_fail_closed(self) -> None: - candidate = "e" * 40 - for queue_head in (source_head(candidate), {"sha": candidate}): - with self.subTest(queue_head=queue_head): - pull: dict[str, Any] = { - "number": 7, - "user": {"login": "author"}, - "state": "open", - "head": queue_head, - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - approved = { - "id": 1, - "state": "APPROVED", - "commit_id": candidate, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + candidate, - } - for changed in [ - {**pull, "state": "closed"}, - {**pull, "head": {"sha": "f" * 40}}, - {**pull, "base": {**pull["base"], "ref": "main"}}, - ]: - with self.subTest(changed=changed): - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=changed, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[approved], - ) - self.assertEqual([w["state"] for w in writes], ["failure", "failure"]) - - def test_missing_policy_review_contract_fails_closed(self) -> None: - self.assertEqual( - [w["state"] for w in self.publish(branch="next", missing_review_contract=True)], - ["failure", "failure"], - ) - - def test_legacy_review_contract_fails_closed_only_on_next(self) -> None: - legacy = { - key: value - for key, value in policy.HUMAN_REVIEW_RULES.items() - if key != "attestation_prefix" - } - legacy["version"] = 1 - for branch, expected in (("next", "failure"), ("main", "success")): - with self.subTest(branch=branch): - self.assertEqual( - [w["state"] for w in self.publish(branch=branch, review_contract=legacy)], - [expected, expected], - ) - - def test_statuses_recheck_source_identity_immediately_before_writing(self) -> None: - source = source_head("b" * 40) - self.assertEqual(len(self.publish(source_changes_during_publication={})), 2) - for changes in ( - {"ref": "other"}, - {"repo": {**source["repo"], "id": 11}}, - {"repo": {**source["repo"], "full_name": "other/example"}}, - {"repo": None}, - ): - with self.subTest(changes=changes): - self.assertEqual(self.publish(source_changes_during_publication=changes), []) - - def test_statuses_reject_a_run_from_a_different_source(self) -> None: - for source in ( - {"head_branch": "other"}, - {"head_repository": {"id": 11, "full_name": "other/repo"}}, - {"head_repository": None}, - ): - with self.subTest(source=source): - self.assertEqual(self.publish(run_overrides=source), []) - - def test_statuses_attach_to_candidate_not_main(self) -> None: - for event in ("pull_request", "merge_group"): - with self.subTest(event=event): - results = self.publish(event_name=event) - self.assertEqual(len(results), 2) - self.assertTrue( - all(r["sha"] == "b" * 40 and r["state"] == "success" for r in results) - ) - - def test_next_cannot_overwrite_main_results(self) -> None: - for main_event in ("pull_request", "merge_group"): - main = self.publish(event_name=main_event, failed_budget=True) - self.assertEqual([r["state"] for r in main], ["success", "failure"]) - for event in ("pull_request", "merge_group"): - with self.subTest(main_event=main_event, event=event): - # Both evaluations use the same commit. A next success - # must not replace the main target's required failure. - result = self.publish(event_name=event, branch="next", previous_statuses=main) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - self.assertTrue( - {r["context"] for r in result}.isdisjoint(r["context"] for r in main) - ) - - def test_foreign_only_associations_retry_source_branch(self) -> None: - foreign = { - "number": 4, - "state": "open", - "head": {**source_head("b" * 40), "ref": "other"}, - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - for explicit in (False, True): - with self.subTest(explicit=explicit): - writes = self.publish( - run_overrides={"pull_requests": [{"number": 4}] if explicit else []}, - associated_pulls=[{"number": 4}], - fallback_pulls=[{"number": 3}], - policy_pull=foreign, - ) - self.assertEqual( - [w["state"] for w in writes], [] if explicit else ["success", "success"] - ) - - def test_same_head_next_pr_does_not_block_required_main_statuses(self) -> None: - for numbers in ([3, 4], [4, 3]): - with self.subTest(numbers=numbers): - results = self.publish( - run_overrides={"pull_requests": []}, - fallback_pulls=[{"number": number} for number in numbers], - same_head_next_pr=True, - failed_budget=True, - ) - self.assertEqual([r["state"] for r in results], ["success", "failure"]) - self.assertTrue(all(r["sha"] == "b" * 40 for r in results)) - next_results = self.publish( - branch="next", - run_overrides={"pull_requests": [{"number": 4}]}, - same_head_next_pr=True, - ) - self.assertEqual([r["state"] for r in next_results], ["success", "success"]) - self.assertTrue(all(r["context"].startswith("Castiron next /") for r in next_results)) - - def test_next_queue_approved_pin_survives_main_advancing(self) -> None: - for changed in (False, True): - with self.subTest(policy_changed=changed): - results = self.publish( - event_name="merge_group", branch="next", policy_changed=changed - ) - self.assertEqual([r["state"] for r in results], ["success", "success"]) - - def test_next_pin_receipt_and_target_freshness(self) -> None: - for event in ("pull_request", "merge_group"): - cases: tuple[dict[str, Any], ...] = ( - {"selected_policy_matches": False}, - {"no_result": True}, - {"base_changed": True}, - ) - for kwargs in cases: - with self.subTest(event=event, kwargs=kwargs): - results = self.publish(event_name=event, branch="next", **kwargs) - self.assertEqual([r["state"] for r in results], ["failure", "failure"]) - results = self.publish(event_name=event, branch="next", policy_sha="e" * 40) - self.assertEqual([r["state"] for r in results], ["success", "success"]) - results = self.publish(event_name="merge_group", branch="next", queue_absent=True) - self.assertEqual([r["state"] for r in results], ["failure", "failure"]) - - def test_main_statuses_bind_target_policy_and_queue_membership(self) -> None: - for event in ("pull_request", "merge_group"): - with self.subTest(event=event): - stale = self.publish(event_name=event, evaluated_branch="next") - self.assertEqual([r["state"] for r in stale], ["failure", "failure"]) - for policy_changed, queue_absent in ((True, False), (False, True)): - with self.subTest(policy_changed=policy_changed, queue_absent=queue_absent): - result = self.publish( - event_name="merge_group", - policy_changed=policy_changed, - queue_absent=queue_absent, - ) - self.assertEqual([r["state"] for r in result], ["failure", "failure"]) - self.assertEqual(self.publish(retarget_during_publication=True), []) - - def test_next_fork_and_unsupported_targets(self) -> None: - result = self.publish( - branch="next", run_overrides={"pull_requests": []}, fallback_pulls=[{"number": 3}] - ) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - for event in ("pull_request", "merge_group"): - for branch in ("Next", "Main", "release", "next-other"): - with self.subTest(event=event, branch=branch): - self.assertEqual(self.publish(event_name=event, branch=branch), []) - - def test_main_queue_membership_paginates(self) -> None: - pages: list[dict[str, Any]] = [ - { - "expected_cursor": None, - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "page2"}, - }, - { - "expected_cursor": "page2", - "nodes": [{"headCommit": {"oid": "b" * 40}}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - result = self.publish(event_name="merge_group", queue_pages=pages) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - - def test_fork_statuses_with_no_commit_association(self) -> None: - options: dict[str, Any] = { - "run_overrides": {"pull_requests": []}, - "fallback_pulls": [{"number": 3}], - } - results = self.publish(**options) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["sha"] == "b" * 40 and r["state"] == "success" for r in results)) - self.assertEqual(self.publish(**options, head_changed=True), []) - self.assertTrue( - all(r["state"] == "failure" for r in self.publish(**options, no_result=True)) - ) - self.assertEqual(self.publish(run_overrides={"pull_requests": []}, fallback_pulls=[]), []) - self.assertEqual( - self.publish( - run_overrides={"pull_requests": []}, - fallback_pulls=[{"number": 3}, {"number": 4}], - ), - [], - ) - - def test_stale_pr_head_is_not_published(self) -> None: - self.assertEqual(self.publish(head_changed=True), []) - - def test_pr_snapshot_survives_main_advancing(self) -> None: - results = self.publish(base_changed=True) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "success" for r in results)) - self.assertTrue(all("a" * 12 in r["description"] for r in results)) - - def test_queue_still_rejects_main_advancing(self) -> None: - results = self.publish(event_name="merge_group", base_changed=True) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "failure" for r in results)) - - def test_missing_evaluation_publishes_actionable_failures(self) -> None: - for event in ("pull_request", "merge_group"): - results = self.publish(event_name=event, no_result=True, base_changed=True) - self.assertEqual(len(results), 2) - for result in results: - self.assertEqual(result["state"], "failure") - self.assertIn("inspect the trusted run", result["description"]) - self.assertTrue(result["target_url"].endswith("/actions/runs/123")) - - def test_main_rejects_mismatched_selector_receipt(self) -> None: - results = self.publish(selected_policy_matches=False) - self.assertTrue(results) - self.assertTrue(all(result["state"] != "success" for result in results)) - - def test_lookalike_workflow_cannot_publish(self) -> None: - self.assertEqual( - self.publish( - run_overrides={ - "path": ".github/workflows/castiron-custom-code.yml@evil.yml", - "workflow_id": 789, - } - ), - [], - ) - - def test_superseded_or_wrong_source_run_cannot_publish(self) -> None: - for overrides in ( - {"run_attempt": 2}, - {"head_sha": "c" * 40}, - {"event": "push"}, - {"path": "other.yml"}, - ): - with self.subTest(overrides=overrides): - self.assertEqual(self.publish(run_overrides=overrides), []) - - def test_independent_check_failures_are_preserved(self) -> None: - results = self.publish(failed_budget=True) - self.assertEqual([r["state"] for r in results], ["success", "failure"]) - - def test_older_evaluation_cannot_overwrite_newer_failure(self) -> None: - for order in ("124:1:125:1", "123:2:125:1", "123:1:125:1", "123:1:123:2"): - with self.subTest(order=order): - previous = { - "context": "Castiron / custom-code budget", - "creator": {"login": "github-actions[bot]"}, - "state": "failure", - "description": f"Failed against newer main. [evaluation {order}]", - "target_url": f"https://github.com/openai/example/actions/runs/{order.split(':')[2]}", - } - self.assertEqual(self.publish(previous_statuses=[previous]), []) - - def test_publication_guard_allows_current_retry_and_ignores_unrelated_statuses( - self, - ) -> None: - previous = { - "context": "Castiron / custom-code budget", - "creator": {"login": "github-actions[bot]"}, - "description": "Failed. [evaluation 123:1:123:1]", - "target_url": "https://github.com/openai/example/actions/runs/123", - } - for overrides in ( - {}, # A partial publication can retry the same evaluation. - { - "description": "Failed. [evaluation 122:9:999:9]", - "target_url": "https://github.com/openai/example/actions/runs/999", - }, - {"description": "Legacy status without evaluation marker"}, - {"context": "Other check", "description": "[evaluation 999:9:999:9]"}, - {"creator": {"login": "someone"}, "description": "[evaluation 999:9:999:9]"}, - { - "description": "[evaluation 999:9:999:9]", - "target_url": "https://example.com/999", - }, - ): - with self.subTest(overrides=overrides): - results = self.publish(previous_statuses=[{**previous, **overrides}]) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "success" for r in results)) - self.assertTrue(all(len(r["description"]) <= 140 for r in results)) diff --git a/scripts/castiron/test_custom_code_strict.py b/scripts/castiron/test_custom_code_strict.py deleted file mode 100644 index b258213e63..0000000000 --- a/scripts/castiron/test_custom_code_strict.py +++ /dev/null @@ -1,774 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regression tests for strict generated-file ownership. -from __future__ import annotations - -import hashlib -import json -import subprocess -import sys -import unittest -from pathlib import Path -from typing import Any, cast -from unittest import mock - -import custom_code_budget as budget -import custom_code_report as report -from custom_code_test_support import GitTestCase - - -class StrictTests(GitTestCase): - workflow = ".github/workflows/castiron-custom-code.yml" - tooling_dir = "scripts/castiron" - checker_path = f"{tooling_dir}/custom_code_budget.py" - manifest_path = f"{tooling_dir}/generated-files.json" - alternate_manifest_path = ( - "scripts/castiron/generated-files.json" - if tooling_dir == "generated/castiron" - else "generated/castiron/generated-files.json" - ) - - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.write("generated.py", "generated\n") - for path in budget.GENERATED_GITHUB_PATHS: - self.write(path.decode(), "workflow\n") - self.inventory(budget.GENERATED_GITHUB_PATHS) - self.manifest() - self.generated, _ = self.baseline() - self.write( - budget.POLICY, - json.dumps( - {"schema_version": 2, "generated_files": "strict", "max_custom_patch_lines": 0} - ), - ) - self.base = self.commit("enable strict ownership") - - def inventory(self, paths: frozenset[bytes]) -> None: - # Checkpoint code is data. A verifier must never execute this statement. - self.write( - self.checker_path, - "raise AssertionError('checkpoint code must not execute')\n" - f"GENERATED_GITHUB_PATHS = frozenset({set(paths)!r})\n", - ) - - def manifest(self, extra: tuple[str, ...] = ()) -> None: - files = { - p: { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / p).read_bytes()).hexdigest(), - } - for p in ( - "generated.py", - *(path.decode() for path in budget.GENERATED_GITHUB_PATHS), - self.checker_path, - *extra, - ) - } - self.write(self.manifest_path, json.dumps({"schema_version": 1, "files": files})) - - def evaluate(self) -> dict[str, Any]: - return budget.evaluate(self.repo, self.base, self.commit(), public=False)[0] - - def test_handwritten_change_passes(self) -> None: - self.write("handwritten.py", "custom behavior\n") - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "success") - - def test_strict_main_checks_migration_target_and_candidate(self) -> None: - self.git("checkout", "-q", "-b", "next", self.base) - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - target = self.commit("next still uses migration policy") - self.write(self.workflow, "manual workflow edit\n") - head = self.commit() - for pull_heads in (None, [head]): - with self.subTest(pull_heads=pull_heads): - result, _ = budget.evaluate( - self.repo, - target, - head, - public=False, - policy_sha=self.base, - pull_heads=pull_heads, - ) - self.assertEqual(result["total"], 0) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, result["strict_violations"][0]) - - def test_exact_main_snapshot_bootstrap_verifies_real_checkpoints(self) -> None: - self.assert_exact_main_snapshot_bootstrap() - - def test_exact_main_snapshot_bootstrap_preserves_go_ratchet(self) -> None: - self.assert_exact_main_snapshot_bootstrap(ratchet="go") - - def test_exact_main_snapshot_bootstrap_accepts_ssh_origin(self) -> None: - self.assert_exact_main_snapshot_bootstrap(origin="git@github.com:openai/example.git") - - def test_next_preparation_rejects_wrong_origin_before_network(self) -> None: - self.git("remote", "add", "origin", "https://github.com/attacker/example.git") - for origin in ( - "https://github.com/attacker/example.git", - "git@github.com:attacker/example.git", - "https://github.com/openai/example-internal.git", - "https://github.com.example.org/openai/example.git", - ): - with self.subTest(origin=origin), mock.patch.object(budget, "read_public_api") as api: - self.git("remote", "set-url", "origin", origin) - with self.assertRaisesRegex(ValueError, "public repository as origin"): - budget.prepare_next_baseline(self.repo, "openai/example", "codex/prepare") - api.assert_not_called() - - def assert_exact_main_snapshot_bootstrap( - self, ratchet: str | None = None, *, origin: str | None = None - ) -> None: - repository = "openai/example" - self.git("remote", "add", "origin", origin or f"https://github.com/{repository}.git") - - def public_snapshot(generated: str, parent: str | None) -> str: - stats = report.read_stats(self.repo, self.git("rev-parse", "HEAD")) - return report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - stats["generation_id"], - report.hash_codegen_commit(self.repo, generated), - "codegen/test", - parent, - ) - - old_public = public_snapshot(self.generated, None) - stats_path = self.repo / ".castiron.stats.yml" - stats_path.write_text(stats_path.read_text() + f"public_codegen_sha: {old_public}\n") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":4000}') - self.write("generated.py", "legacy custom edit\n") - legacy = self.commit("legacy unpinned next") - self.write("generated.py", "generated\n") - checker = self.repo / self.checker_path - checker.write_text(checker.read_text() + "NEXT_POLICY_VERSION = 1\n") - self.manifest() - # Generated checkpoints exclude integration metadata and repository policy. - stats_path.unlink() - (self.repo / budget.POLICY).unlink() - generated, _ = self.baseline() - current_public = public_snapshot(generated, old_public) - self.git("update-ref", "refs/remotes/origin/codegen/test", current_public) - stats_path.write_text(stats_path.read_text() + f"public_codegen_sha: {current_public}\n") - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - if ratchet is not None: - strict["ratchet"] = ratchet - self.write(budget.POLICY, json.dumps(strict)) - main = self.commit("verified strict main") - self.git("update-ref", "refs/remotes/origin/main", main) - self.git("update-ref", "refs/remotes/origin/next", legacy) - real_git = budget.report.git - - def offline_git(repo: Path, *args: str, **kwargs: Any) -> bytes: - # Only network transport is mocked; refs, trees, metadata and verification are real Git. - if args[0] == "fetch": - for spec in args: - if spec.startswith("+refs/heads/"): - source, destination = spec[1:].split(":", 1) - revision = ( - real_git( - repo, - "rev-parse", - source.replace("refs/heads/", "refs/remotes/origin/", 1), - ) - .decode() - .strip() - ) - real_git(repo, "update-ref", destination, revision) - return b"" - return cast(bytes, real_git(repo, *args, **kwargs)) - - def public_api(path: str) -> dict[str, Any]: - if path == f"repos/{repository}": - return {"private": False, "default_branch": "main", "full_name": repository} - return {"object": {"sha": main if path.endswith("/main") else legacy}} - - original_index = self.git("write-tree") - with ( - mock.patch.object(budget.report, "git", side_effect=offline_git), - mock.patch.object(budget, "read_public_api", side_effect=public_api), - ): - prepared = budget.prepare_next_baseline(self.repo, repository, "codex/adopt-main") - candidate = prepared["head_sha"] - self.assertEqual( - budget.read_policy(self.repo, candidate), - {**strict, "next_policy": {"repository": repository, "main_sha": main}}, - ) - self.assertEqual(self.git("rev-parse", "refs/heads/codex/adopt-main"), candidate) - self.assertEqual(self.git("write-tree"), original_index) - self.assertEqual(self.git("rev-parse", "HEAD"), main) - self.assertEqual( - self.git("show", f"{candidate}:.castiron.stats.yml"), - self.git("show", f"{main}:.castiron.stats.yml"), - ) - refs_read = 0 - - def moving_api(path: str) -> dict[str, Any]: - nonlocal refs_read - if path.endswith("/main"): - refs_read += 1 - if refs_read > 1: - return {"object": {"sha": legacy}} - return public_api(path) - - with ( - mock.patch.object(budget.report, "git", side_effect=offline_git), - mock.patch.object(budget, "read_public_api", side_effect=moving_api), - self.assertRaisesRegex(ValueError, "refs changed"), - ): - budget.prepare_next_baseline(self.repo, repository, "codex/stale-main") - self.assertEqual(self.git("for-each-ref", "refs/heads/codex/stale-main"), "") - self.git("checkout", "--detach", "-q", candidate) - wrapper = self.git( - "commit-tree", - f"{candidate}^{{tree}}", - "-p", - legacy, - "-p", - candidate, - "-m", - "verified queue wrapper", - ) - for heads in (None, [candidate]): - with self.subTest(queue=heads is not None): - result, _ = budget.evaluate( - self.repo, - legacy, - candidate if heads is None else wrapper, - public=True, - target_branch="next", - policy_sha=main, - main_snapshot=main, - repository=repository, - pull_heads=heads, - ) - self.assertEqual(result["total"], 0) - self.assertTrue( - all(c["state"] == "success" for c in result["checks"].values()), result - ) - # Exercise the real hosted evaluator with independent, initially empty - # object stores. The selector's origin/main ref is not shared with them. - from test_custom_code_github import GitHubAPI - - remote = self.repo / "bootstrap-remote.git" - self.git("clone", "--bare", str(self.repo), str(remote)) - real_git(remote, "update-ref", "refs/heads/main", main) - real_git(remote, "update-ref", "refs/heads/codegen/test", current_public) - real_git(remote, "update-ref", "refs/heads/queue", wrapper) - named_fetches: list[Path] = [] - - def transport(repo: Path, *args: str, **kwargs: Any) -> bytes: - if args[0] == "fetch": - self.assertIn("origin", args) - if "+refs/heads/main:refs/remotes/origin/main" in args: - named_fetches.append(repo) - args = tuple(str(remote) if arg == "origin" else arg for arg in args) - self.assertNotIn("checkout", args) - return cast(bytes, real_git(repo, *args, **kwargs)) - - for mode in ("pr", "reused-pr", "queue", "moved-main"): - with self.subTest(object_store=mode): - queue = mode in ("queue", "moved-main") - revision = wrapper if queue else candidate - hosted_api = GitHubAPI(main, revision, "merge_group" if queue else "pull_request") - hosted_api.next = legacy - hosted_api.pull["base"]["ref"] = "next" - if queue: - hosted_api.run["head_branch"] = "gh-readonly-queue/next/pr-3-test" - objects = self.repo / f"bootstrap-{mode}.git" - out = self.repo / f"bootstrap-{mode}-report" - receipt = { - "schema_version": 1, - "target_branch": "next", - "base_sha": legacy, - "head_sha": revision, - "policy_sha": main, - "bootstrap": True, - "main_snapshot": main, - } - with ( - mock.patch.object(report, "git", side_effect=transport), - mock.patch.object(budget.report, "git", side_effect=transport), - mock.patch.object(report, "api", side_effect=hosted_api), - mock.patch.object(budget.report, "api", side_effect=hosted_api), - mock.patch.object( - budget, "queued_entries", return_value=[(candidate, wrapper)] - ), - ): - if mode == "reused-pr": - report.trusted_report(objects, repository, 123, 1, out, base=main) - self.assertEqual( - real_git(objects, "for-each-ref", "refs/remotes/origin/main"), b"" - ) - if mode == "moved-main": - # API receipt remains captured at main while Git's named - # ref changes before fetch: the evaluator must fail closed. - real_git(remote, "update-ref", "refs/heads/main", legacy) - with self.assertRaisesRegex(ValueError, "authenticated main changed"): - budget.github_evaluate( - objects, - repository, - { - "repository": {"full_name": repository}, - "workflow_run": hosted_api.run, - }, - main, - selection=receipt, - ) - else: - result, _ = budget.github_evaluate( - objects, - repository, - { - "repository": {"full_name": repository}, - "workflow_run": hosted_api.run, - }, - main, - out if mode == "reused-pr" else None, - selection=receipt, - ) - self.assertTrue( - all(c["state"] == "success" for c in result["checks"].values()), result - ) - self.assertEqual(result["total"], 0) - self.assertEqual( - real_git(objects, "rev-parse", "refs/remotes/origin/main") - .decode() - .strip(), - main, - ) - self.assertIn(objects, named_fetches) - self.assertFalse((objects / "scripts").exists()) - self.write("generated.py", "unauthorized custom edit\n") - tampered = self.commit("tampered candidate") - result, _ = budget.evaluate( - self.repo, - legacy, - tampered, - public=True, - target_branch="next", - policy_sha=main, - main_snapshot=main, - repository=repository, - ) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_activation_accepts_stale_pr_without_policy_edits(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.git("checkout", "-q", "-b", "migration", f"{self.base}^") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - migration = self.commit("migration policy before activation") - self.git("checkout", "-q", "-b", "stale", migration) - self.write("handwritten.py", "custom behavior\n") - stale = self.commit("PR predates strict activation") - self.git("checkout", "-q", "migration") - self.write(budget.POLICY, strict) - activated = self.commit("activate strict policy") - result, _ = budget.evaluate(self.repo, activated, stale, public=False) - self.assertTrue(all(c["state"] == "success" for c in result["checks"].values())) - self.git("merge", "--no-ff", "-m", "queue stale PR", stale) - result, _ = budget.evaluate( - self.repo, - activated, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[stale], - ) - self.assertTrue(all(c["state"] == "success" for c in result["checks"].values())) - - def test_stale_policy_edit_cannot_downgrade_activated_base(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.git("checkout", "-q", "-b", "migration", f"{self.base}^") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":10}') - migration = self.commit("migration policy before activation") - self.git("checkout", "-q", "-b", "stale", migration) - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - stale = self.commit("stale policy increase") - self.git("checkout", "-q", "migration") - self.write(budget.POLICY, strict) - activated = self.commit("activate strict policy") - result, _ = budget.evaluate(self.repo, activated, stale, public=False) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertIn("cannot be downgraded", result["checks"]["isolation"]["description"]) - - def test_workflow_edit_fails_despite_zero_legacy_patch(self) -> None: - self.write(self.workflow, "edited workflow\n") - result = self.evaluate() - self.assertEqual(result["total"], 0) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, result["strict_violations"][0]) - - def test_merge_queue_checks_combined_generated_ownership(self) -> None: - self.git("checkout", "-q", "-b", "handwritten", self.base) - self.write("handwritten.py", "custom behavior\n") - handwritten = self.commit() - self.git("checkout", "-q", "-b", "workflow", self.base) - self.write(self.workflow, "manual workflow edit\n") - workflow = self.commit() - self.git("checkout", "-q", "-b", "queue", self.base) - self.git("merge", "--no-ff", "-m", "queue handwritten", handwritten) - passing, _ = budget.evaluate( - self.repo, - self.base, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[handwritten], - ) - self.assertTrue(all(c["state"] == "success" for c in passing["checks"].values())) - self.git("merge", "--no-ff", "-m", "queue workflow", workflow) - failing, _ = budget.evaluate( - self.repo, - self.base, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[handwritten, workflow], - ) - self.assertEqual(failing["total"], 0) - self.assertEqual(failing["checks"]["isolation"]["state"], "success") - self.assertEqual(failing["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, failing["strict_violations"][0]) - - def test_content_deletion_rename_mode_and_symlink_fail(self) -> None: - for mutation in ("edit", "delete", "rename", "mode", "symlink"): - with self.subTest(mutation=mutation): - self.git("reset", "--hard", self.base) - path = self.repo / "generated.py" - if mutation == "edit": - path.write_text("header removed and manually changed\n") - elif mutation == "delete": - path.unlink() - elif mutation == "rename": - path.rename(self.repo / "renamed.py") - elif mutation == "mode": - path.chmod(0o755) - else: - path.unlink() - path.symlink_to("handwritten.py") - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "failure") - - def test_candidate_cannot_rewrite_its_manifest_or_disable_strict(self) -> None: - self.write(self.workflow, "custom\n") - self.manifest() - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 1000})) - result = self.evaluate() - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_missing_manifest_cannot_activate(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / self.manifest_path).unlink() - checkpoint = self.commit(self.git("show", "-s", "--format=%B", self.generated)) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("checkout", "-q", "main") - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write( - ".castiron.stats.yml", - stats.replace(self.generated, checkpoint).replace( - report.hash_codegen_commit(self.repo, self.generated), - report.hash_codegen_commit(self.repo, checkpoint), - ), - ) - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "failure") - - def test_legitimate_new_generated_checkpoint_passes(self) -> None: - self.git("checkout", "-q", self.generated) - self.write("generated.py", "new generated output\n") - self.write(self.workflow, "new generated workflow\n") - self.manifest() - checkpoint = self.commit(self.git("show", "-s", "--format=%B", self.generated)) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("checkout", "-q", "main") - for path in ("generated.py", self.workflow, self.manifest_path): - self.write(path, self.git("show", f"{checkpoint}:{path}") + "\n") - # Preserve exact manifest bytes (the fixture writes JSON without a newline). - self.write(self.manifest_path, self.git("show", f"{checkpoint}:{self.manifest_path}")) - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write( - ".castiron.stats.yml", - stats.replace(self.generated, checkpoint).replace( - report.hash_codegen_commit(self.repo, self.generated), - report.hash_codegen_commit(self.repo, checkpoint), - ), - ) - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "success") - - def test_worktree_verification_catches_uncommitted_edits(self) -> None: - self.assertEqual( - budget.verify_generated(self.repo, self.generated, self.base, worktree=True), [] - ) - self.write(self.workflow, "uncommitted\n") - failures = budget.verify_generated(self.repo, self.generated, self.base, worktree=True) - self.assertEqual(len(failures), 1) - self.assertIn(self.workflow, failures[0]) - - def test_activation_is_policy_only_and_checks_workflows(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - migration = self.commit() - self.write(budget.POLICY, strict) - activation = self.commit() - result, _ = budget.evaluate(self.repo, migration, activation, public=False) - self.assertTrue(all(check["state"] == "success" for check in result["checks"].values())) - self.write(self.workflow, "custom workflow\n") - result, _ = budget.evaluate(self.repo, migration, self.commit(), public=False) - self.assertTrue(all(check["state"] == "failure" for check in result["checks"].values())) - - def test_manifest_must_cover_snapshot_and_match_its_bytes(self) -> None: - self.git("checkout", "-q", self.generated) - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"].pop("generated.py") - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "cover.*exactly"): - budget.generated_manifest(self.repo, self.commit()) - self.manifest() - self.write("generated.py", "snapshot changed without updating manifest\n") - with self.assertRaisesRegex(ValueError, "disagrees"): - budget.generated_manifest(self.repo, self.commit()) - - def test_relocated_tooling_preserves_ownership_and_retirement_checks(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - old_manifest = self.manifest_path - new_manifest = self.alternate_manifest_path - old_checker = self.checker_path - new_checker = str(Path(new_manifest).with_name("custom_code_budget.py")) - manifest = json.loads((self.repo / old_manifest).read_text()) - manifest["files"][new_checker] = manifest["files"].pop(old_checker) - checker = (self.repo / old_checker).read_text() - self.write(new_checker, checker) - self.write(new_manifest, json.dumps(manifest)) - (self.repo / old_checker).unlink() - (self.repo / old_manifest).unlink() - relocated = self.commit("relocate generated tooling") - self.assertEqual( - budget.verify_generated(self.repo, relocated, relocated, previous=self.generated), [] - ) - ownership = budget.generated_manifest(self.repo, relocated) - self.assertIn(new_manifest.encode(), ownership) - self.assertIn(new_checker.encode(), ownership) - self.assertNotIn(old_manifest.encode(), ownership) - self.write(old_checker, checker) - self.write("generated.py", "handwritten modification\n") - failures = budget.verify_generated( - self.repo, relocated, self.commit(), previous=self.generated - ) - self.assertTrue( - any(old_checker in failure and "retired" in failure for failure in failures) - ) - self.assertTrue(any("generated.py: differs" in failure for failure in failures)) - # Candidate manifest edits cannot change the verified ownership contract. - self.write(new_manifest, '{"schema_version": 1, "files": {}}') - failures = budget.verify_generated(self.repo, relocated, self.commit()) - self.assertTrue( - any(new_manifest in failure and "differs" in failure for failure in failures) - ) - - def test_checkpoint_rejects_ambiguous_manifest_locations(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - self.write( - self.alternate_manifest_path, - (self.repo / self.manifest_path).read_text(), - ) - with self.assertRaisesRegex(ValueError, "ambiguous"): - budget.generated_manifest(self.repo, self.commit()) - - def test_relocated_manifest_requires_colocated_checker_inventory(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - self.write( - self.alternate_manifest_path, - (self.repo / self.manifest_path).read_text(), - ) - (self.repo / self.manifest_path).unlink() - with self.assertRaisesRegex(ValueError, "missing its generated GitHub inventory"): - budget.generated_manifest(self.repo, self.commit()) - - def test_manifest_preserves_inherited_sdk_owned_github_files(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - inherited = (".github/workflows/ci.yml", ".github/actions/setup-go/action.yml") - for path in inherited: - self.write(path, "SDK-owned configuration\n") - generated = self.commit("checkpoint preserves protected parent subtrees") - manifest = budget.generated_manifest(self.repo, generated) - for path in inherited: - self.assertNotIn(path.encode(), manifest) - self.write(path, "SDK-owned configuration update\n") - head = self.commit("handwritten configuration remains outside generated ownership") - self.assertEqual(budget.verify_generated(self.repo, generated, head), []) - - def test_private_manifest_checks_excluded_workflows_and_actions(self) -> None: - self.assertIn(self.workflow.encode(), budget.GENERATED_GITHUB_PATHS) - for path in (self.workflow, ".github/actions/generated/action.yml"): - for mutation in ("omit", "content", "mode"): - with ( - self.subTest(path=path, mutation=mutation), - mock.patch.object( - budget, - "GENERATED_GITHUB_PATHS", - budget.GENERATED_GITHUB_PATHS | {path.encode()}, - ), - ): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "generated workflow or action\n") - self.inventory(budget.GENERATED_GITHUB_PATHS) - self.manifest((path,)) - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"][path] = { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / path).read_bytes()).hexdigest(), - } - if mutation == "omit": - del manifest["files"][path] - elif mutation == "content": - self.write(path, "unrecorded change\n") - else: - (self.repo / path).chmod(0o755) - self.write(self.manifest_path, json.dumps(manifest)) - error = ( - "omits generated GitHub inventory path" - if mutation == "omit" - else "disagrees" - ) - with self.assertRaisesRegex(ValueError, error): - budget.generated_manifest(self.repo, self.commit()) - - def test_checkpoint_inventory_can_add_and_retire_generated_ci(self) -> None: - for path in (".github/workflows/new.yml", ".github/actions/new/action.yml"): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "new generated CI\n") - self.inventory(budget.GENERATED_GITHUB_PATHS | {path.encode()}) - self.manifest((path,)) - newer = self.commit("new compiler inventory") - # The executing verifier still has the old inventory. - self.assertNotIn(path.encode(), budget.GENERATED_GITHUB_PATHS) - self.assertEqual( - budget.verify_generated(self.repo, newer, newer, previous=self.generated), [] - ) - (self.repo / path).unlink() - public = self.commit("public checkpoint omits private CI blob") - self.assertEqual( - budget.verify_generated(self.repo, public, newer, previous=self.generated), [] - ) - # A newer verifier must also read the prior inventory to enforce retirement. - self.assertEqual( - budget.verify_generated(self.repo, self.generated, self.base, previous=public), - [], - ) - self.git("checkout", "--detach", "-q", self.base) - self.write(path, "retired generated CI\n") - failures = budget.verify_generated( - self.repo, self.generated, self.commit(), previous=public - ) - self.assertTrue( - any(path in failure and "retired" in failure for failure in failures) - ) - - def test_manifest_cannot_claim_sdk_owned_workflows_or_actions(self) -> None: - for path in (".github/workflows/ci.yml", ".github/actions/setup-go/action.yml"): - for keep_blob in (True, False): - with self.subTest(path=path, keep_blob=keep_blob): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "SDK-owned configuration\n") - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"][path] = { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / path).read_bytes()).hexdigest(), - } - if not keep_blob: - (self.repo / path).unlink() - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "SDK-owned GitHub path"): - budget.generated_manifest(self.repo, self.commit()) - - def test_public_snapshot_requires_every_inventory_identity(self) -> None: - for path in (self.workflow, ".github/actions/generated/action.yml"): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "generated CI\n") - self.inventory(budget.GENERATED_GITHUB_PATHS | {path.encode()}) - self.manifest((path,)) - (self.repo / path).unlink() - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"][path] - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "omits generated GitHub inventory path"): - budget.generated_manifest( - self.repo, self.commit("incomplete public checkpoint") - ) - - def test_public_snapshot_keeps_strict_workflow_identity_without_private_blob(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - (self.repo / self.workflow).unlink() - public = self.commit("sanitized public checkpoint") - self.assertIn(self.workflow.encode(), budget.generated_manifest(self.repo, public)) - self.assertEqual(budget.verify_generated(self.repo, public, self.base), []) - self.git("checkout", "--detach", "-q", self.base) - self.write(self.workflow, "manual workflow edit\n") - violations = budget.verify_generated(self.repo, public, self.commit()) - self.assertEqual(len(violations), 1) - self.assertIn(self.workflow, violations[0]) - - def test_generated_deletion_requires_removing_retired_file(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / "generated.py").unlink() - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"]["generated.py"] - self.write(self.manifest_path, json.dumps(manifest)) - new = self.commit() - self.assertEqual(budget.verify_generated(self.repo, new, new, previous=self.generated), []) - self.write("generated.py", "retained handwritten replacement\n") - failures = budget.verify_generated(self.repo, new, self.commit(), previous=self.generated) - self.assertEqual(len(failures), 1) - self.assertIn("retired generated file", failures[0]) - - def test_generated_file_can_become_a_directory(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / "generated.py").unlink() - self.write("generated.py/nested.py", "new generated file\n") - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"]["generated.py"] - manifest["files"]["generated.py/nested.py"] = { - "mode": "100644", - "sha256": hashlib.sha256(b"new generated file\n").hexdigest(), - } - self.write(self.manifest_path, json.dumps(manifest)) - new = self.commit() - for worktree in (False, True): - with self.subTest(worktree=worktree): - self.assertEqual( - budget.verify_generated( - self.repo, new, new, previous=self.generated, worktree=worktree - ), - [], - ) - - def test_compiler_entrypoint_preserves_failed_worktree(self) -> None: - self.write(self.workflow, "uncommitted custom workflow\n") - result = subprocess.run( - [ - sys.executable, - "-I", - str(Path(budget.__file__)), - "verify-generated", - "--repo", - str(self.repo), - "--policy-ref", - self.base, - "--generated", - self.generated, - "--head", - self.base, - "--worktree", - ], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 1) - self.assertIn(self.workflow, result.stderr) - self.assertEqual((self.repo / self.workflow).read_text(), "uncommitted custom workflow\n") - - -if __name__ == "__main__": - unittest.main()