diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index eac8e669b..2fc06595b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -99,6 +99,25 @@ uncovered behavior instead of lowering the threshold. The thresholds live in `protocol/vitest.config.ts`. Swift, Android, and standalone telemetry tests do not yet enforce coverage thresholds. +CI updates one **Coverage Report** comment on same-repository pull requests for +Web, React Native JavaScript, and Embedded Checkout Protocol (TS). The table +shows status emojis, line, branch, and function coverage percentages, and links +to the full reports. Jobs that are pending, skipped, failed, or missing a report +show their status instead of coverage from an older run. GitHub Actions jobs also +write their coverage tables to the job summary. Fork builds and builds without a +write token keep their artifacts and log summaries without posting a comment. + +Platform jobs store numeric results in GitHub checks. The `Coverage Report` +workflow is the only comment writer and serializes updates per PR. It refreshes +on CI workflow events, reading all current results so simultaneous completions +do not lose data. A manual run of `Coverage Report` with the PR number refreshes +interrupted reporting. + +The publisher executes trusted default-branch code and validates the numeric +results; it never executes PR-provided reporting code. This workflow begins +publishing once it exists on `main`. Coverage thresholds remain in the test +runners; combining the comments does not change their enforcement. + ### Testing PR builds with Tophat [Tophat](https://github.com/Shopify/tophat) is a macOS menu-bar app that diff --git a/.github/actions/report-coverage/action.yml b/.github/actions/report-coverage/action.yml new file mode 100644 index 000000000..4e4e6f7f0 --- /dev/null +++ b/.github/actions/report-coverage/action.yml @@ -0,0 +1,38 @@ +name: Record coverage +description: Save a platform result for the shared coverage comment publisher +inputs: + platform: + required: true + description: Coverage platform key + report-path: + required: true + description: Coverage summary JSON, relative to the repository root + report-directory: + required: true + description: Directory containing the downloadable reports + outcome: + required: true + description: Outcome of the coverage test step +runs: + using: composite + steps: + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + id: artifact + if: ${{ hashFiles(inputs.report-path) != '' }} + with: + name: ${{ inputs.platform }}-coverage-${{ github.run_attempt }} + path: ${{ inputs.report-directory }} + retention-days: 7 + - name: Record coverage result + shell: bash + working-directory: ${{ github.workspace }} + env: + COVERAGE_PLATFORM: ${{ inputs.platform }} + COVERAGE_PATH: ${{ inputs.report-path }} + COVERAGE_TEST_OUTCOME: ${{ inputs.outcome }} + COVERAGE_REPOSITORY: ${{ github.repository }} + COVERAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + COVERAGE_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} + COVERAGE_TOKEN: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }} + COVERAGE_REPORT_URL: ${{ steps.artifact.outputs.artifact-url }} + run: ruby scripts/report_coverage "$COVERAGE_PLATFORM" "$COVERAGE_PATH" diff --git a/.github/scripts/publish-coverage.cjs b/.github/scripts/publish-coverage.cjs new file mode 100644 index 000000000..c106b39b9 --- /dev/null +++ b/.github/scripts/publish-coverage.cjs @@ -0,0 +1,155 @@ +const marker = ""; +const jsMetrics = ["Lines", "Statements", "Branches", "Functions"]; +const platforms = [ + {id: "web", title: "Web", job: "Web / Lint, test, build, verify", metrics: jsMetrics}, + {id: "react-native", title: "React Native", job: "React Native / Run jest tests", metrics: jsMetrics}, + {id: "protocol", title: "Embedded Checkout Protocol (TypeScript)", displayTitle: "Embedded Checkout Protocol (TS)", job: "Protocol / Test", skippedJob: "Protocol", metrics: jsMetrics}, +]; + +const latest = (items) => [...items].sort((a, b) => b.id - a.id)[0]; +const isComment = (comment) => comment.user?.type === "Bot" && + comment.user.login === "github-actions[bot]" && comment.body?.startsWith(marker); + +async function resolvePR({github, context}) { + if (context.eventName === "workflow_dispatch") { + const pr = Number(context.payload.inputs?.pr); + return Number.isSafeInteger(pr) && pr > 0 ? pr : null; + } + const run = context.payload.workflow_run; + if (run?.event !== "pull_request" || run.path !== ".github/workflows/ci.yml") return null; + const prs = run.pull_requests?.length ? run.pull_requests : + await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, { + ...context.repo, commit_sha: run.head_sha, per_page: 100, + }); + return prs.find((pr) => pr.head.sha === run.head_sha && pr.head.repo?.id === run.head_repository?.id)?.number ?? null; +} + +async function sources(github, repo, pr) { + const [runs, checks] = await Promise.all([ + github.paginate(github.rest.actions.listWorkflowRuns, { + ...repo, workflow_id: "ci.yml", event: "pull_request", head_sha: pr.head.sha, per_page: 100, + }), + github.paginate(github.rest.checks.listForRef, { + ...repo, ref: pr.head.sha, filter: "all", per_page: 100, + }), + ]); + const run = latest(runs.filter((item) => item.head_sha === pr.head.sha && item.head_repository?.id === pr.head.repo.id && item.head_branch === pr.head.ref)); + return {run, checks}; +} + +function reportURL(value, repo, run) { + if (!value) return null; + const url = new URL(value); + const artifactPrefix = `/${repo.owner}/${repo.repo}/actions/runs/${run?.id}/artifacts/`; + const valid = url.hostname === "github.com" && url.pathname.startsWith(artifactPrefix) && /^[0-9]+$/.test(url.pathname.slice(artifactPrefix.length)); + if (!valid || url.protocol !== "https:" || url.username || url.password || url.search || url.hash) + throw new Error("Invalid coverage report link"); + return url.href; +} + +function readResult(check, platform, expected, pr, repo, run) { + if (check.head_sha !== pr.head.sha || !check.output?.text || check.output.text.length > 32768) + throw new Error("Invalid coverage check"); + const result = JSON.parse(check.output.text); + if (result.version !== 1 || result.platform !== platform.id || result.pr !== pr.number || + result.headSha !== pr.head.sha || result.source?.provider !== expected.provider || + result.source.runId !== expected.runId || (result.source.runAttempt ?? 1) !== expected.runAttempt || + !["success", "failed", "skipped", "unavailable"].includes(result.state) || !Array.isArray(result.rows)) + throw new Error("Coverage result does not match this run"); + if (["skipped", "unavailable"].includes(result.state) && result.rows.length) + throw new Error("Unexpected measurements for an unmeasured platform"); + if (result.rows.length || result.state === "success") { + if (result.rows.length !== platform.metrics.length) throw new Error("Missing coverage metrics"); + result.rows.forEach((row, index) => { + if (!Array.isArray(row) || row.length !== 3 || row[0] !== platform.metrics[index] || + !Number.isSafeInteger(row[1]) || !Number.isSafeInteger(row[2]) || row[1] < 0 || row[2] < row[1]) + throw new Error("Invalid coverage counts"); + }); + } + return {...result, reportUrl: reportURL(result.reportUrl, repo, run)}; +} + +function platformResult(platform, snapshot, jobs, pr, repo, core) { + const {run, checks} = snapshot; + let state = "pending"; + if (!run) return {state}; + // A failed-jobs rerun reuses successful jobs from an earlier attempt. Match + // each result to the attempt that actually ran that platform's test job. + const job = latest(jobs.filter((item) => item.name === platform.job)) || + latest(jobs.filter((item) => item.name === (platform.skippedJob || platform.title))); + const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt}; + const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas")); + if (plan?.status === "completed" && plan.conclusion !== "success") return {state: "unavailable"}; + if (job?.conclusion === "skipped") return {state: "skipped"}; + if (job?.status === "completed") state = job.conclusion === "success" ? "unavailable" : "failed"; + else if (!job && run.status === "completed") state = "unavailable"; + const externalId = `coverage:${platform.id}:${expected.runId}:${expected.runAttempt}`; + const check = latest(checks.filter((item) => item.app?.slug === expected.provider && + item.external_id === externalId && item.name === `Coverage — ${platform.title}`)); + if (!check) return {state}; + try { + return readResult(check, platform, expected, pr, repo, run); + } catch (error) { + core.warning(`${platform.title}: ${error.message}`); + return {state: "unavailable"}; + } +} + +function metric(row) { + if (!row) return "—"; + const [, covered, total] = row; + if (!total) return "N/A"; + const percentage = Number((100 * covered / total).toFixed(2)); + return `${percentage}%`; +} + +function render(results) { + const lines = [marker, "# Coverage Report", "", + "| Status | Platform / target | Lines | Branches | Functions | Report |", + "| :---: | --- | ---: | ---: | ---: | --- |"]; + const statuses = { + success: ["✅", "—"], pending: ["⏳", "Waiting for coverage"], + skipped: ["⏭️", "Not run for this change"], failed: ["❌", "Tests or coverage collection failed"], + unavailable: ["⚠️", "Coverage report unavailable"], + }; + for (const platform of platforms) { + const result = results[platform.id]; + const rows = result.rows || []; + const [emoji, status] = statuses[result.state]; + const report = result.reportUrl ? `[Full report](${result.reportUrl})` : status; + const find = (name) => rows.find((row) => row[0] === name); + lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${metric(find("Lines"))} | ${metric(find("Branches"))} | ${metric(find("Functions"))} | ${report} |`); + } + return lines.join("\n") + "\n"; +} + +const sourceKey = ({run}) => `${run?.id}:${run?.run_attempt}`; + +async function publish({github, context, core, prNumber}) { + const repo = context.repo; + const {data: pr} = await github.rest.pulls.get({...repo, pull_number: prNumber}); + // Match the producer policy: forks keep artifacts/logs without comment writes. + if (pr.state !== "open" || pr.draft || pr.head.repo?.full_name !== `${repo.owner}/${repo.repo}`) return; + const snapshot = await sources(github, repo, pr); + const jobs = snapshot.run ? await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...repo, run_id: snapshot.run.id, filter: "all", per_page: 100, + }) : []; + const results = Object.fromEntries(platforms.map((platform) => + [platform.id, platformResult(platform, snapshot, jobs, pr, repo, core)])); + const body = render(results); + const comments = await github.paginate(github.rest.issues.listComments, {...repo, issue_number: pr.number, per_page: 100}); + const existing = comments.find(isComment); + // Wait for change detection before opening a comment for an affected platform. + // Existing comments still refresh so skipped runs replace stale measurements. + const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas")); + const inScope = Object.values(results).some((result) => result.state !== "skipped"); + if (!existing && (plan?.status !== "completed" || !inScope)) return; + const {data: current} = await github.rest.pulls.get({...repo, pull_number: pr.number}); + if (current.state !== "open" || current.draft || current.head.sha !== pr.head.sha || current.base.sha !== pr.base.sha) return; + if (sourceKey(await sources(github, repo, current)) !== sourceKey(snapshot)) return; + if (existing?.body === body) return; + if (existing) await github.rest.issues.updateComment({...repo, comment_id: existing.id, body}); + else await github.rest.issues.createComment({...repo, issue_number: pr.number, body}); +} + +module.exports = {resolvePR, publish, readResult, platformResult, render, platforms, marker, isComment}; diff --git a/.github/scripts/publish-coverage.test.cjs b/.github/scripts/publish-coverage.test.cjs new file mode 100644 index 000000000..faa95ee9b --- /dev/null +++ b/.github/scripts/publish-coverage.test.cjs @@ -0,0 +1,254 @@ +const {test} = require("node:test"); +const assert = require("node:assert/strict"); +const reporter = require("./publish-coverage.cjs"); + +function row(body, title) { + return body.split("\n").find((line) => line.includes(`| ${title} |`)); +} + +function fixture() { + const f = { + pr: {number: 12, state: "open", draft: false, head: {sha: "head", ref: "feature", repo: {id: 1, full_name: "example/sdk"}}, base: {sha: "base"}}, + runs: [{id: 100, run_attempt: 1, head_sha: "head", head_branch: "feature", head_repository: {id: 1}, status: "completed", conclusion: "success"}], + checks: [], + jobs: reporter.platforms.map((platform, i) => + ({id: 300 + i, name: platform.job, run_attempt: 1, status: "completed", conclusion: "success"})), + comments: [], writes: [], warnings: [], + }; + f.jobs.unshift({id: 250, name: "Detect Changed Areas", run_attempt: 1, status: "completed", conclusion: "success"}); + const rest = { + pulls: {get: async () => ({data: structuredClone(f.pr)})}, + actions: {listWorkflowRuns: "runs", listJobsForWorkflowRun: "jobs"}, + checks: {listForRef: "checks"}, + repos: {listPullRequestsAssociatedWithCommit: "prs"}, + issues: { + listComments: "comments", + createComment: async (params) => f.writes.push({method: "create", ...params}), + updateComment: async (params) => f.writes.push({method: "update", ...params}), + }, + }; + f.github = {rest, paginate: async (method) => { + if (method === "prs") return [f.pr]; + return structuredClone(f[method]); + }}; + f.context = {repo: {owner: "example", repo: "sdk"}, eventName: "workflow_dispatch", payload: {inputs: {pr: "12"}}}; + f.publish = () => reporter.publish({github: f.github, context: f.context, core: {warning: (message) => f.warnings.push(message)}, prNumber: 12}); + f.add = (id, overrides = {}) => { + const platform = reporter.platforms.find((platform) => platform.id === id); + const source = {provider: "github-actions", runId: 100, runAttempt: 1}; + const result = {version: 1, platform: id, pr: 12, headSha: "head", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides}; + const check = { + id: 400 + f.checks.length, name: `Coverage — ${platform.title}`, head_sha: "head", + app: {slug: source.provider}, external_id: `coverage:${id}:${source.runId}:${source.runAttempt}`, + output: {text: JSON.stringify(result)}, + }; + f.checks.push(check); + return check; + }; + return f; +} + +test("one comment combines the existing JavaScript reports without native coverage rows", async () => { + const f = fixture(); + for (const platform of reporter.platforms) f.add(platform.id); + await f.publish(); + const body = f.writes[0].body; + assert.equal(f.writes.length, 1); + assert.ok(body.startsWith(reporter.marker)); + assert.ok(body.includes("| Status | Platform / target | Lines | Branches | Functions | Report |")); + for (const platform of reporter.platforms.filter((item) => item.id !== "protocol")) + assert.ok(body.includes(`| ✅ | ${platform.title} | 70% |`)); + assert.ok(body.includes("| ✅ | Embedded Checkout Protocol (TS) | 70% | 70% | 70% |")); + assert.ok(!/Android|Swift|Kotlin/.test(body)); + assert.ok(!body.includes("shields.io")); + assert.ok(!body.includes("
")); + assert.ok(!body.includes("PR head:")); + assert.ok(!body.includes("Report only;")); +}); + +test("updates only the shared Actions comment and ignores human lookalikes", async () => { + const f = fixture(); + f.comments = [ + {id: 50, user: {login: "writer", type: "User"}, body: reporter.marker}, + {id: 51, user: {login: "github-actions[bot]", type: "Bot"}, body: reporter.marker}, + ]; + await f.publish(); + assert.equal(f.writes[0].method, "update"); + assert.equal(f.writes[0].comment_id, 51); + f.comments[1].body = f.writes[0].body; + await f.publish(); + assert.equal(f.writes.length, 1, "identical refreshes do not edit the comment"); +}); + +test("waits for change detection before creating a comment", async () => { + for (const status of [null, "queued", "in_progress"]) { + const f = fixture(); + f.runs[0].status = "in_progress"; + if (status) f.jobs[0].status = status; + else f.jobs.shift(); + await f.publish(); + assert.equal(f.writes.length, 0, String(status)); + } + + const f = fixture(); + f.runs[0].status = "in_progress"; + f.jobs.find((job) => job.name.startsWith("Web")).status = "in_progress"; + await f.publish(); + assert.equal(f.writes[0].method, "create"); + assert.ok(row(f.writes[0].body, "Web").includes("Waiting for coverage")); +}); + +test("skips a new comment when every reported platform is skipped but clears existing measurements", async () => { + const f = fixture(); + for (const platform of reporter.platforms) f.add(platform.id); + await f.publish(); + const previous = f.writes[0].body; + f.writes = []; + for (const job of f.jobs.filter((job) => job.name !== "Detect Changed Areas")) job.conclusion = "skipped"; + for (const platform of reporter.platforms) + f.add(platform.id, {state: "skipped", rows: []}); + await f.publish(); + assert.equal(f.writes.length, 0); + + f.comments = [{id: 51, user: {login: "github-actions[bot]", type: "Bot"}, body: previous}]; + await f.publish(); + assert.equal(f.writes[0].method, "update"); + assert.equal(f.writes[0].comment_id, 51); + assert.ok(!f.writes[0].body.includes("70%")); + assert.ok(row(f.writes[0].body, "Web").includes("Not run for this change")); +}); + +test("renders pending, skipped, failed and missing reports without old values", async () => { + const f = fixture(); + f.jobs.find((job) => job.name.startsWith("Web")).conclusion = "skipped"; + f.jobs.find((job) => job.name.startsWith("React Native")).conclusion = "failure"; + await f.publish(); + const body = f.writes[0].body; + assert.match(body, /\| ⏭️ \| Web \| — \| — \| — \| Not run for this change/); + assert.match(body, /\| ❌ \| React Native \| — \| — \| — \| Tests or coverage collection failed/); + assert.ok(body.includes("Coverage report unavailable")); + f.jobs.find((job) => job.name.startsWith("Protocol")).status = "in_progress"; + f.jobs.find((job) => job.name.startsWith("Protocol")).conclusion = null; + await f.publish(); + assert.match(f.writes[1].body, /\| ⏳ \| Embedded Checkout Protocol \(TS\) \| — \| — \| — \| Waiting for coverage/); + assert.ok(!body.includes("shields.io")); +}); + +test("failed tests may retain their measured coverage without looking successful", async () => { + const f = fixture(); + f.add("web", {state: "failed"}); + await f.publish(); + assert.match(f.writes[0].body, /\| ❌ \| Web \| 70% \| 70% \| 70% \| Tests or coverage collection failed/); +}); + +test("partial reruns reuse successful jobs but reject old results for rerun jobs", async () => { + const f = fixture(); + f.runs[0].run_attempt = 2; + f.add("web"); + f.add("react-native"); + f.jobs.push({id: 500, name: "React Native / Run jest tests", run_attempt: 2, status: "completed", conclusion: "failure"}); + await f.publish(); + const body = f.writes[0].body; + assert.match(body, /\| ✅ \| Web \| 70%/); + assert.ok(!row(body, "React Native").includes("70%")); +}); + +test("newer CI runs supersede older results even at the same commit", async () => { + const f = fixture(); + f.add("web"); + f.add("protocol"); + await f.publish(); + f.comments = [{id: 51, user: {login: "github-actions[bot]", type: "Bot"}, body: f.writes[0].body}]; + f.writes = []; + f.runs.push({...f.runs[0], id: 101, status: "in_progress"}); + f.jobs = []; + await f.publish(); + assert.ok(!f.writes[0].body.includes("70%")); +}); + +test("coalesced notifications rebuild from all durable platform results", async () => { + const f = fixture(); + f.add("protocol"); + f.add("web"); + // The event that happened to survive concurrency queuing is irrelevant. + f.context = {...f.context, eventName: "workflow_run", payload: {workflow_run: {head_sha: "an-older-event"}}}; + await f.publish(); + const body = f.writes[0].body; + assert.equal((body.match(/\| ✅ /g) || []).length, 2); +}); + +test("rejects malformed, stale, oversized and unsafe results", async () => { + for (const mutate of [ + (result) => { result.headSha = "old"; }, + (result) => { result.pr = 99; }, + (result) => { result.source.runAttempt = 2; }, + (result) => { result.rows[0][1] = 11; }, + (result) => { result.rows[0][0] = "@everyone"; }, + (result) => { result.reportUrl = "https://evil.example/report"; }, + ]) { + const f = fixture(); + const check = f.add("web"); + const result = JSON.parse(check.output.text); + mutate(result); + check.output.text = JSON.stringify(result); + await f.publish(); + assert.equal(f.warnings.length, 1); + assert.ok(!f.writes[0].body.includes("70%")); + } + const f = fixture(); + f.add("web").output.text = "x".repeat(32769); + await f.publish(); + assert.equal(f.warnings.length, 1); +}); + +test("accepts artifact links only for the matching run and ignores untrusted check apps", async () => { + const f = fixture(); + f.add("web", {reportUrl: "https://github.com/example/sdk/actions/runs/100/artifacts/123"}); + f.add("protocol", {reportUrl: "https://github.com/example/sdk/actions/runs/100/artifacts/456"}); + f.add("react-native").app.slug = "untrusted-app"; + await f.publish(); + assert.equal(f.warnings.length, 0); + assert.equal((f.writes[0].body.match(/Full report/g) || []).length, 2); + assert.ok(!row(f.writes[0].body, "React Native").includes("70%")); +}); + +test("does not mistake another skipped React Native job for the actual test job", async () => { + const f = fixture(); + f.add("react-native"); + f.jobs.push({id: 999, name: "React Native", status: "completed", conclusion: "skipped"}); + await f.publish(); + assert.ok(row(f.writes[0].body, "React Native").includes("70%")); +}); + +test("head changes during collection and closed, draft or fork PRs cannot be written", async () => { + for (const scenario of ["head", "closed", "draft", "fork", "base", "run"]) { + const f = fixture(); + if (scenario === "closed") f.pr.state = "closed"; + if (scenario === "draft") f.pr.draft = true; + if (scenario === "fork") f.pr.head.repo.full_name = "contributor/sdk"; + let gets = 0; + f.github.rest.pulls.get = async () => { + if (++gets === 2) { + if (scenario === "head") f.pr.head.sha = "new-head"; + if (scenario === "base") f.pr.base.sha = "new-base"; + if (scenario === "run") f.runs[0].run_attempt++; + } + return {data: structuredClone(f.pr)}; + }; + await f.publish(); + assert.equal(f.writes.length, 0, scenario); + } +}); + +test("resolves CI and manual notifications, rejecting unrelated workflows", async () => { + const f = fixture(); + assert.equal(await reporter.resolvePR(f), 12); + f.context = {...f.context, eventName: "workflow_dispatch", payload: {inputs: {pr: "12"}}}; + assert.equal(await reporter.resolvePR(f), 12); + f.context = {...f.context, eventName: "workflow_run", payload: {workflow_run: { + event: "pull_request", path: ".github/workflows/ci.yml", head_sha: "head", head_repository: {id: 1}, pull_requests: [], + }}}; + assert.equal(await reporter.resolvePR(f), 12); + f.context.payload.workflow_run.path = ".github/workflows/other.yml"; + assert.equal(await reporter.resolvePR(f), null); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e8a97972..324c99a93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,9 @@ jobs: # Common React Native infra: JS-level jobs (jest, lint, packed # files) and shared config that should trigger all RN jobs. reactNativeCommon: + - '.github/actions/report-coverage/**' + - 'scripts/report_coverage' + - 'scripts/lib/coverage_report.rb' - '.github/workflows/rn-test.yml' - '.github/workflows/rn-check-packed-files.yml' - '.github/workflows/rn-lint.yml' @@ -87,6 +90,9 @@ jobs: - '.github/workflows/rn-test-android.yml' - '.github/workflows/rn-build-android.yml' web: + - '.github/actions/report-coverage/**' + - 'scripts/report_coverage' + - 'scripts/lib/coverage_report.rb' - '.github/workflows/web.yml' - '.github/actions/setup/**' - '.ci/changed-file-filters.yml' @@ -97,11 +103,18 @@ jobs: - '.ci/changed-file-filters.yml' - '.github/workflows/ci.yml' protocol: + - '.github/actions/report-coverage/**' + - 'scripts/report_coverage' + - 'scripts/lib/coverage_report.rb' - '.github/workflows/protocol-test.yml' - '.github/actions/setup/**' - '.ci/changed-file-filters.yml' - '.github/workflows/ci.yml' scripts: + - '.github/scripts/*coverage*' + - '.github/workflows/coverage-report.yml' + - '.github/actions/report-coverage/**' + - 'scripts/report_coverage' - '.github/scripts/measure-package-size' - '.github/scripts/*bundle-size*' - '.github/workflows/package-size.yml' @@ -151,7 +164,8 @@ jobs: if: needs.changes.outputs.reactNative == 'true' permissions: contents: read - pull-requests: write + checks: write + pull-requests: read uses: ./.github/workflows/rn-test.yml rn-test-android: @@ -184,7 +198,8 @@ jobs: if: needs.changes.outputs.web == 'true' permissions: contents: read - pull-requests: write + checks: write + pull-requests: read uses: ./.github/workflows/web.yml telemetry-test: @@ -201,7 +216,8 @@ jobs: if: needs.changes.outputs.protocol == 'true' permissions: contents: read - pull-requests: write + checks: write + pull-requests: read uses: ./.github/workflows/protocol-test.yml scripts-test: @@ -215,7 +231,7 @@ jobs: - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0 with: ruby-version: .ruby-version - - run: node --test .github/scripts/*bundle-size*.test.cjs + - run: node --test .github/scripts/*bundle-size*.test.cjs .github/scripts/*coverage*.test.cjs - run: ./scripts/test_ruby - run: ./e2e/scripts/check_hide_keyboard_usage diff --git a/.github/workflows/coverage-report.yml b/.github/workflows/coverage-report.yml new file mode 100644 index 000000000..73b77cc5c --- /dev/null +++ b/.github/workflows/coverage-report.yml @@ -0,0 +1,65 @@ +name: Coverage Report + +on: + workflow_run: + workflows: [CI] + types: [requested, in_progress, completed] + workflow_dispatch: + inputs: + pr: + description: Pull request to refresh + required: true + type: number + +permissions: + contents: read + +jobs: + resolve: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + pr: ${{ steps.resolve.outputs.pr }} + steps: + # This publisher only executes trusted default-branch code. PR jobs supply + # numeric coverage data through check runs, never executable report code. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + id: resolve + with: + script: | + const {resolvePR} = require('./.github/scripts/publish-coverage.cjs'); + const pr = await resolvePR({github, context}); + if (pr) core.setOutput('pr', pr); + + publish: + needs: resolve + if: needs.resolve.outputs.pr != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + checks: read + pull-requests: write + concurrency: + group: coverage-comment-${{ needs.resolve.outputs.pr }} + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ needs.resolve.outputs.pr }} + with: + script: | + const {publish} = require('./.github/scripts/publish-coverage.cjs'); + await publish({github, context, core, prNumber: Number(process.env.PR_NUMBER)}); diff --git a/.github/workflows/protocol-test.yml b/.github/workflows/protocol-test.yml index b152ed764..df439fabd 100644 --- a/.github/workflows/protocol-test.yml +++ b/.github/workflows/protocol-test.yml @@ -11,7 +11,8 @@ jobs: timeout-minutes: 10 permissions: contents: read - pull-requests: write + checks: write + pull-requests: read defaults: run: working-directory: protocol @@ -27,13 +28,14 @@ jobs: working-directory: protocol - name: Test with coverage + id: coverage_tests run: pnpm test --coverage - - name: Coverage comment - if: github.event_name == 'pull_request' - uses: MishaKav/jest-coverage-comment@642ef024cc554a34b7082cea12c7bf63575ae151 # main + - name: Record coverage + if: ${{ !cancelled() }} + uses: ./.github/actions/report-coverage with: - coverage-summary-path: protocol/coverage/coverage-summary.json - title: 'Protocol — Coverage Report' - unique-id-for-comment: protocol - create-new-comment: false + platform: protocol + report-path: protocol/coverage/coverage-summary.json + report-directory: protocol/coverage + outcome: ${{ steps.coverage_tests.outcome }} diff --git a/.github/workflows/rn-test.yml b/.github/workflows/rn-test.yml index 21c397fa6..4a7e8cd06 100644 --- a/.github/workflows/rn-test.yml +++ b/.github/workflows/rn-test.yml @@ -11,7 +11,8 @@ jobs: timeout-minutes: 10 permissions: contents: read - pull-requests: write + checks: write + pull-requests: read defaults: run: working-directory: platforms/react-native @@ -27,16 +28,17 @@ jobs: working-directory: platforms/react-native - name: Run tests with coverage + id: coverage_tests run: pnpm test --coverage --testPathPatterns="modules/@shopify/checkout-kit-react-native/tests" --coverageReporters=json-summary - - name: Coverage comment - if: github.event_name == 'pull_request' - uses: MishaKav/jest-coverage-comment@642ef024cc554a34b7082cea12c7bf63575ae151 # main + - name: Record coverage + if: ${{ !cancelled() }} + uses: ./.github/actions/report-coverage with: - coverage-summary-path: platforms/react-native/coverage/coverage-summary.json - title: 'React Native — Coverage Report' - unique-id-for-comment: react-native - create-new-comment: false + platform: react-native + report-path: platforms/react-native/coverage/coverage-summary.json + report-directory: platforms/react-native/coverage + outcome: ${{ steps.coverage_tests.outcome }} # Runs without --coverage so the coverage report above stays scoped to the # published module. diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 9deb8ea53..4236e8642 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -11,7 +11,8 @@ jobs: timeout-minutes: 10 permissions: contents: read - pull-requests: write + checks: write + pull-requests: read defaults: run: working-directory: platforms/web @@ -30,16 +31,17 @@ jobs: run: pnpm lint - name: Test + id: coverage_tests run: pnpm test - - name: Coverage comment - if: github.event_name == 'pull_request' - uses: MishaKav/jest-coverage-comment@642ef024cc554a34b7082cea12c7bf63575ae151 # main + - name: Record coverage + if: ${{ !cancelled() }} + uses: ./.github/actions/report-coverage with: - coverage-summary-path: platforms/web/coverage/coverage-summary.json - title: 'Web — Coverage Report' - unique-id-for-comment: web - create-new-comment: false + platform: web + report-path: platforms/web/coverage/coverage-summary.json + report-directory: platforms/web/coverage + outcome: ${{ steps.coverage_tests.outcome }} - name: Build run: pnpm build diff --git a/scripts/lib/coverage_report.rb b/scripts/lib/coverage_report.rb new file mode 100644 index 000000000..aee888cb0 --- /dev/null +++ b/scripts/lib/coverage_report.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require "json" +require_relative "json_http_client" + +class CoverageReport + JAVASCRIPT_METRICS = {"lines" => "Lines", "statements" => "Statements", "branches" => "Branches", "functions" => "Functions"}.freeze + TITLES = {"web" => "Web", "react-native" => "React Native", "protocol" => "Embedded Checkout Protocol (TypeScript)"}.freeze + + attr_reader :platform, :rows + + def initialize(platform, contents) + @platform = platform + @rows = case platform + when "web", "react-native", "protocol" then javascript_rows(contents) + else raise ArgumentError, "Unknown coverage platform: #{platform}" + end + end + + def marker + "" + end + + def markdown(report_url: nil) + label = TITLES.fetch(platform) + lines = [marker, "# #{label} — Coverage Report", ""] + lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| #{@rows.map { "---" }.join(" | ")} |"]) + cells = @rows.map do |name, covered, total| + metric(covered, total, badge: name == "Lines", report_url: report_url) + end + lines << "| #{cells.join(' | ')} |" + lines.concat(["", "[Full coverage reports](#{report_url})"]) unless report_url.to_s.empty? + lines.join("\n") + "\n" + end + + private + + def javascript_rows(contents) + totals = JSON.parse(contents).fetch("total") + JAVASCRIPT_METRICS.map do |key, label| + counts = totals.fetch(key) + row(label, counts.fetch("covered"), counts.fetch("total")) + end + end + + def metric(covered, total, badge:, report_url:) + return "N/A" if total.zero? && badge + return "N/A (0/0)" if total.zero? + + percentage = 100.0 * covered / total + formatted = format("%.2f", percentage).sub(/\.?0+\z/, "") + counts = "(#{covered}/#{total})" + return "#{formatted}% #{counts}" unless badge + + # Match the existing jest-coverage-comment badge palette. Colors describe + # coverage, independently of whether a platform enforces a threshold. + color = case percentage + when 0...40 then "red" + when 40...60 then "orange" + when 60...80 then "yellow" + when 80...90 then "green" + else "brightgreen" + end + image = "![Coverage: #{formatted}%](https://img.shields.io/badge/Coverage-#{formatted}%25-#{color}.svg)" + image = "[#{image}](#{report_url})" unless report_url.to_s.empty? + image + end + + def row(label, covered, total) + unless covered.is_a?(Integer) && total.is_a?(Integer) && covered >= 0 && total >= covered + raise "Invalid coverage counts for #{label}" + end + + [label, covered, total] + end +end + +class CoverageResultPublisher + def initialize(repository:, pr_number:, sha:, token:, source:, client: nil) + @repository = repository + @pr_number = pr_number + @sha = sha + @source = source + @client = client || JsonHttpClient.new(host: "api.github.com", error_label: "GitHub", default_headers: {"Accept" => "application/vnd.github+json"}) do |request| + request["Authorization"] = "Bearer #{token}" + end + end + + def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: false) + pr = @client.get("/repos/#{@repository}/pulls/#{@pr_number}") + return "Skipped coverage result: pull request is closed or this build is superseded." unless pr["state"] == "open" && pr.dig("head", "sha") == @sha + return "Skipped coverage result: fork pull request." unless pr.dig("head", "repo", "full_name") == @repository + + checks = check_runs + external_id = "coverage:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}" + existing = checks.find { |check| check["external_id"] == external_id && check.dig("app", "slug") == @source.fetch("provider") } + unless existing && preserve_existing + result = {version: 1, platform: platform, pr: @pr_number.to_i, headSha: @sha, source: @source, state: state, rows: report&.rows || [], reportUrl: report_url} + payload = { + name: "Coverage — #{CoverageReport::TITLES.fetch(platform)}", + external_id: external_id, + status: "completed", + conclusion: state == "skipped" ? "skipped" : "neutral", + output: { + title: "#{CoverageReport::TITLES.fetch(platform)} coverage", + summary: report ? report.markdown(report_url: report_url) : "Coverage #{state}.", + text: JSON.generate(result) + } + } + if existing + @client.patch_json("/repos/#{@repository}/check-runs/#{existing.fetch('id')}", payload) + else + @client.post_json("/repos/#{@repository}/check-runs", payload.merge(head_sha: @sha)) + end + end + "Published #{platform} coverage result." + end + + private + + def check_runs + results = [] + page = 1 + loop do + checks = @client.get("/repos/#{@repository}/commits/#{@sha}/check-runs?filter=all&per_page=100&page=#{page}").fetch("check_runs") + results.concat(checks) + return results if checks.length < 100 + + page += 1 + end + end +end diff --git a/scripts/report_coverage b/scripts/report_coverage new file mode 100644 index 000000000..7b980ecdd --- /dev/null +++ b/scripts/report_coverage @@ -0,0 +1,35 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "lib/coverage_report" + +abort "Usage: ruby scripts/report_coverage " unless ARGV.length == 2 +platform = ARGV[0] +outcome = ENV.fetch("COVERAGE_TEST_OUTCOME", "success") +state = outcome == "success" ? "success" : "failed" +report = CoverageReport.new(platform, File.read(ARGV[1])) if File.file?(ARGV[1]) && %w[success failure].include?(outcome) +state = "unavailable" if state == "success" && report.nil? +sha = ENV["COVERAGE_SHA"] +body = report ? report.markdown(report_url: ENV["COVERAGE_REPORT_URL"]) : "#{CoverageReport::TITLES.fetch(platform)} coverage #{state}." +puts body +summary_path = ENV["GITHUB_STEP_SUMMARY"] +File.open(summary_path, "a") { |file| file.puts(body) } unless summary_path.to_s.empty? + +pr_number = ENV["COVERAGE_PR"] +token = ENV["COVERAGE_TOKEN"] +if pr_number.to_s.empty? || token.to_s.empty? + puts "Skipped coverage result: no pull request or GitHub write token." + exit +end +abort "COVERAGE_PR must be a pull request number" unless pr_number.match?(/\A[1-9]\d*\z/) +abort "COVERAGE_SHA is required to publish coverage" if sha.to_s.empty? + +source = {"provider" => "github-actions", "runId" => Integer(ENV.fetch("GITHUB_RUN_ID")), "runAttempt" => Integer(ENV.fetch("GITHUB_RUN_ATTEMPT", "1"))} +publisher = CoverageResultPublisher.new( + repository: ENV.fetch("COVERAGE_REPOSITORY"), + pr_number: pr_number, + sha: sha, + token: token, + source: source +) +puts publisher.publish(platform: platform, state: state, report: report, report_url: ENV["COVERAGE_REPORT_URL"]) diff --git a/scripts/test/coverage_report_test.rb b/scripts/test/coverage_report_test.rb new file mode 100644 index 000000000..68d271d85 --- /dev/null +++ b/scripts/test/coverage_report_test.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "open3" +require "tmpdir" +require_relative "../lib/coverage_report" + +class CoverageReportTest < Minitest::Test + def javascript_json(covered: 7, total: 10) + JSON.generate("total" => CoverageReport::JAVASCRIPT_METRICS.keys.to_h do |key| + [key, {"covered" => covered, "total" => total}] + end) + end + + def test_javascript_reports_preserve_metric_names_and_summary_format + %w[web react-native protocol].each do |platform| + report = CoverageReport.new(platform, javascript_json) + assert_equal ["Lines", "Statements", "Branches", "Functions"], report.rows.map(&:first) + assert_includes report.markdown, "70% (7/10)" + assert_includes report.markdown, "Coverage-70%25-yellow.svg" + end + end + + def test_empty_metrics_are_not_reported_as_full_coverage + report = CoverageReport.new("web", javascript_json(covered: 0, total: 0)) + assert_includes report.markdown, "| N/A | N/A (0/0) |" + refute_includes report.markdown, "100%" + end + + def test_missing_or_impossible_counts_fail + assert_raises(KeyError) { CoverageReport.new("web", '{"total":{}}') } + assert_raises(RuntimeError) { CoverageReport.new("web", javascript_json(covered: 11)) } + assert_raises(RuntimeError) { CoverageReport.new("web", javascript_json(covered: -1)) } + end + + def test_cli_keeps_a_job_summary_without_a_comment_token + Dir.mktmpdir("coverage report ") do |directory| + input = File.join(directory, "coverage-summary.json") + summary = File.join(directory, "summary.md") + File.write(input, javascript_json) + File.write(summary, "Existing summary\n") + output, error, status = Open3.capture3( + {"COVERAGE_TOKEN" => "", "COVERAGE_PR" => "123", "COVERAGE_SHA" => "abc123", + "COVERAGE_REPORT_URL" => "https://example.com/reports", "GITHUB_STEP_SUMMARY" => summary}, + "ruby", File.expand_path("../report_coverage", __dir__), "web", input + ) + + assert status.success?, "#{output}#{error}" + assert_includes output, "Skipped coverage result" + markdown = File.read(summary) + assert markdown.start_with?("Existing summary\n") + assert_includes markdown, "[![Coverage: 70%](https://img.shields.io/badge/Coverage-70%25-yellow.svg)](https://example.com/reports)" + assert_includes markdown, "[Full coverage reports](https://example.com/reports)" + end + end +end + +class CoverageResultPublisherTest < Minitest::Test + class FakeClient + attr_reader :writes + + def initialize(pr, checks) + @pr = pr + @checks = checks + @writes = [] + end + + def get(path) + return @pr if path == "/repos/example/sdk/pulls/123" + + {"check_runs" => @checks} + end + + def post_json(path, body) + @writes << [:post, path, body] + end + + def patch_json(path, body) + @writes << [:patch, path, body] + end + end + + def setup + @report = CoverageReport.new("web", JSON.generate("total" => CoverageReport::JAVASCRIPT_METRICS.keys.to_h do |key| + [key, {"covered" => 1, "total" => 2}] + end)) + @pr = {"state" => "open", "head" => {"sha" => "abc123", "repo" => {"full_name" => "example/sdk"}}} + @source = {"provider" => "github-actions", "runId" => 100, "runAttempt" => 2} + end + + def publish(checks = [], **options) + @client = FakeClient.new(@pr, checks) + CoverageResultPublisher.new(repository: "example/sdk", pr_number: 123, sha: "abc123", token: "test-token", source: @source, client: @client) + .publish(platform: "web", state: "success", report: @report, **options) + end + + def test_persists_numeric_results_against_the_exact_run_without_a_pr_comment + assert_includes publish, "Published web" + method, path, payload = @client.writes.fetch(0) + assert_equal :post, method + assert_equal "/repos/example/sdk/check-runs", path + assert_equal "neutral", payload[:conclusion] + assert_equal "abc123", payload[:head_sha] + assert_equal "coverage:web:100:2", payload[:external_id] + result = JSON.parse(payload[:output][:text]) + assert_equal 123, result.fetch("pr") + assert_equal @source, result.fetch("source") + assert_equal [["Lines", 1, 2], ["Statements", 1, 2], ["Branches", 1, 2], ["Functions", 1, 2]], result.fetch("rows") + assert_equal 1, @client.writes.length + end + + def test_updates_the_matching_check_without_creating_duplicates + publish([{"id" => 42, "external_id" => "coverage:web:100:2", "app" => {"slug" => "github-actions"}}]) + assert_equal [:patch, "/repos/example/sdk/check-runs/42"], @client.writes.first.take(2) + end + + def test_superseded_closed_and_fork_prs_are_not_published + ["stale", "closed", "fork"].each do |scenario| + setup + @pr["head"]["sha"] = "new-head" if scenario == "stale" + @pr["state"] = "closed" if scenario == "closed" + @pr["head"]["repo"]["full_name"] = "contributor/sdk" if scenario == "fork" + assert_includes publish, "Skipped coverage result" + assert_empty @client.writes + end + end + +end