diff --git a/.ci/changed-file-filters.yml b/.ci/changed-file-filters.yml index 6d26256c9..62824dbe4 100644 --- a/.ci/changed-file-filters.yml +++ b/.ci/changed-file-filters.yml @@ -61,10 +61,14 @@ ciFilters: # decide which macOS jobs run, so a change to one has to rerun them all, while a Maestro # flow edit under e2e/ must not. iosCiConfig: + - 'scripts/report_coverage' + - 'scripts/lib/coverage_report.rb' - 'e2e/bitrise.yml' - 'e2e/config/ios_ci.yml' - 'e2e/lib/ios_ci_run_plan.rb' - 'e2e/lib/ios_ci_reporter.rb' + - 'e2e/lib/swift_coverage_status.rb' - 'e2e/scripts/ios_ci_run_plan' - 'e2e/scripts/report_ios_ci_results' + - 'e2e/scripts/report_swift_coverage_status' - 'e2e/scripts/bitrise_ci_helpers' diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d9ed31d51..23230bede 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -96,12 +96,19 @@ pnpm --dir protocol test --coverage A failing threshold fails the test command and its CI job. Add tests for the uncovered behavior instead of lowering the threshold. The thresholds live in `platforms/web/vite.config.ts`, `platforms/react-native/jest.config.js`, and -`protocol/vitest.config.ts`. Swift, Android, and standalone telemetry tests do -not yet enforce coverage thresholds. +`protocol/vitest.config.ts`. + +Swift and Android generate coverage reports without enforcing a minimum yet: + +| Platform | Local command | Reports | CI artifact | +| --- | --- | --- | --- | +| Android | `dev android test coverage` | `platforms/android/lib/build/reports/coverage/test/debug/` (HTML and XML) | `android-coverage-` in the GitHub Actions Android test job | +| Swift | `dev swift test coverage` | `.xcresults/coverage/` (JSON and text) | Coverage reports in the Bitrise Swift package test job | CI updates one **Coverage Report** comment on same-repository pull requests for -Web, React Native JavaScript, and Embedded Checkout Protocol (TS). The table -shows status emojis, line, branch, and function coverage percentages, and links +Web, React Native JavaScript, Android, both Swift SDK targets, and Embedded +Checkout Protocol (TS, Kotlin, and Swift). The table shows status emojis, line, +branch, and function/method coverage percentages where available, and links to the full reports. Jobs that are pending, skipped, failed, or missing a report show their status instead of coverage from an older run. GitHub Actions jobs also write their coverage tables to the job summary. Fork builds and builds without a @@ -115,15 +122,27 @@ without a delta. Baseline jobs do not gate the PR or replace its test results. Platform jobs store numeric results in GitHub checks. The `Coverage Report` workflow is the only comment writer and serializes updates per PR. It refreshes -on CI workflow events, reading all current results so simultaneous completions -do not lose data. A manual run of `Coverage Report` with the PR number refreshes -interrupted reporting. +on CI workflow events and Bitrise's `coverage-updated` repository dispatch, +reading all current results so simultaneous completions do not lose data. +Bitrise uses its existing GitHub App token (checks and contents write permissions) +and records skipped/failed Swift jobs when the iOS pipeline finishes. A manual +run of `Coverage Report` with the PR number refreshes interrupted reporting. The publisher executes trusted default-branch code and validates the numeric results; it never executes PR-provided reporting code. This workflow begins publishing once it exists on `main`. Coverage thresholds remain in the test runners; combining the comments does not change their enforcement. +Android reports cover the library's debug unit tests, including Robolectric +tests and the library's telemetry code. Swift reports contain per-target and +per-file line coverage from the package tests; inspect `ShopifyCheckoutKit` and +`ShopifyAcceleratedCheckouts` for the SDK coverage. Xcode also lists test and +dependency targets, sometimes with no executable lines; compare the named +production targets rather than the overall total. The full Swift result bundle +is available locally at `.xcresults/ShopifyCheckoutKit-Package-coverage.xcresult`. +Neither report includes sample app or end-to-end tests. Standalone TypeScript +telemetry tests do not yet collect coverage. + ### Testing PR builds with Tophat [Tophat](https://github.com/Shopify/tophat) is a macOS menu-bar app that diff --git a/.github/actions/report-coverage/action.yml b/.github/actions/report-coverage/action.yml index 1f1d5d32b..97a109e2a 100644 --- a/.github/actions/report-coverage/action.yml +++ b/.github/actions/report-coverage/action.yml @@ -6,7 +6,7 @@ inputs: description: Coverage platform key report-path: required: true - description: Coverage summary JSON, relative to the repository root + description: Coverage summary JSON or JaCoCo XML, relative to the repository root report-directory: required: true description: Directory containing the downloadable reports diff --git a/.github/scripts/publish-coverage.cjs b/.github/scripts/publish-coverage.cjs index 07707e9f7..d35e3a7d6 100644 --- a/.github/scripts/publish-coverage.cjs +++ b/.github/scripts/publish-coverage.cjs @@ -3,16 +3,22 @@ const jsMetrics = ["Lines", "Statements", "Branches", "Functions"]; const platforms = [ {id: "web", title: "Web", job: "Web / Lint, test, build, verify", metrics: jsMetrics}, {id: "react-native", title: "React Native", job: "React Native / Run jest tests", metrics: jsMetrics}, + {id: "android", title: "Android", job: "Android / test", metrics: ["Lines", "Instructions", "Branches", "Methods"]}, + {id: "swift", title: "Swift", metrics: ["ShopifyCheckoutKit", "ShopifyAcceleratedCheckouts"]}, {id: "protocol", title: "Embedded Checkout Protocol (TypeScript)", displayTitle: "Embedded Checkout Protocol (TS)", job: "Protocol / Test", skippedJob: "Protocol", metrics: jsMetrics}, + {id: "protocol-kotlin", title: "Embedded Checkout Protocol (Kotlin)", job: "Android / test", skippedJob: "Android", metrics: ["Lines", "Instructions", "Branches", "Methods"]}, + {id: "protocol-swift", title: "Embedded Checkout Protocol (Swift)", metrics: ["Lines", "Functions"]}, ]; const latest = (items) => [...items].sort((a, b) => b.id - a.id)[0]; const isComment = (comment) => comment.user?.type === "Bot" && comment.user.login === "github-actions[bot]" && comment.body?.startsWith(marker); +const pipelineCheck = (checks) => latest(checks.filter((check) => + check.app?.slug === "bitrise" && check.name === "ci/bitrise/ci-ios/pr")); async function resolvePR({github, context}) { - if (context.eventName === "workflow_dispatch") { - const pr = Number(context.payload.inputs?.pr); + if (context.eventName === "repository_dispatch" || context.eventName === "workflow_dispatch") { + const pr = Number(context.payload.client_payload?.pr ?? context.payload.inputs?.pr); return Number.isSafeInteger(pr) && pr > 0 ? pr : null; } const run = context.payload.workflow_run; @@ -34,14 +40,16 @@ async function sources(github, repo, pr) { }), ]); const run = latest(runs.filter((item) => item.head_sha === pr.head.sha && item.head_repository?.id === pr.head.repo.id && item.head_branch === pr.head.ref)); - return {run, checks}; + return {run, checks, pipeline: pipelineCheck(checks.filter((check) => check.head_sha === pr.head.sha))}; } -function reportURL(value, repo, run) { +function reportURL(value, repo, platform, run) { if (!value) return null; const url = new URL(value); const artifactPrefix = `/${repo.owner}/${repo.repo}/actions/runs/${run?.id}/artifacts/`; - const valid = url.hostname === "github.com" && url.pathname.startsWith(artifactPrefix) && /^[0-9]+$/.test(url.pathname.slice(artifactPrefix.length)); + const valid = !platform.job + ? url.hostname === "app.bitrise.io" && /^\/app\/[\w-]+\/build\/[\w-]+$/.test(url.pathname) + : url.hostname === "github.com" && url.pathname.startsWith(artifactPrefix) && /^[0-9]+$/.test(url.pathname.slice(artifactPrefix.length)); if (!valid || url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Invalid coverage report link"); return url.href; @@ -69,26 +77,34 @@ function readResult(check, platform, expected, pr, repo, run) { throw new Error("Invalid coverage counts"); }); } - return {...result, reportUrl: reportURL(result.reportUrl, repo, run)}; + return {...result, reportUrl: reportURL(result.reportUrl, repo, platform, run)}; } function platformResult(platform, snapshot, jobs, pr, repo, core, revision = "head") { - const {run, checks} = snapshot; - let state = "pending"; - if (!run) return {state}; - // A failed-jobs rerun reuses successful jobs from an earlier attempt. Match - // each result to the attempt that actually ran that platform's test job. + const {run, pipeline, checks} = snapshot; const baseline = revision === "base"; - const jobName = baseline ? `Coverage baseline (${platform.id}) / Test` : platform.job; - const skippedJob = baseline ? `Coverage baseline (${platform.id})` : platform.skippedJob || platform.title; - const job = latest(jobs.filter((item) => item.name === jobName)) || - latest(jobs.filter((item) => item.name === skippedJob)); - const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt, revision}; - const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas")); - if (plan?.status === "completed" && plan.conclusion !== "success") return {state: "unavailable"}; - if (job?.conclusion === "skipped") return {state: "skipped"}; - if (job?.status === "completed") state = job.conclusion === "success" ? "unavailable" : "failed"; - else if (!job && run.status === "completed") state = "unavailable"; + if (baseline && !platform.job) return {state: "unavailable"}; + let expected, state = "pending"; + if (!platform.job) { + if (!pipeline) return {state}; + expected = {provider: "bitrise", runId: pipeline.id, runAttempt: pipeline.external_id ?? 1}; + if (pipeline.status === "completed") state = pipeline.conclusion === "success" ? "unavailable" : "failed"; + } else { + if (!run) return {state}; + // A failed-jobs rerun reuses successful jobs from an earlier attempt. Match + // each result to the attempt that actually ran that platform's test job. + const jobName = baseline ? `Coverage baseline (${platform.id}) / Test` : platform.job; + const skippedJob = baseline ? `Coverage baseline (${platform.id})` : platform.skippedJob || platform.title; + const job = latest(jobs.filter((item) => item.name === jobName)) || + latest(jobs.filter((item) => item.name === skippedJob)); + expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt}; + const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas")); + if (plan?.status === "completed" && plan.conclusion !== "success") return {state: "unavailable"}; + if (job?.conclusion === "skipped") return {state: "skipped"}; + if (job?.status === "completed") state = job.conclusion === "success" ? "unavailable" : "failed"; + else if (!job && run.status === "completed") state = "unavailable"; + } + expected.revision = revision; const externalId = `coverage${baseline ? "-base" : ""}:${platform.id}:${expected.runId}:${expected.runAttempt}`; const check = latest(checks.filter((item) => item.app?.slug === expected.provider && item.external_id === externalId && item.name === `Coverage${baseline ? " base" : ""} — ${platform.title}`)); @@ -130,13 +146,18 @@ function render(results, baselines = {}) { const comparable = result.state === "success" && baseline?.state === "success" && result.baseSha && result.baseSha === baseline.baseSha; const value = (name) => metric(find(name), comparable ? baseline.rows.find((row) => row[0] === name) : undefined); - lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${value("Lines")} | ${value("Branches")} | ${value("Functions")} | ${report} |`); + if (platform.id === "swift") { + for (const target of platform.metrics) + lines.push(`| ${emoji} | Swift · ${target} | ${value(target)} | — | — | ${report} |`); + } else { + lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${value("Lines")} | ${value("Branches")} | ${value(find("Functions") ? "Functions" : "Methods")} | ${report} |`); + } } lines.push("", "Changes in parentheses are **percentage points versus the PR’s base commit**. No delta appears when matching base coverage is unavailable."); return lines.join("\n") + "\n"; } -const sourceKey = ({run}) => `${run?.id}:${run?.run_attempt}`; +const sourceKey = ({run, pipeline}) => `${run?.id}:${run?.run_attempt}:${pipeline?.id}:${pipeline?.external_id}`; async function publish({github, context, core, prNumber}) { const repo = context.repo; diff --git a/.github/scripts/publish-coverage.test.cjs b/.github/scripts/publish-coverage.test.cjs index f61c073c5..89e8a8999 100644 --- a/.github/scripts/publish-coverage.test.cjs +++ b/.github/scripts/publish-coverage.test.cjs @@ -10,8 +10,8 @@ function fixture() { const f = { pr: {number: 12, state: "open", draft: false, head: {sha: "head", ref: "feature", repo: {id: 1, full_name: "example/sdk"}}, base: {sha: "base"}}, runs: [{id: 100, run_attempt: 1, head_sha: "head", head_branch: "feature", head_repository: {id: 1}, status: "completed", conclusion: "success"}], - checks: [], - jobs: reporter.platforms.map((platform, i) => + checks: [{id: 200, head_sha: "head", name: "ci/bitrise/ci-ios/pr", app: {slug: "bitrise"}, status: "in_progress"}], + jobs: reporter.platforms.filter((platform, index, all) => platform.job && all.findIndex((other) => other.job === platform.job) === index).map((platform, i) => ({id: 300 + i, name: platform.job, run_attempt: 1, status: "completed", conclusion: "success"})), comments: [], writes: [], warnings: [], }; @@ -31,11 +31,11 @@ function fixture() { if (method === "prs") return [f.pr]; return structuredClone(f[method]); }}; - f.context = {repo: {owner: "example", repo: "sdk"}, eventName: "workflow_dispatch", payload: {inputs: {pr: "12"}}}; + f.context = {repo: {owner: "example", repo: "sdk"}, eventName: "repository_dispatch", payload: {client_payload: {pr: 12}}}; f.publish = () => reporter.publish({github: f.github, context: f.context, core: {warning: (message) => f.warnings.push(message)}, prNumber: 12}); f.add = (id, overrides = {}) => { const platform = reporter.platforms.find((platform) => platform.id === id); - const source = {provider: "github-actions", runId: 100, runAttempt: 1}; + const source = !platform.job ? {provider: "bitrise", runId: 200, runAttempt: 1} : {provider: "github-actions", runId: 100, runAttempt: 1}; const result = {version: 1, platform: id, pr: 12, headSha: "head", baseSha: "base", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides}; const baseline = result.revision === "base"; const check = { @@ -123,7 +123,7 @@ test("malformed and untrusted baselines cannot hide valid head coverage", async } }); -test("one comment combines the existing JavaScript reports without native coverage rows", async () => { +test("one comment combines every platform with native metric names and no obsolete boilerplate", async () => { const f = fixture(); for (const platform of reporter.platforms) f.add(platform.id); await f.publish(); @@ -131,10 +131,11 @@ test("one comment combines the existing JavaScript reports without native covera assert.equal(f.writes.length, 1); assert.ok(body.startsWith(reporter.marker)); assert.ok(body.includes("| Status | Platform / target | Lines | Branches | Functions | Report |")); - for (const platform of reporter.platforms.filter((item) => item.id !== "protocol")) + for (const platform of reporter.platforms.filter((item) => !["swift", "protocol"].includes(item.id))) assert.ok(body.includes(`| ✅ | ${platform.title} | 70% |`)); assert.ok(body.includes("| ✅ | Embedded Checkout Protocol (TS) | 70% | 70% | 70% |")); - assert.ok(!/Android|Swift|Kotlin/.test(body)); + assert.ok(body.includes("| ✅ | Swift · ShopifyAcceleratedCheckouts | 70% | — | — |")); + assert.ok(body.includes("| ✅ | Embedded Checkout Protocol (Swift) | 70% | — | 70% |")); assert.ok(!body.includes("shields.io")); assert.ok(!body.includes("
")); assert.ok(!body.includes("PR head:")); @@ -196,16 +197,13 @@ test("skips a new comment when every reported platform is skipped but clears exi test("renders pending, skipped, failed and missing reports without old values", async () => { const f = fixture(); f.jobs.find((job) => job.name.startsWith("Web")).conclusion = "skipped"; - f.jobs.find((job) => job.name.startsWith("React Native")).conclusion = "failure"; + f.jobs.find((job) => job.name.startsWith("Android")).conclusion = "failure"; await f.publish(); const body = f.writes[0].body; assert.match(body, /\| ⏭️ \| Web \| — \| — \| — \| Not run for this change/); - assert.match(body, /\| ❌ \| React Native \| — \| — \| — \| Tests or coverage collection failed/); + assert.match(body, /\| ❌ \| Android \| — \| — \| — \| Tests or coverage collection failed/); + assert.match(body, /\| ⏳ \| Swift · ShopifyCheckoutKit \| — \| — \| — \| Waiting for coverage/); assert.ok(body.includes("Coverage report unavailable")); - f.jobs.find((job) => job.name.startsWith("Protocol")).status = "in_progress"; - f.jobs.find((job) => job.name.startsWith("Protocol")).conclusion = null; - await f.publish(); - assert.match(f.writes[1].body, /\| ⏳ \| Embedded Checkout Protocol \(TS\) \| — \| — \| — \| Waiting for coverage/); assert.ok(!body.includes("shields.io")); }); @@ -220,22 +218,23 @@ test("partial reruns reuse successful jobs but reject old results for rerun jobs const f = fixture(); f.runs[0].run_attempt = 2; f.add("web"); - f.add("react-native"); - f.jobs.push({id: 500, name: "React Native / Run jest tests", run_attempt: 2, status: "completed", conclusion: "failure"}); + f.add("android"); + f.jobs.push({id: 500, name: "Android / test", run_attempt: 2, status: "completed", conclusion: "failure"}); await f.publish(); const body = f.writes[0].body; assert.match(body, /\| ✅ \| Web \| 70%/); - assert.ok(!row(body, "React Native").includes("70%")); + assert.ok(!row(body, "Android").includes("70%")); }); -test("newer CI runs supersede older results even at the same commit", async () => { +test("newer CI and Bitrise runs supersede older results even at the same commit", async () => { const f = fixture(); f.add("web"); - f.add("protocol"); + f.add("swift"); await f.publish(); f.comments = [{id: 51, user: {login: "github-actions[bot]", type: "Bot"}, body: f.writes[0].body}]; f.writes = []; f.runs.push({...f.runs[0], id: 101, status: "in_progress"}); + f.checks.push({...f.checks[0], id: 201}); f.jobs = []; await f.publish(); assert.ok(!f.writes[0].body.includes("70%")); @@ -243,13 +242,22 @@ test("newer CI runs supersede older results even at the same commit", async () = test("coalesced notifications rebuild from all durable platform results", async () => { const f = fixture(); - f.add("protocol"); + f.add("swift"); f.add("web"); // The event that happened to survive concurrency queuing is irrelevant. - f.context = {...f.context, eventName: "workflow_run", payload: {workflow_run: {head_sha: "an-older-event"}}}; + f.context.payload.client_payload.sha = "an-older-event"; await f.publish(); const body = f.writes[0].body; - assert.equal((body.match(/\| ✅ /g) || []).length, 2); + assert.equal((body.match(/\| ✅ /g) || []).length, 3); +}); + +test("a retried Bitrise attempt cannot reuse the previous attempt's coverage", async () => { + const f = fixture(); + f.add("swift"); + f.checks[0].external_id = "apps/example/pipelines/current/attempts/retry"; + await f.publish(); + assert.ok(!f.writes[0].body.includes("70%")); + assert.ok(f.writes[0].body.includes("Waiting for coverage")); }); test("rejects malformed, stale, oversized and unsafe results", async () => { @@ -279,12 +287,12 @@ test("rejects malformed, stale, oversized and unsafe results", async () => { test("accepts artifact links only for the matching run and ignores untrusted check apps", async () => { const f = fixture(); f.add("web", {reportUrl: "https://github.com/example/sdk/actions/runs/100/artifacts/123"}); - f.add("protocol", {reportUrl: "https://github.com/example/sdk/actions/runs/100/artifacts/456"}); - f.add("react-native").app.slug = "untrusted-app"; + f.add("swift", {reportUrl: "https://app.bitrise.io/app/example/build/abc-123"}); + f.add("android").app.slug = "untrusted-app"; await f.publish(); assert.equal(f.warnings.length, 0); - assert.equal((f.writes[0].body.match(/Full report/g) || []).length, 2); - assert.ok(!row(f.writes[0].body, "React Native").includes("70%")); + assert.equal((f.writes[0].body.match(/Full report/g) || []).length, 3); + assert.ok(!row(f.writes[0].body, "Android").includes("70%")); }); test("does not mistake another skipped React Native job for the actual test job", async () => { @@ -315,7 +323,7 @@ test("head changes during collection and closed, draft or fork PRs cannot be wri } }); -test("resolves CI and manual notifications, rejecting unrelated workflows", async () => { +test("resolves CI, Bitrise and manual notifications, rejecting unrelated workflows", async () => { const f = fixture(); assert.equal(await reporter.resolvePR(f), 12); f.context = {...f.context, eventName: "workflow_dispatch", payload: {inputs: {pr: "12"}}}; @@ -327,3 +335,18 @@ test("resolves CI and manual notifications, rejecting unrelated workflows", asyn f.context.payload.workflow_run.path = ".github/workflows/other.yml"; assert.equal(await reporter.resolvePR(f), null); }); + +test("native protocol results retain their own state while sharing a runner", async () => { + const f = fixture(); + f.add("android", {state: "failed"}); + f.add("protocol-kotlin"); + f.add("swift", {state: "failed"}); + f.add("protocol-swift", {reportUrl: "https://app.bitrise.io/app/example/build/abc-123"}); + await f.publish(); + const body = f.writes[0].body; + assert.ok(row(body, "Android").startsWith("| ❌")); + assert.ok(row(body, "Embedded Checkout Protocol (Kotlin)").startsWith("| ✅")); + assert.ok(row(body, "Swift · ShopifyCheckoutKit").startsWith("| ❌")); + assert.ok(row(body, "Embedded Checkout Protocol (Swift)").includes("[Full report]")); + assert.equal(f.warnings.length, 0); +}); diff --git a/.github/workflows/android-test.yml b/.github/workflows/android-test.yml index 574cda06f..a6dc59064 100644 --- a/.github/workflows/android-test.yml +++ b/.github/workflows/android-test.yml @@ -11,6 +11,10 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + contents: read + checks: write + pull-requests: read defaults: run: working-directory: platforms/android @@ -28,12 +32,33 @@ jobs: java-version: 17 cache: 'gradle' - - name: Run Tests - run: ./gradlew test --console=plain + - name: Run tests and generate library coverage + id: coverage_tests + run: ./gradlew test :lib:createDebugUnitTestCoverageReport --console=plain - - name: Run Protocol Unit Tests + - name: Record coverage + if: ${{ !cancelled() }} + uses: ./.github/actions/report-coverage + with: + platform: android + report-path: platforms/android/lib/build/reports/coverage/test/debug/report.xml + report-directory: platforms/android/lib/build/reports/coverage/test/debug/ + outcome: ${{ steps.coverage_tests.outcome }} + + - name: Run Embedded Checkout Protocol tests with coverage + id: protocol_coverage_tests + if: ${{ !cancelled() }} working-directory: protocol/languages/kotlin - run: ./gradlew test --console=plain + run: ./gradlew test jacocoTestReport --console=plain + + - name: Record Embedded Checkout Protocol coverage + if: ${{ !cancelled() }} + uses: ./.github/actions/report-coverage + with: + platform: protocol-kotlin + report-path: protocol/languages/kotlin/embedded-checkout-protocol/build/reports/jacoco/test/jacocoTestReport.xml + report-directory: protocol/languages/kotlin/embedded-checkout-protocol/build/reports/jacoco/test/ + outcome: ${{ steps.protocol_coverage_tests.outcome }} - name: Setup sample app environment id: sample_env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bedf43bf..eea4837db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,9 @@ jobs: with: filters: | android: + - '.github/actions/report-coverage/**' + - 'scripts/report_coverage' + - 'scripts/lib/coverage_report.rb' - '.github/workflows/android-publish.yml' - '.github/workflows/android-protocol-publish.yml' - '.github/workflows/android-test.yml' @@ -144,6 +147,10 @@ jobs: name: Android needs: changes if: needs.changes.outputs.android == 'true' + permissions: + contents: read + checks: write + pull-requests: read uses: ./.github/workflows/android-test.yml swift-lint: diff --git a/.github/workflows/coverage-report.yml b/.github/workflows/coverage-report.yml index 73b77cc5c..3658c709e 100644 --- a/.github/workflows/coverage-report.yml +++ b/.github/workflows/coverage-report.yml @@ -4,6 +4,8 @@ on: workflow_run: workflows: [CI] types: [requested, in_progress, completed] + repository_dispatch: + types: [coverage-updated] workflow_dispatch: inputs: pr: diff --git a/dev.yml b/dev.yml index c4010abb6..880e09e81 100644 --- a/dev.yml +++ b/dev.yml @@ -237,6 +237,16 @@ commands: *) echo "Usage: dev protocol test [kotlin|swift|typescript] [-- ]"; exit 1 ;; esac + coverage: + desc: "Run native protocol tests with coverage. Usage: dev protocol coverage " + syntax: "" + run: | + case "$1" in + swift) ./protocol/scripts/test_swift_coverage ;; + kotlin) protocol/languages/kotlin/gradlew -p protocol/languages/kotlin :embedded-checkout-protocol:jacocoTestReport --console=plain ;; + *) echo "Usage: dev protocol coverage "; exit 1 ;; + esac + lint: desc: "Lint Kotlin protocol source and TypeScript test files" run: | @@ -377,6 +387,14 @@ commands: platforms/android/gradlew -p platforms/android clean test --console=plain platforms/android/samples/CheckoutKitAndroidDemo/gradlew -p platforms/android/samples/CheckoutKitAndroidDemo :app:testDebugUnitTest --console=plain subcommands: + coverage: + desc: Run Android library unit tests and generate HTML/XML coverage reports + syntax: + optional: --local + run: | + source platforms/android/scripts/parse_local_protocol_flag + parse_local_protocol_flag "$@" + platforms/android/gradlew -p platforms/android :lib:createDebugUnitTestCoverageReport --console=plain samples: desc: Run JVM tests for the Checkout Kit Android sample app syntax: @@ -654,6 +672,9 @@ commands: ./Scripts/test_samples fi subcommands: + coverage: + desc: Run Swift package tests and export Xcode coverage reports + run: cd platforms/swift && ./Scripts/test_coverage sample: desc: Run the sample app test target syntax: "[test_class_name]" diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index e4227c212..402863c5c 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -342,14 +342,61 @@ workflows: - git-clone@8: {} - bundle::create-ios-simulator: {} - script@1: - title: Run Swift package tests + title: Run Swift package tests with coverage timeout: 3600 no_output_timeout: 1800 inputs: - content: |- set -euo pipefail cd platforms/swift - ./Scripts/xcode_run test ShopifyCheckoutKit-Package + ./Scripts/test_coverage + - deploy-to-bitrise-io@2: + is_always_run: false + inputs: + - deploy_path: "$BITRISE_SOURCE_DIR/.xcresults/coverage" + - script@1: + title: Record Swift coverage for the combined report + is_always_run: true + inputs: + - content: |- + set -euo pipefail + export COVERAGE_REPOSITORY="Shopify/checkout-kit" + export COVERAGE_PR="${BITRISE_PULL_REQUEST:-}" + export COVERAGE_SHA="${BITRISE_GIT_COMMIT:-}" + export COVERAGE_REPORT_URL="${BITRISE_BUILD_URL:-}" + export COVERAGE_TOKEN="${OVERRIDE_GITHUB_TOKEN:-${GIT_HTTP_PASSWORD:-}}" + export COVERAGE_TEST_OUTCOME=success + if [ "${BITRISE_BUILD_STATUS:-0}" != "0" ]; then + export COVERAGE_TEST_OUTCOME=failure + fi + ruby scripts/report_coverage swift .xcresults/coverage/coverage.json + + - script@1: + title: Run Embedded Checkout Protocol Swift coverage + inputs: + - content: |- + set -euo pipefail + ./protocol/scripts/test_swift_coverage + - deploy-to-bitrise-io@2: + is_always_run: false + inputs: + - deploy_path: "$BITRISE_SOURCE_DIR/protocol/languages/swift/.build/coverage" + - script@1: + title: Record Embedded Checkout Protocol Swift coverage + is_always_run: true + inputs: + - content: |- + set -euo pipefail + export COVERAGE_REPOSITORY="Shopify/checkout-kit" + export COVERAGE_PR="${BITRISE_PULL_REQUEST:-}" + export COVERAGE_SHA="${BITRISE_GIT_COMMIT:-}" + export COVERAGE_REPORT_URL="${BITRISE_BUILD_URL:-}" + export COVERAGE_TOKEN="${OVERRIDE_GITHUB_TOKEN:-${GIT_HTTP_PASSWORD:-}}" + export COVERAGE_TEST_OUTCOME=success + if [ "${BITRISE_BUILD_STATUS:-0}" != "0" ]; then + export COVERAGE_TEST_OUTCOME=failure + fi + ruby scripts/report_coverage protocol-swift protocol/languages/swift/.build/coverage/coverage.json ci-ios-swift-samples: meta: @@ -498,6 +545,13 @@ workflows: printf '%s\n' "${BITRISEIO_FINISHED_WORKFLOWS:-}" e2e_log "Reporting iOS CI results to GitHub" ruby e2e/scripts/report_ios_ci_results + - script@1: + title: Finalize Swift coverage status + is_always_run: true + inputs: + - content: |- + set -euo pipefail + ruby e2e/scripts/report_swift_coverage_status e2e-produce-browserstack-run-plan: steps: diff --git a/e2e/lib/swift_coverage_status.rb b/e2e/lib/swift_coverage_status.rb new file mode 100644 index 000000000..ab185cd55 --- /dev/null +++ b/e2e/lib/swift_coverage_status.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module SwiftCoverageStatus + def self.state(stages, selected) + return "failed" unless stages.stage("ci-ios-plan")&.status == "succeeded" + return "skipped" unless selected.include?("swift-package-tests") + + stages.stage("ci-ios-swift-package-tests")&.status == "succeeded" ? "unavailable" : "failed" + end +end diff --git a/e2e/scripts/report_swift_coverage_status b/e2e/scripts/report_swift_coverage_status new file mode 100644 index 000000000..2ba72d751 --- /dev/null +++ b/e2e/scripts/report_swift_coverage_status @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../lib/bitrise_pipeline_stages" +require_relative "../lib/github_token" +require_relative "../lib/swift_coverage_status" +require_relative "../../scripts/lib/coverage_report" + +pr = ENV["BITRISE_PULL_REQUEST"].to_s +token = GitHubToken.resolve +if !pr.match?(/\A[1-9]\d*\z/) || token.nil? + puts "No pull request or GitHub token; skipping Swift coverage status." + exit +end + +stages = BitrisePipelineStages.from_json(ENV["BITRISEIO_FINISHED_WORKFLOWS"]) +selected = ENV["CI_IOS_SELECTED_JOBS"].to_s.split(",").map(&:strip) +state = SwiftCoverageStatus.state(stages, selected) + +# Preserve the package job's measured result. If that job never published, record +# why coverage is missing instead of leaving the combined comment pending forever. +publisher = CoverageResultPublisher.new( + repository: ENV.fetch("GITHUB_REPOSITORY", "Shopify/checkout-kit"), + pr_number: pr, + sha: ENV.fetch("BITRISE_GIT_COMMIT"), + token: token, + source: {"provider" => "bitrise", "pipelineUrl" => ENV.fetch("BITRISEIO_PIPELINE_BUILD_URL")} +) +%w[swift protocol-swift].each do |platform| + puts publisher.publish(platform: platform, state: state, preserve_existing: true) +end diff --git a/e2e/test/swift_coverage_status_test.rb b/e2e/test/swift_coverage_status_test.rb new file mode 100644 index 000000000..04531ac59 --- /dev/null +++ b/e2e/test/swift_coverage_status_test.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../lib/bitrise_pipeline_stages" +require_relative "../lib/swift_coverage_status" + +class SwiftCoverageStatusTest < Minitest::Test + def state(plan: "succeeded", swift: nil, selected: true) + entries = [{"name" => "ci-ios-plan", "status" => plan, "external_id" => "plan-build"}] + entries << {"name" => "ci-ios-swift-package-tests", "status" => swift, "external_id" => "swift-build"} if swift + SwiftCoverageStatus.state(BitrisePipelineStages.new(entries), selected ? ["swift-package-tests"] : []) + end + + def test_only_deliberately_unselected_jobs_are_skipped + assert_equal "skipped", state(selected: false) + assert_equal "failed", state(plan: "failed", selected: false) + assert_equal "failed", state + assert_equal "failed", state(swift: "failed") + end + + def test_a_successful_job_without_a_published_measurement_is_unavailable + assert_equal "unavailable", state(swift: "succeeded") + end +end diff --git a/platforms/android/lib/build.gradle b/platforms/android/lib/build.gradle index 8592eea4d..3c1a14039 100644 --- a/platforms/android/lib/build.gradle +++ b/platforms/android/lib/build.gradle @@ -73,6 +73,9 @@ android { } buildTypes { + debug { + enableUnitTestCoverage true + } release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt') @@ -180,6 +183,17 @@ tasks.withType(Test).configureEach { dependsOn verifyPublishedProtocol } +plugins.withId('jacoco') { + tasks.withType(Test).configureEach { + jacoco { + // Robolectric loads SDK classes without a source location. Include + // those classes, but avoid instrumenting its runtime or JDK internals. + includeNoLocationClasses = true + includes = ['com.shopify.checkoutkit.*'] + } + } +} + // Local protocol sources are for joint development only. Maven Local remains // available for React Native's explicit --local builds. tasks.withType(PublishToMavenRepository).configureEach { diff --git a/platforms/swift/Scripts/test_coverage b/platforms/swift/Scripts/test_coverage new file mode 100755 index 000000000..a20fdf76a --- /dev/null +++ b/platforms/swift/Scripts/test_coverage @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +report_dir="$repo_root/.xcresults/coverage" +result_bundle="$repo_root/.xcresults/ShopifyCheckoutKit-Package-coverage.xcresult" + +# Remove old exports before running so a failed test run cannot publish stale coverage. +mkdir -p "$report_dir" +rm -f "$report_dir/coverage.json" "$report_dir/coverage.txt" + +ENABLE_CODE_COVERAGE=1 "$script_dir/xcode_run" test ShopifyCheckoutKit-Package + +xcrun xccov view --report --json "$result_bundle" > "$report_dir/coverage.json" +xcrun xccov view --report "$result_bundle" > "$report_dir/coverage.txt" +xcrun xccov view --report --only-targets "$result_bundle" +printf '\nCoverage reports: %s\n' "$report_dir" diff --git a/platforms/swift/Scripts/xcode_run b/platforms/swift/Scripts/xcode_run index 90e363cb6..abe3c792b 100755 --- a/platforms/swift/Scripts/xcode_run +++ b/platforms/swift/Scripts/xcode_run @@ -42,15 +42,21 @@ xcodebuild_cmd="xcodebuild $ACTION -scheme $SCHEME -sdk iphonesimulator -destina # Retry failed tests up to three total attempts, including the initial run. if [[ "$ACTION" == *"test"* ]]; then xcodebuild_cmd="$xcodebuild_cmd -retry-tests-on-failure -test-iterations 3" + if [[ "${ENABLE_CODE_COVERAGE:-0}" == "1" ]]; then + xcodebuild_cmd="$xcodebuild_cmd -enableCodeCoverage YES" + fi fi if [[ "$SCHEME" == "ShopifyCheckoutKit-Package" && "$ACTION" == *"build"* && "$ACTION" != *"test"* ]]; then xcodebuild_cmd="$xcodebuild_cmd SWIFT_TREAT_WARNINGS_AS_ERRORS=YES" fi -if [[ "$CI" == "true" && ( "$ACTION" == *"build"* || "$ACTION" == *"test"* ) ]]; then +if [[ ( "$CI" == "true" || "${ENABLE_CODE_COVERAGE:-0}" == "1" ) && ( "$ACTION" == *"build"* || "$ACTION" == *"test"* ) ]]; then result_bundle_dir="$(cd "$SCRIPT_DIR/../../.." && pwd)/.xcresults" result_bundle_path="$result_bundle_dir/$SCHEME.xcresult" + if [[ "${ENABLE_CODE_COVERAGE:-0}" == "1" ]]; then + result_bundle_path="$result_bundle_dir/$SCHEME-coverage.xcresult" + fi rm -rf "$result_bundle_path" mkdir -p "$result_bundle_dir" xcodebuild_cmd="$xcodebuild_cmd -resultBundlePath \"$result_bundle_path\"" diff --git a/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle b/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle index 13d523fae..ba37806d1 100644 --- a/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle +++ b/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle @@ -7,6 +7,7 @@ plugins { id 'signing' id 'maven-publish' id 'java-library' + id 'jacoco' id 'org.jetbrains.kotlin.jvm' id 'org.jetbrains.kotlin.plugin.serialization' id 'io.gitlab.arturbosch.detekt' @@ -121,3 +122,13 @@ publishing { } } } + +// The standalone protocol suite measures its own artifact, including generated +// wire models, independently of the Android SDK's published dependency. +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} diff --git a/protocol/scripts/test_swift_coverage b/protocol/scripts/test_swift_coverage new file mode 100755 index 000000000..8a85eb4e5 --- /dev/null +++ b/protocol/scripts/test_swift_coverage @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +package_path="$repo_root/protocol/languages/swift" +report_dir="$package_path/.build/coverage" +mkdir -p "$report_dir" +rm -f "$report_dir/coverage.json" +swift test --package-path "$package_path" --enable-code-coverage +coverage_path="$(swift test --package-path "$package_path" --show-codecov-path)" +cp "$coverage_path" "$report_dir/coverage.json" diff --git a/scripts/lib/coverage_report.rb b/scripts/lib/coverage_report.rb index 50f3d3d87..039b5b14c 100644 --- a/scripts/lib/coverage_report.rb +++ b/scripts/lib/coverage_report.rb @@ -4,14 +4,19 @@ require_relative "json_http_client" class CoverageReport + SWIFT_TARGETS = %w[ShopifyCheckoutKit ShopifyAcceleratedCheckouts].freeze + ANDROID_METRICS = {"LINE" => "Lines", "INSTRUCTION" => "Instructions", "BRANCH" => "Branches", "METHOD" => "Methods"}.freeze JAVASCRIPT_METRICS = {"lines" => "Lines", "statements" => "Statements", "branches" => "Branches", "functions" => "Functions"}.freeze - TITLES = {"web" => "Web", "react-native" => "React Native", "protocol" => "Embedded Checkout Protocol (TypeScript)"}.freeze + TITLES = {"swift" => "Swift", "android" => "Android", "web" => "Web", "react-native" => "React Native", "protocol" => "Embedded Checkout Protocol (TypeScript)", "protocol-swift" => "Embedded Checkout Protocol (Swift)", "protocol-kotlin" => "Embedded Checkout Protocol (Kotlin)"}.freeze attr_reader :platform, :rows def initialize(platform, contents) @platform = platform @rows = case platform + when "swift" then swift_rows(contents) + when "android", "protocol-kotlin" then android_rows(contents) + when "protocol-swift" then swift_protocol_rows(contents) when "web", "react-native", "protocol" then javascript_rows(contents) else raise ArgumentError, "Unknown coverage platform: #{platform}" end @@ -24,11 +29,18 @@ def marker def markdown(report_url: nil) label = TITLES.fetch(platform) lines = [marker, "# #{label} — Coverage Report", ""] - lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| #{@rows.map { "---" }.join(" | ")} |"]) - cells = @rows.map do |name, covered, total| - metric(covered, total, badge: name == "Lines", report_url: report_url) + if platform == "swift" + lines.concat(["| Target | Lines |", "| --- | --- |"]) + @rows.each do |name, covered, total| + lines << "| #{name} | #{metric(covered, total, badge: true, report_url: report_url)} |" + end + else + lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| #{@rows.map { "---" }.join(" | ")} |"]) + cells = @rows.map do |name, covered, total| + metric(covered, total, badge: name == "Lines", report_url: report_url) + end + lines << "| #{cells.join(' | ')} |" end - lines << "| #{cells.join(' | ')} |" lines.concat(["", "[Full coverage reports](#{report_url})"]) unless report_url.to_s.empty? lines.join("\n") + "\n" end @@ -66,6 +78,49 @@ def metric(covered, total, badge:, report_url:) image end + def swift_rows(contents) + targets = JSON.parse(contents).fetch("targets") + SWIFT_TARGETS.map do |name| + target = targets.find { |entry| entry.fetch("name") == name } + raise "Missing Swift coverage target: #{name}" unless target + + row(name, target.fetch("coveredLines"), target.fetch("executableLines")) + end + end + + def swift_protocol_rows(contents) + files = JSON.parse(contents).fetch("data").flat_map { |data| data.fetch("files") } + .select { |file| file.fetch("filename").include?("/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/") } + raise "Missing Swift protocol coverage files" if files.empty? + raise "Duplicate Swift protocol coverage files" unless files.map { |file| file.fetch("filename") }.uniq.length == files.length + + {"lines" => "Lines", "functions" => "Functions"}.map do |key, label| + counts = files.map do |file| + metric = file.fetch("summary").fetch(key) + row(label, metric.fetch("covered"), metric.fetch("count")) + end + row(label, counts.sum { |entry| entry[1] }, counts.sum { |entry| entry[2] }) + end + end + + def android_rows(contents) + # Bitrise's Swift runner only needs JSON and does not install the XML gem. + require "rexml/document" + + document = REXML::Document.new(contents) + ANDROID_METRICS.map do |type, label| + # Nested package/class counters duplicate the report totals. + counter = document.elements["report/counter[@type='#{type}']"] + raise "Missing Android coverage counter: #{type}" unless counter + + covered = Integer(counter.attributes["covered"]) + missed = Integer(counter.attributes["missed"]) + raise "Invalid Android coverage counter: #{type}" if missed.negative? + + row(label, covered, covered + missed) + end + end + def row(label, covered, total) unless covered.is_a?(Integer) && total.is_a?(Integer) && covered >= 0 && total >= covered raise "Invalid coverage counts for #{label}" @@ -96,6 +151,16 @@ def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: return "Skipped coverage result: pull request base changed." if revision == "base" && pr.dig("base", "sha") != base_sha checks = check_runs + if @source.fetch("provider") == "bitrise" + pipeline = checks.select do |check| + check.dig("app", "slug") == "bitrise" && check["name"] == "ci/bitrise/ci-ios/pr" && + check["details_url"] == @source.fetch("pipelineUrl") + end.max_by { |check| check.fetch("id") } + raise "Could not identify the Bitrise pipeline check" unless pipeline + + @source["runId"] = pipeline.fetch("id") + @source["runAttempt"] = pipeline.fetch("external_id", 1) + end prefix = revision == "base" ? "coverage-base" : "coverage" external_id = "#{prefix}:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}" existing = checks.find { |check| check["external_id"] == external_id && check.dig("app", "slug") == @source.fetch("provider") } @@ -118,6 +183,11 @@ def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: @client.post_json("/repos/#{@repository}/check-runs", payload.merge(head_sha: @sha)) end end + # Persist before notifying. The serialized publisher reloads all results, so + # coalesced notifications cannot lose a platform that finished concurrently. + if @source.fetch("provider") == "bitrise" + @client.post_json("/repos/#{@repository}/dispatches", {event_type: "coverage-updated", client_payload: {pr: @pr_number.to_i, sha: @sha}}) + end "Published #{platform} coverage result." end diff --git a/scripts/report_coverage b/scripts/report_coverage index 9810f5db7..001e34413 100644 --- a/scripts/report_coverage +++ b/scripts/report_coverage @@ -24,7 +24,11 @@ end abort "COVERAGE_PR must be a pull request number" unless pr_number.match?(/\A[1-9]\d*\z/) abort "COVERAGE_SHA is required to publish coverage" if sha.to_s.empty? -source = {"provider" => "github-actions", "runId" => Integer(ENV.fetch("GITHUB_RUN_ID")), "runAttempt" => Integer(ENV.fetch("GITHUB_RUN_ATTEMPT", "1"))} +source = if ENV["GITHUB_RUN_ID"] + {"provider" => "github-actions", "runId" => Integer(ENV.fetch("GITHUB_RUN_ID")), "runAttempt" => Integer(ENV.fetch("GITHUB_RUN_ATTEMPT", "1"))} +else + {"provider" => "bitrise", "pipelineUrl" => ENV.fetch("BITRISEIO_PIPELINE_BUILD_URL")} +end publisher = CoverageResultPublisher.new( repository: ENV.fetch("COVERAGE_REPOSITORY"), pr_number: pr_number, diff --git a/scripts/test/coverage_report_test.rb b/scripts/test/coverage_report_test.rb index 388a45097..ba2a1cbf3 100644 --- a/scripts/test/coverage_report_test.rb +++ b/scripts/test/coverage_report_test.rb @@ -12,15 +12,6 @@ def javascript_json(covered: 7, total: 10) end) end - def test_javascript_reports_preserve_metric_names_and_summary_format - %w[web react-native protocol].each do |platform| - report = CoverageReport.new(platform, javascript_json) - assert_equal ["Lines", "Statements", "Branches", "Functions"], report.rows.map(&:first) - assert_includes report.markdown, "70% (7/10)" - assert_includes report.markdown, "Coverage-70%25-yellow.svg" - end - end - def test_empty_metrics_are_not_reported_as_full_coverage report = CoverageReport.new("web", javascript_json(covered: 0, total: 0)) assert_includes report.markdown, "| N/A | N/A (0/0) |" @@ -33,26 +24,136 @@ def test_missing_or_impossible_counts_fail assert_raises(RuntimeError) { CoverageReport.new("web", javascript_json(covered: -1)) } end + def swift_json + JSON.generate("targets" => [ + {"name" => "ShopifyCheckoutKitTests", "coveredLines" => 1000, "executableLines" => 1000}, + {"name" => "ShopifyCheckoutKit", "coveredLines" => 9, "executableLines" => 10}, + {"name" => "ShopifyAcceleratedCheckouts", "coveredLines" => 2, "executableLines" => 3}, + {"name" => "EmbeddedCheckoutProtocol", "coveredLines" => 0, "executableLines" => 0} + ]) + end + + def android_xml + <<~XML + + + + + + + + + + + + XML + end + + def test_swift_reports_sdk_targets_only_even_below_85_percent + markdown = CoverageReport.new("swift", swift_json).markdown + + assert_includes markdown, "# Swift — Coverage Report" + assert_includes markdown, "| Target | Lines |" + assert_includes markdown, "| ShopifyCheckoutKit | ![Coverage: 90%](https://img.shields.io/badge/Coverage-90%25-brightgreen.svg) |" + assert_includes markdown, "| ShopifyAcceleratedCheckouts | ![Coverage: 66.67%](https://img.shields.io/badge/Coverage-66.67%25-yellow.svg) |" + refute_includes markdown, "ShopifyCheckoutKitTests" + refute_includes markdown, "EmbeddedCheckoutProtocol" + end + + def test_android_uses_report_totals_and_handles_no_executable_branches + markdown = CoverageReport.new("android", android_xml).markdown + + assert_includes markdown, "# Android — Coverage Report" + assert_includes markdown, "| Lines | Instructions | Branches | Methods |" + assert_includes markdown, "| ![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg) | 90% (9/10) | N/A (0/0) | 25% (1/4) |" + end + + def test_swift_protocol_uses_only_library_files_and_includes_generated_wire_models + files = [ + ["/package/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Client.swift", 8, 10], + ["/package/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Generated/Models.swift", 3, 5], + ["/package/Tests/EmbeddedCheckoutProtocolTests/ClientTests.swift", 100, 100], + ["/package/.build/runner.swift", 100, 100] + ].map do |filename, covered, total| + {"filename" => filename, "summary" => {"lines" => {"covered" => covered, "count" => total}, "functions" => {"covered" => 1, "count" => 2}}} + end + report = CoverageReport.new("protocol-swift", JSON.generate("data" => [{"files" => files}])) + assert_equal [["Lines", 11, 15], ["Functions", 2, 4]], report.rows + assert_includes report.markdown, "Embedded Checkout Protocol (Swift)" + assert_includes report.markdown, "| Lines | Functions |" + end + + def test_swift_protocol_rejects_missing_or_duplicate_files + assert_raises(RuntimeError) { CoverageReport.new("protocol-swift", '{"data":[{"files":[]}]}') } + file = {"filename" => "/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Client.swift"} + assert_raises(RuntimeError) { CoverageReport.new("protocol-swift", JSON.generate("data" => [{"files" => [file, file]}])) } + end + + def test_kotlin_protocol_has_its_own_title_and_comment_marker + report = CoverageReport.new("protocol-kotlin", android_xml) + assert_equal CoverageReport.new("android", android_xml).rows, report.rows + assert_includes report.markdown, "Embedded Checkout Protocol (Kotlin)" + refute_equal CoverageReport.new("android", android_xml).marker, report.marker + end + + def test_missing_targets_and_counters_fail_instead_of_showing_partial_coverage + assert_raises(RuntimeError) { CoverageReport.new("swift", '{"targets":[]}') } + assert_raises(RuntimeError) { CoverageReport.new("android", '') } + end + + def test_javascript_reports_share_the_native_format_without_changing_metric_names + contents = JSON.generate("total" => CoverageReport::JAVASCRIPT_METRICS.keys.to_h { |key| [key, {"covered" => 7, "total" => 10}] }) + %w[web react-native protocol].each do |platform| + report = CoverageReport.new(platform, contents) + assert_equal ["Lines", "Statements", "Branches", "Functions"], report.rows.map(&:first) + assert_includes report.markdown, "70% (7/10)" + assert_includes report.markdown, "Coverage-70%25-yellow.svg" + end + end + + def test_impossible_counts_fail + invalid = swift_json.sub('"coveredLines":9', '"coveredLines":11') + assert_raises(RuntimeError) { CoverageReport.new("swift", invalid) } + assert_raises(RuntimeError) { CoverageReport.new("android", android_xml.sub('missed="1"', 'missed="-1"')) } + end + def test_cli_keeps_a_job_summary_without_a_comment_token Dir.mktmpdir("coverage report ") do |directory| - input = File.join(directory, "coverage-summary.json") + input = File.join(directory, "report.xml") summary = File.join(directory, "summary.md") - File.write(input, javascript_json) + File.write(input, android_xml) File.write(summary, "Existing summary\n") output, error, status = Open3.capture3( {"COVERAGE_TOKEN" => "", "COVERAGE_PR" => "123", "COVERAGE_SHA" => "abc123", "COVERAGE_REPORT_URL" => "https://example.com/reports", "GITHUB_STEP_SUMMARY" => summary}, - "ruby", File.expand_path("../report_coverage", __dir__), "web", input + "ruby", File.expand_path("../report_coverage", __dir__), "android", input ) assert status.success?, "#{output}#{error}" assert_includes output, "Skipped coverage result" markdown = File.read(summary) assert markdown.start_with?("Existing summary\n") - assert_includes markdown, "[![Coverage: 70%](https://img.shields.io/badge/Coverage-70%25-yellow.svg)](https://example.com/reports)" + assert_includes markdown, "[![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)](https://example.com/reports)" assert_includes markdown, "[Full coverage reports](https://example.com/reports)" end end + + def test_swift_cli_runs_without_androids_xml_dependency + Dir.mktmpdir("swift coverage ") do |directory| + Dir.mkdir(File.join(directory, "rexml")) + File.write(File.join(directory, "rexml/document.rb"), 'raise LoadError, "XML is unavailable on the Swift runner"') + input = File.join(directory, "coverage.json") + File.write(input, swift_json) + output, error, status = Open3.capture3( + {"COVERAGE_TOKEN" => "", "GITHUB_STEP_SUMMARY" => ""}, + "ruby", "-I", directory, File.expand_path("../report_coverage", __dir__), "swift", input + ) + + assert status.success?, "#{output}#{error}" + assert_includes output, "# Swift — Coverage Report" + assert_includes output, "Coverage: 66.67%" + end + end end class CoverageResultPublisherTest < Minitest::Test @@ -81,8 +182,8 @@ def patch_json(path, body) end def setup - @report = CoverageReport.new("web", JSON.generate("total" => CoverageReport::JAVASCRIPT_METRICS.keys.to_h do |key| - [key, {"covered" => 1, "total" => 2}] + @report = CoverageReport.new("swift", JSON.generate("targets" => CoverageReport::SWIFT_TARGETS.map do |name| + {"name" => name, "coveredLines" => 1, "executableLines" => 2} end)) @pr = {"state" => "open", "head" => {"sha" => "abc123", "repo" => {"full_name" => "example/sdk"}}, "base" => {"sha" => "base123"}} @source = {"provider" => "github-actions", "runId" => 100, "runAttempt" => 2} @@ -91,26 +192,26 @@ def setup def publish(checks = [], **options) @client = FakeClient.new(@pr, checks) CoverageResultPublisher.new(repository: "example/sdk", pr_number: 123, sha: "abc123", token: "test-token", source: @source, client: @client) - .publish(platform: "web", state: "success", report: @report, **options) + .publish(platform: "swift", state: "success", report: @report, **options) end def test_persists_numeric_results_against_the_exact_run_without_a_pr_comment - assert_includes publish, "Published web" + assert_includes publish, "Published swift" method, path, payload = @client.writes.fetch(0) assert_equal :post, method assert_equal "/repos/example/sdk/check-runs", path assert_equal "neutral", payload[:conclusion] assert_equal "abc123", payload[:head_sha] - assert_equal "coverage:web:100:2", payload[:external_id] + assert_equal "coverage:swift:100:2", payload[:external_id] result = JSON.parse(payload[:output][:text]) assert_equal 123, result.fetch("pr") assert_equal @source, result.fetch("source") - assert_equal [["Lines", 1, 2], ["Statements", 1, 2], ["Branches", 1, 2], ["Functions", 1, 2]], result.fetch("rows") + assert_equal [["ShopifyCheckoutKit", 1, 2], ["ShopifyAcceleratedCheckouts", 1, 2]], result.fetch("rows") assert_equal 1, @client.writes.length end def test_updates_the_matching_check_without_creating_duplicates - publish([{"id" => 42, "external_id" => "coverage:web:100:2", "app" => {"slug" => "github-actions"}}]) + publish([{"id" => 42, "external_id" => "coverage:swift:100:2", "app" => {"slug" => "github-actions"}}]) assert_equal [:patch, "/repos/example/sdk/check-runs/42"], @client.writes.first.take(2) end @@ -118,8 +219,8 @@ def test_base_results_are_stored_separately_on_the_pr_head publish(revision: "base", base_sha: "base123") payload = @client.writes.first.last assert_equal "abc123", payload[:head_sha] - assert_equal "coverage-base:web:100:2", payload[:external_id] - assert_equal "Coverage base — Web", payload[:name] + assert_equal "coverage-base:#{@report.platform}:100:2", payload[:external_id] + assert_equal "Coverage base — #{CoverageReport::TITLES.fetch(@report.platform)}", payload[:name] result = JSON.parse(payload[:output][:text]) assert_equal "base", result.fetch("revision") assert_equal "base123", result.fetch("baseSha") @@ -127,7 +228,7 @@ def test_base_results_are_stored_separately_on_the_pr_head end def test_head_results_record_the_tested_base_without_overwriting_base_results - publish([{"id" => 42, "external_id" => "coverage-base:web:100:2", "app" => {"slug" => "github-actions"}}], base_sha: "base123") + publish([{"id" => 42, "external_id" => "coverage-base:#{@report.platform}:100:2", "app" => {"slug" => "github-actions"}}], base_sha: "base123") assert_equal :post, @client.writes.first.first result = JSON.parse(@client.writes.first.last[:output][:text]) assert_equal "head", result.fetch("revision") @@ -152,4 +253,24 @@ def test_superseded_closed_and_fork_prs_are_not_published end end + def test_bitrise_persists_before_dispatching_and_finalization_preserves_measurements + @source = {"provider" => "bitrise", "pipelineUrl" => "https://app.bitrise.io/app/test/pipelines/current"} + attempt = "apps/example/pipelines/current/attempts/first" + pipeline = {"id" => 200, "external_id" => attempt, "name" => "ci/bitrise/ci-ios/pr", "app" => {"slug" => "bitrise"}, "details_url" => @source["pipelineUrl"]} + publish([pipeline]) + assert_equal "/repos/example/sdk/check-runs", @client.writes.first[1] + assert_equal "/repos/example/sdk/dispatches", @client.writes.last[1] + assert_equal({event_type: "coverage-updated", client_payload: {pr: 123, sha: "abc123"}}, @client.writes.last[2]) + assert_equal "coverage:swift:200:#{attempt}", @client.writes.first[2][:external_id] + + existing = {"id" => 201, "external_id" => "coverage:swift:200:#{attempt}", "app" => {"slug" => "bitrise"}} + publish([pipeline, existing], preserve_existing: true) + assert_equal ["/repos/example/sdk/dispatches"], @client.writes.map { |write| write[1] } + end + + def test_unknown_bitrise_pipeline_does_not_publish_untraceable_results + @source = {"provider" => "bitrise", "pipelineUrl" => "https://app.bitrise.io/app/test/pipelines/missing"} + assert_raises(RuntimeError) { publish } + assert_empty @client.writes + end end diff --git a/scripts/test/swift_coverage_test.rb b/scripts/test/swift_coverage_test.rb new file mode 100644 index 000000000..d4ebc67fc --- /dev/null +++ b/scripts/test/swift_coverage_test.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "fileutils" +require "minitest/autorun" +require "open3" +require "tmpdir" + +class SwiftCoverageTest < Minitest::Test + SCRIPTS = File.expand_path("../../platforms/swift/Scripts", __dir__) + + def setup + @directory = Dir.mktmpdir("swift coverage ") + @scripts = File.join(@directory, "platforms/swift/Scripts") + @bin = File.join(@directory, "bin") + @reports = File.join(@directory, ".xcresults/coverage") + @arguments = File.join(@directory, "xcodebuild-arguments") + FileUtils.mkdir_p([@scripts, @bin, @reports]) + File.write(File.join(@directory, "Package.swift"), "// test package\n") + %w[xcode_run test_coverage].each { |name| FileUtils.cp(File.join(SCRIPTS, name), @scripts) } + executable("xcodebuild", <<~BASH) + # Reject invocations that could silently change the package pins. + [[ " $* " == *" -disableAutomaticPackageResolution "* ]] || exit 2 + printf '%s\\n' "$@" > "$COVERAGE_TEST_ARGUMENTS" + exit "${COVERAGE_TEST_BUILD_STATUS:-0}" + BASH + executable("xcbeautify", 'cat') + executable("xcrun", <<~BASH) + [ "$1" = xccov ] || exit 1 + [ "${COVERAGE_TEST_EXPORT_STATUS:-0}" = 0 ] || exit "$COVERAGE_TEST_EXPORT_STATUS" + if [[ " $* " == *" --json "* ]]; then + printf '%s\\n' '{"targets":[]}' + else + printf '%s\\n' 'Swift coverage report' + fi + BASH + end + + def teardown + FileUtils.remove_entry(@directory) + end + + def test_coverage_enables_instrumentation_and_exports_reports_locally + output, error, status = run_script("test_coverage") + + assert status.success?, "#{output}#{error}" + arguments = File.readlines(@arguments, chomp: true) + assert_includes arguments, "-disableAutomaticPackageResolution" + assert_equal "YES", arguments[arguments.index("-enableCodeCoverage") + 1] + assert_equal File.join(@directory, ".xcresults/ShopifyCheckoutKit-Package-coverage.xcresult"), + arguments[arguments.index("-resultBundlePath") + 1] + assert_equal "{\"targets\":[]}\n", File.read(File.join(@reports, "coverage.json")) + assert_equal "Swift coverage report\n", File.read(File.join(@reports, "coverage.txt")) + end + + def test_failed_tests_remove_stale_exports_and_fail_the_command + %w[coverage.json coverage.txt].each { |name| File.write(File.join(@reports, name), "stale") } + + _output, _error, status = run_script("test_coverage", {"COVERAGE_TEST_BUILD_STATUS" => "65"}) + + refute status.success? + assert_empty Dir.children(@reports) + end + + def test_export_failure_fails_the_command + _output, _error, status = run_script("test_coverage", {"COVERAGE_TEST_EXPORT_STATUS" => "1"}) + + refute status.success? + end + + def test_regular_local_tests_do_not_enable_coverage_or_request_a_result_bundle + output, error, status = run_script("xcode_run", {}, "test", "ShopifyCheckoutKit-Package") + + assert status.success?, "#{output}#{error}" + arguments = File.readlines(@arguments, chomp: true) + refute_includes arguments, "-enableCodeCoverage" + refute_includes arguments, "-resultBundlePath" + end + + private + + def executable(name, body) + path = File.join(@bin, name) + File.write(path, "#!/bin/bash\nset -euo pipefail\n#{body}\n") + FileUtils.chmod(0o755, path) + end + + def run_script(name, environment = {}, *arguments) + Open3.capture3( + { + "PATH" => "#{@bin}:#{ENV.fetch("PATH")}", + "CI" => "false", + "ENABLE_CODE_COVERAGE" => "0", + "CURRENT_SIMULATOR_UUID" => "test-simulator", + "COVERAGE_TEST_ARGUMENTS" => @arguments + }.merge(environment), + "bash", File.join(@scripts, name), *arguments, + chdir: @directory + ) + end +end