From 387e9a15a76aa592c288f9879506b57ec44b5c01 Mon Sep 17 00:00:00 2001 From: wgqqqqq Date: Thu, 15 Oct 2026 20:42:18 +0800 Subject: [PATCH] ci(macos): resume notary polling without repeating uploads --- .github/workflows/desktop-package.yml | 17 +++++ docs/development/releasing.md | 13 ++++ scripts/check-github-config.test.mjs | 94 ++++++++++++++++++++++++++- scripts/ci/verify-macos-signing.sh | 76 ++++++++++++++++++++-- 4 files changed, 193 insertions(+), 7 deletions(-) diff --git a/.github/workflows/desktop-package.yml b/.github/workflows/desktop-package.yml index 5e4e17f8ff..1cc119f532 100644 --- a/.github/workflows/desktop-package.yml +++ b/.github/workflows/desktop-package.yml @@ -424,6 +424,7 @@ jobs: } - name: Verify Apple signature and notarization + id: apple_verification if: runner.os == 'macOS' shell: bash run: | @@ -443,6 +444,22 @@ jobs: shell: bash run: bash scripts/ci/verify-appimage-fcitx.sh "${{ matrix.platform.target }}" + - name: Preserve macOS verification recovery bundles + if: failure() && runner.os == 'macOS' && steps.apple_verification.outcome == 'failure' + shell: bash + run: | + tar -czf macos-verification-recovery.tar.gz \ + -C "target/${{ matrix.platform.target }}/release" bundle + + - name: Upload macOS verification recovery bundles + if: failure() && runner.os == 'macOS' && steps.apple_verification.outcome == 'failure' + uses: actions/upload-artifact@v6 + with: + name: openbitfun-${{ needs.prepare.outputs.release_tag }}-${{ matrix.platform.name }}-verification-recovery + path: macos-verification-recovery.tar.gz + if-no-files-found: error + retention-days: 7 + - name: Upload bundles uses: actions/upload-artifact@v6 with: diff --git a/docs/development/releasing.md b/docs/development/releasing.md index aa4556c76b..c470981b83 100644 --- a/docs/development/releasing.md +++ b/docs/development/releasing.md @@ -47,6 +47,19 @@ Missing, duplicate, expired, or unverified producers stop finalization before publication. Do not combine `artifact_run_ids` with a single-platform target or `relay_image_only`. +macOS verification uploads a DMG once, records its submission ID and SHA256 in +`.notary-submission.json`, then waits for that submission. A transient +`NSURLErrorDomain` connection loss (`-1005`) or offline response (`-1009`) +retries the wait, without uploading the DMG again. Apple must explicitly return +`Accepted`; invalid results, other errors, and changed DMG bytes remain failures. +Stapling updates the receipt's hash to the exact stapled DMG bytes. + +If Apple verification fails, the workflow retains a seven-day +`*-verification-recovery` artifact containing a tar archive of the bundles and +receipt. The archive preserves executable modes and symlinks for re-verification. +It is unverified recovery input, never a release installer: it does not match +the normal `*-bundle` publication pattern or satisfy the complete producer gate. + ## Mobile Android releases Mobile Android packages are released independently from Desktop packages. Use a diff --git a/scripts/check-github-config.test.mjs b/scripts/check-github-config.test.mjs index f1d6426290..3c31660c5e 100644 --- a/scripts/check-github-config.test.mjs +++ b/scripts/check-github-config.test.mjs @@ -1906,6 +1906,39 @@ test('macOS release verification uses the pinned workflow revision without chang assert.equal(readFileSync(artifact, 'utf8'), 'signed immutable fixture'); }); +test('failed Mac verification retains a tar archive without publishing it', (t) => { + if (process.platform === 'win32') { t.skip('Mac archive mode checks run on Unix'); return; } + const workflow = yaml.parse(readFileSync(path.join(repoRoot, '.github/workflows/desktop-package.yml'), 'utf8')); + const steps = workflow.jobs.package.steps; + const verify = steps.find((step) => step.name === 'Verify Apple signature and notarization'); + const preserve = steps.find((step) => step.name === 'Preserve macOS verification recovery bundles'); + const upload = steps.find((step) => step.name === 'Upload macOS verification recovery bundles'); + assert.equal(verify.id, 'apple_verification'); + assert.equal(verify['continue-on-error'], undefined); + for (const step of [preserve, upload]) { + for (const failed of [true, false]) for (const os of ['macOS', 'Linux', 'Windows']) for (const outcome of ['success', 'failure']) { + const enabled = Function('failure', 'runner', 'steps', `return (${step.if});`)(() => failed, { os }, { apple_verification: { outcome } }); + assert.equal(enabled, failed && os === 'macOS' && outcome === 'failure'); + } + } + assert.match(upload.with.name, /-verification-recovery$/); + assert.doesNotMatch(upload.with.name, /-bundle$/); + const cwd = mkdtempSync(path.join(tmpdir(), 'openbitfun-notary-archive-')); + t.after(() => rmSync(cwd, { recursive: true, force: true })); + const app = path.join(cwd, 'target/fixture/release/bundle/macos/Test.app'); + mkdirSync(app, { recursive: true }); + writeFileSync(path.join(app, 'tool'), 'signed executable fixture', { mode: 0o755 }); + assert.equal(spawnSync('ln', ['-s', 'tool', path.join(app, 'tool-link')], { windowsHide: true }).status, 0); + const archived = spawnSync('bash', ['-c', preserve.run.replaceAll('${{ matrix.platform.target }}', 'fixture')], { cwd, encoding: 'utf8', windowsHide: true }); + assert.equal(archived.status, 0, archived.stderr); + mkdirSync(path.join(cwd, 'restored')); + assert.equal(spawnSync('tar', ['-xzf', 'macos-verification-recovery.tar.gz', '-C', 'restored'], { cwd, windowsHide: true }).status, 0); + const restored = path.join(cwd, 'restored/bundle/macos/Test.app'); + assert.equal(readFileSync(path.join(restored, 'tool-link'), 'utf8'), 'signed executable fixture'); + assert.equal(spawnSync('bash', ['-c', 'test -x "$1"', 'fixture', path.join(restored, 'tool')], { windowsHide: true }).status, 0); + assert.equal(spawnSync('readlink', [path.join(restored, 'tool-link')], { encoding: 'utf8', windowsHide: true }).stdout.trim(), 'tool'); +}); + for (const scenario of [ { name: 'accepts a transient app ticket connection loss', failures: 2, network: true, attempts: 3, status: 0, submissions: 1 }, { name: 'retries DMG stapling without resubmitting notarization', action: 'staple', kind: 'dmg', failures: 1, network: true, attempts: 2, status: 0, submissions: 1 }, @@ -1916,6 +1949,14 @@ for (const scenario of [ { name: 'rejects signature errors before contacting Apple', signatureStatus: 1, attempts: 0, status: 1, submissions: 0 }, { name: 'preserves Gatekeeper rejection', gatekeeperStatus: 3, attempts: 1, status: 3, submissions: 0 }, { name: 'preserves notarization submission rejection', notaryStatus: 4, attempts: 1, status: 4, submissions: 1 }, + { name: 'resumes an uploaded DMG after transient offline polling', waitFailures: 2, waitCode: -1009, waitAttempts: 3, attempts: 1, status: 0, submissions: 1 }, + { name: 'resumes polling after a lost connection', waitFailures: 1, waitCode: -1005, waitAttempts: 2, attempts: 1, status: 0, submissions: 1 }, + { name: 'retains a receipt after exhausted polling retries', waitFailures: 99, waitCode: -1009, waitAttempts: 3, attempts: 1, status: 1, submissions: 1, resume: true }, + { name: 'does not retry unrelated polling failures', waitFailures: 99, waitCode: -1001, waitAttempts: 1, attempts: 1, status: 1, submissions: 1 }, + { name: 'rejects a missing submission ID without retrying upload', submissionId: 'invalid', waitAttempts: 0, attempts: 1, status: 1, submissions: 1 }, + { name: 'rejects a response for another submission', responseId: '22222222-2222-4222-8222-222222222222', waitAttempts: 1, attempts: 1, status: 1, submissions: 1 }, + { name: 'rejects an Invalid notarization response', waitStatus: 'Invalid', waitAttempts: 1, attempts: 1, status: 1, submissions: 1 }, + ]) { test(`macOS verification ${scenario.name}`, (t) => { const cwd = mkdtempSync(path.join(tmpdir(), 'openbitfun-apple-ticket-')); @@ -1939,7 +1980,23 @@ for (const scenario of [ sleep() { printf 'sleep %s\\n' "$*" >> "$FIXTURE_CALLS"; } xcrun() { printf 'xcrun %s\\n' "$*" >> "$FIXTURE_CALLS" - if [[ "$1" == notarytool ]]; then return "$NOTARY_STATUS"; fi + if [[ "$1" == notarytool ]]; then + if [[ "$2" == submit ]]; then + [[ "$NOTARY_STATUS" == 0 ]] || return "$NOTARY_STATUS" + printf '{"id":"%s"}\\n' "$SUBMISSION_ID" + else + local waits=0 + [[ ! -f waits ]] || read -r waits < waits + waits=$((waits + 1)) + printf '%s\\n' "$waits" > waits + if [[ "$waits" -le "$WAIT_FAILURES" ]]; then + printf 'Error Domain=NSURLErrorDomain Code=%s "Network error"\\n' "$WAIT_CODE" >&2 + return 1 + fi + printf '{"id":"%s","status":"%s"}\\n' "$RESPONSE_ID" "$WAIT_STATUS" + fi + return 0 + fi local kind=app count=0 [[ "$3" == *.dmg ]] && kind=dmg if [[ "$2" == "$TICKET_ACTION" && "$kind" == "$TICKET_KIND" ]]; then @@ -1962,6 +2019,8 @@ for (const scenario of [ `], { cwd, encoding: 'utf8', windowsHide: true, env: { ...process.env, OPENBITFUN_APPLE_SIGNING_CONFIGURED: 'true', FIXTURE_CALLS: path.join(cwd, 'calls'), SIGNATURE_STATUS: String(scenario.signatureStatus ?? 0), GATEKEEPER_STATUS: String(scenario.gatekeeperStatus ?? 0), + SUBMISSION_ID: scenario.submissionId ?? '11111111-1111-4111-8111-111111111111', RESPONSE_ID: scenario.responseId ?? '11111111-1111-4111-8111-111111111111', + WAIT_FAILURES: String(scenario.waitFailures ?? 0), WAIT_CODE: String(scenario.waitCode ?? -1009), WAIT_STATUS: scenario.waitStatus ?? 'Accepted', NOTARY_STATUS: String(scenario.notaryStatus ?? 0), APPLE_API_ISSUER: 'fixture', APPLE_API_KEY: 'fixture', APPLE_API_KEY_PATH: 'fixture', TICKET_ACTION: scenario.action ?? 'validate', TICKET_KIND: scenario.kind ?? 'app', TICKET_FAILURES: String(scenario.failures ?? 0), TICKET_NETWORK: String(scenario.network ?? false), @@ -1973,7 +2032,38 @@ for (const scenario of [ assert.equal((calls.match(/^microphone$/gm) ?? []).length, scenario.signatureStatus ? 0 : 1, calls); const attempts = scenario.attempts === 0 ? 0 : Number(readFileSync(path.join(cwd, 'attempts'), 'utf8')); assert.equal(attempts, scenario.attempts, calls); - assert.equal((calls.match(/^sleep /gm) ?? []).length, Math.max(0, scenario.attempts - 1), calls); + assert.equal((calls.match(/^sleep /gm) ?? []).length, Math.max(0, scenario.attempts - 1) + Math.max(0, (scenario.waitAttempts ?? 1) - 1), calls); + assert.equal((calls.match(/^xcrun notarytool wait /gm) ?? []).length, + scenario.waitAttempts ?? (scenario.notaryStatus ? 0 : scenario.submissions), calls); + if (scenario.resume) { + const receipt = path.join(cwd, 'target/fixture/release/bundle/dmg/Test App.dmg.notary-submission.json'); + assert.equal(JSON.parse(readFileSync(receipt, 'utf8')).id, '11111111-1111-4111-8111-111111111111'); + const retryScript = ` + codesign() { printf 'Authority=Developer ID Application: Fixture\\nflags=0x10000(runtime)\\nTimestamp=fixture\\n'; } + file() { printf 'Mach-O fixture\\n'; } + spctl() { return 0; } + xcrun() { + printf '%s\\n' "$*" >> resumed + if [[ "$1 $2" == 'notarytool submit' ]]; then return 99; fi + if [[ "$1 $2" == 'notarytool wait' ]]; then + printf '{"id":"11111111-1111-4111-8111-111111111111","status":"Accepted"}\\n' + fi + } + export -f codesign file spctl xcrun + bash scripts/ci/verify-macos-signing.sh fixture + `; + const retryOptions = { cwd, encoding: 'utf8', windowsHide: true, env: { ...process.env, + OPENBITFUN_APPLE_SIGNING_CONFIGURED: 'true', FIXTURE_CALLS: path.join(cwd, 'calls'), + APPLE_API_ISSUER: 'fixture', APPLE_API_KEY: 'fixture', APPLE_API_KEY_PATH: 'fixture' } }; + const retry = spawnSync('bash', ['-c', retryScript], retryOptions); + assert.equal(retry.status, 0, retry.stderr); + assert.doesNotMatch(readFileSync(path.join(cwd, 'resumed'), 'utf8'), /notarytool submit/); + writeFileSync(path.join(cwd, 'target/fixture/release/bundle/dmg/Test App.dmg'), 'changed after receipt'); + const changed = spawnSync('bash', ['-c', retryScript], retryOptions); + assert.notEqual(changed.status, 0); + assert.match(changed.stderr, /receipt does not match/); + + } if (scenario.status === 0) assert.match(result.stdout, /Verified Developer ID signatures/); else assert.doesNotMatch(result.stdout, /Verified Developer ID signatures/); }); diff --git a/scripts/ci/verify-macos-signing.sh b/scripts/ci/verify-macos-signing.sh index 81cf93b2a8..12ffefe025 100755 --- a/scripts/ci/verify-macos-signing.sh +++ b/scripts/ci/verify-macos-signing.sh @@ -39,6 +39,66 @@ stapler_with_network_retry() { done } +# Separate upload from polling so a lost connection never re-submits a DMG. +# Keep the receipt with the bundle for a verification-only recovery run. +notarize_dmg() { + local dmg="$1" receipt="${1}.notary-submission.json" submission_id output exit_code attempt + local auth=(--issuer "${APPLE_API_ISSUER}" --key-id "${APPLE_API_KEY}" --key "${APPLE_API_KEY_PATH}") + if [[ -f "${receipt}" ]]; then + submission_id="$(node -e ' + const fs = require("node:fs"), crypto = require("node:crypto"); + const [dmg, receipt] = process.argv.slice(1); + const saved = JSON.parse(fs.readFileSync(receipt, "utf8")); + if (!/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(saved.id) || + saved.sha256 !== crypto.createHash("sha256").update(fs.readFileSync(dmg)).digest("hex")) { + throw new Error("Notary receipt does not match this DMG; refusing to reuse its submission"); + } + console.log(saved.id); + ' "${dmg}" "${receipt}")" + else + if output="$(xcrun notarytool submit "${dmg}" "${auth[@]}" --no-wait --output-format json 2>&1)"; then + printf '%s\n' "${output}" + else + exit_code=$? + printf '%s\n' "${output}" >&2 + return "${exit_code}" + fi + submission_id="$(printf '%s' "${output}" | node -e ' + const fs = require("node:fs"), crypto = require("node:crypto"); + const [dmg, receipt] = process.argv.slice(1); + const { id } = JSON.parse(fs.readFileSync(0, "utf8")); + if (typeof id !== "string" || !/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(id)) { + throw new Error("Apple did not return a valid submission ID; refusing to resubmit"); + } + const sha256 = crypto.createHash("sha256").update(fs.readFileSync(dmg)).digest("hex"); + fs.writeFileSync(receipt + ".part", JSON.stringify({ id, sha256 }) + "\n", { mode: 0o600 }); + fs.renameSync(receipt + ".part", receipt); + console.log(id); + ' "${dmg}" "${receipt}")" + fi + for attempt in 1 2 3; do + if output="$(xcrun notarytool wait "${submission_id}" "${auth[@]}" --output-format json 2>&1)"; then + printf '%s\n' "${output}" + printf '%s' "${output}" | node -e ' + const result = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + if (result.id !== process.argv[1] || result.status !== "Accepted") { + throw new Error("Apple notarization was not accepted: " + result.status); + } + ' "${submission_id}" + return + else + exit_code=$? + fi + printf '%s\n' "${output}" >&2 + if [[ "${attempt}" -eq 3 ]] || + ! grep -Eq 'Error Domain=NSURLErrorDomain Code=-(1005|1009)([^0-9]|$)' <<<"${output}"; then + return "${exit_code}" + fi + echo "Apple notary connection was lost; waiting again on submission ${submission_id} (${attempt}/3)." >&2 + sleep "$((attempt * 10))" + done +} + for app in "${apps[@]}"; do codesign --verify --deep --strict --verbose=2 "${app}" bash "$(dirname "$0")/verify-macos-microphone.sh" "${app}" @@ -56,12 +116,18 @@ for app in "${apps[@]}"; do done for dmg in "${dmgs[@]}"; do - xcrun notarytool submit "${dmg}" \ - --issuer "${APPLE_API_ISSUER}" \ - --key-id "${APPLE_API_KEY}" \ - --key "${APPLE_API_KEY_PATH}" \ - --wait + notarize_dmg "${dmg}" stapler_with_network_retry staple "${dmg}" + # Stapling legitimately changes the accepted DMG. Bind a retained receipt to + # those exact bytes before validation, so recovery can safely reuse its ID. + node -e ' + const fs = require("node:fs"), crypto = require("node:crypto"); + const dmg = process.argv[1], receipt = dmg + ".notary-submission.json"; + const saved = JSON.parse(fs.readFileSync(receipt, "utf8")); + saved.sha256 = crypto.createHash("sha256").update(fs.readFileSync(dmg)).digest("hex"); + fs.writeFileSync(receipt + ".part", JSON.stringify(saved) + "\n", { mode: 0o600 }); + fs.renameSync(receipt + ".part", receipt); + ' "${dmg}" stapler_with_network_retry validate "${dmg}" spctl --assess --type open --context context:primary-signature --verbose=4 "${dmg}" done