ci: recover notarized Mac bundles without rebuilding

.github/workflows/desktop-package.yml

@@ -31,6 +31,10 @@ on:
         description: "Finalize from complete verified artifacts in previous Desktop Package runs (comma-separated IDs); skips rebuilding."
         required: false
         type: string
+      verification_recovery_run_id:
+        description: "Restore a failed Mac verification archive and verify without compiling; select one Mac target."
+        required: false
+        type: string
       target_platform:
         description: "Build one Desktop platform as signed artifacts only; all retains full release publication."
         required: false
@@ -160,6 +164,8 @@ jobs:
           TARGET_PLATFORM: ${{ inputs.target_platform || 'all' }}
           RELAY_IMAGE_ONLY: ${{ inputs.relay_image_only }}
           ARTIFACT_RUN_IDS: ${{ inputs.artifact_run_ids }}
+          VERIFICATION_RECOVERY_RUN_ID: ${{ inputs.verification_recovery_run_id }}
+          UPLOAD_TO_RELEASE: ${{ steps.meta.outputs.upload_to_release }}
         run: |
           node --input-type=module <<'NODE'
           import fs from 'node:fs';
@@ -175,6 +181,12 @@ jobs:
           if (target !== 'all' && process.env.RELAY_IMAGE_ONLY === 'true') {
             throw new Error('A Desktop recovery target cannot be combined with relay_image_only');
           }
+          if (process.env.VERIFICATION_RECOVERY_RUN_ID &&
+              (!/^\d+$/.test(process.env.VERIFICATION_RECOVERY_RUN_ID) || target === 'all' ||
+               process.env.ARTIFACT_RUN_IDS || process.env.RELAY_IMAGE_ONLY === 'true' ||
+               process.env.UPLOAD_TO_RELEASE !== 'true')) {
+            throw new Error('Verification recovery requires one Mac target, signing, and no other recovery mode');
+          }
           const selected = target === 'all' ? all : all.filter((platform) => platform.name === target);
           if (selected.length === 0) throw new Error('No Desktop package platforms selected');
           fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify(selected)}\n`);
@@ -185,7 +197,7 @@ jobs:
     name: Package (${{ matrix.platform.name }})
     runs-on: ${{ matrix.platform.os }}
     needs: prepare
-    if: needs.prepare.outputs.relay_image_only != 'true' && !inputs.artifact_run_ids
+    if: needs.prepare.outputs.relay_image_only != 'true' && !inputs.artifact_run_ids && !inputs.verification_recovery_run_id
     env:
       NODE_OPTIONS: --max-old-space-size=6144
       OPENBITFUN_ENABLE_UPDATER_ARTIFACTS: ${{ needs.prepare.outputs.upload_to_release }}
@@ -471,6 +483,84 @@ jobs:
             src/apps/desktop/target/release/bundle
             OpenBitFun-Installer/src-tauri/target/release/openbitfun-installer.exe
 
+  recover-macos:
+    name: Package (${{ matrix.platform.name }})
+    needs: prepare
+    if: inputs.verification_recovery_run_id != ''
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJSON(needs.prepare.outputs.package_matrix) }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+      RECOVERY_RUN: ${{ inputs.verification_recovery_run_id }}
+      RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}
+      RELEASE_VERSION: ${{ needs.prepare.outputs.version }}
+      MACOS_PLATFORM: ${{ matrix.platform.name }}
+      MACOS_TARGET: ${{ matrix.platform.target }}
+      OPENBITFUN_SIGNING_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
+      OPENBITFUN_SIGNING_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
+      OPENBITFUN_SIGNING_PUBKEY: ${{ secrets.TAURI_UPDATER_PUBKEY }}
+    steps:
+      # Packaging policy comes from this pinned workflow; payload bytes come only
+      # from the validated original tag producer. No product build runs here.
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '22.12.0'
+          package-manager-cache: false
+      - name: Select Xcode notarization tools
+        run: |
+          test -d /Applications/Xcode_26.2.app/Contents/Developer
+          echo "DEVELOPER_DIR=/Applications/Xcode_26.2.app/Contents/Developer" >> "$GITHUB_ENV"
+      - name: Restore prior macOS bundles
+        run: |
+          node scripts/restore-macos-verification-bundle.mjs --repo "$GITHUB_REPOSITORY" \
+            --tag "$RELEASE_TAG" --platform "$MACOS_PLATFORM" --run-id "$RECOVERY_RUN"
+          test "$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' \
+            "target/$MACOS_TARGET/release/bundle/macos/OpenBitFun.app/Contents/Info.plist")" = "$RELEASE_VERSION"
+          case "$MACOS_TARGET" in
+            x86_64-apple-darwin) architecture=x86_64 ;;
+            aarch64-apple-darwin) architecture=arm64 ;;
+          esac
+          lipo -verify_arch "$architecture" "target/$MACOS_TARGET/release/bundle/macos/OpenBitFun.app/Contents/MacOS/openbitfun-desktop"
+      - name: Configure Apple Developer ID signing and notarization
+        env:
+          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
+          APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
+          APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }}
+          APPLE_API_PRIVATE_KEY: ${{ secrets.APPLE_API_PRIVATE_KEY }}
+          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+          OPENBITFUN_REQUIRE_APPLE_SIGNING: 'true'
+        run: bash scripts/ci/setup-macos-signing.sh
+      - name: Repair missing application ticket
+        run: bash scripts/ci/repair-macos-stapling.sh "$MACOS_TARGET"
+      - name: Verify Apple signature and notarization
+        id: apple_verification
+        run: bash scripts/ci/verify-macos-signing.sh "$MACOS_TARGET"
+      - name: Preserve macOS verification recovery bundles
+        if: failure() && steps.apple_verification.outcome == 'failure'
+        run: tar -czf macos-verification-recovery.tar.gz -C "target/$MACOS_TARGET/release" bundle
+      - name: Upload macOS verification recovery bundles
+        if: failure() && steps.apple_verification.outcome == 'failure'
+        uses: actions/upload-artifact@v6
+        with:
+          name: openbitfun-${{ needs.prepare.outputs.release_tag }}-${{ matrix.platform.name }}-verification-recovery
+          path: macos-verification-recovery.tar.gz
+          if-no-files-found: error
+          retention-days: 7
+      - name: Upload bundles
+        uses: actions/upload-artifact@v6
+        with:
+          name: openbitfun-${{ needs.prepare.outputs.release_tag }}-${{ matrix.platform.name }}-bundle
+          path: target/${{ matrix.platform.target }}/release/bundle/
+          if-no-files-found: error
+
   linux-binaries:
     name: Linux CLI and Relay Server
     needs: prepare

docs/development/releasing.md

@@ -60,6 +60,23 @@ receipt. The archive preserves executable modes and symlinks for re-verification
 It is unverified recovery input, never a release installer: it does not match
 the normal `*-bundle` publication pattern or satisfy the complete producer gate.
 
+To resume that checkpoint without compiling again, dispatch `Desktop Package`
+with `verification_recovery_run_id`, the original tag/channel, one Mac
+`target_platform`, and `upload_to_release=true`. Do not combine it with
+`artifact_run_ids` or `relay_image_only`. Recovery accepts only a trusted tag
+producer with successful build, failed Apple verification, and a successfully
+retained, unexpired recovery archive. A recovery that fails Apple verification
+again can be resumed from its own checkpoint, retaining a matching DMG receipt.
+
+When the bundler reports Accepted but leaves the application without a stapled
+ticket, recovery explicitly staples and validates it, then rebuilds both the
+DMG and updater around that app. It verifies the original updater signature and
+Developer ID before repair, signs the replacement DMG and updater, and runs the
+unchanged complete Apple verification. Other ticket failures are not repaired;
+an existing DMG submission is never silently invalidated by repackaging.
+Only a successful verification produces the normal `*-bundle` artifact accepted
+by full release finalization. No Rust or frontend compilation runs in recovery.
+
 To add a verified Mac package to an already public partial stable release,
 dispatch `Publish Ready Desktop Assets` with `tag_name`, `macos_run_id`, and
 `macos_platform`; leave `source_run_id` empty. The selected producer must have

scripts/check-github-config.test.mjs

@@ -1579,6 +1579,152 @@ test('Desktop recovery selects only one Mac target and cannot publish an incompl
   assert.equal(workflow.jobs.package.env.OPENBITFUN_ENABLE_UPDATER_ARTIFACTS, '${{ needs.prepare.outputs.upload_to_release }}');
   assert.match(workflow.concurrency.group, /inputs.target_platform/);
   assert.equal(workflow.concurrency['cancel-in-progress'], false);
+  for (const [target, upload, run, artifacts, success] of [
+    ['macos-x64', 'true', '123', '', true], ['macos-arm64', 'true', '123', '', true],
+    ['all', 'true', '123', '', false], ['macos-x64', 'false', '123', '', false],
+    ['macos-x64', 'true', 'invalid', '', false], ['macos-x64', 'true', '123', '456', false],
+  ]) {
+    const result = spawnSync('bash', ['-c', step.run], {
+      cwd: repoRoot, encoding: 'utf8', windowsHide: true,
+      env: { ...process.env, TARGET_PLATFORM: target, RELAY_IMAGE_ONLY: 'false', ARTIFACT_RUN_IDS: artifacts,
+        VERIFICATION_RECOVERY_RUN_ID: run, UPLOAD_TO_RELEASE: upload, GITHUB_OUTPUT: path.join(root, 'verify-recovery') },
+    });
+    assert.equal(result.status === 0, success, result.stderr);
+  }
+  assert.match(workflow.jobs.package.if, /!inputs.verification_recovery_run_id/);
+  const recover = workflow.jobs['recover-macos'];
+  assert.equal(recover.name, workflow.jobs.package.name);
+  assert.equal(recover.if, "inputs.verification_recovery_run_id != ''");
+  assert.ok(recover.steps.find((entry) => entry.name === 'Verify Apple signature and notarization'));
+  const checkpoint = recover.steps.find((entry) => entry.name === 'Upload macOS verification recovery bundles');
+  assert.equal(checkpoint.if, "failure() && steps.apple_verification.outcome == 'failure'");
+  assert.doesNotMatch(JSON.stringify(recover.steps), /desktop:build|cargo build|pnpm install/);
+});
+
+test('Mac verification recovery requires the exact failed trusted producer and unexpired checkpoint', async () => {
+  const { planMacVerificationRecovery } = await import('./restore-macos-verification-bundle.mjs');
+  const repo = 'GCWing/OpenBitFun', tag = 'v1.0.6', platform = 'macos-x64';
+  const source = {
+    run: { id: 123, repository: { full_name: repo }, head_repository: { full_name: repo },
+      path: '.github/workflows/desktop-package.yml', event: 'workflow_dispatch', head_sha: 'a'.repeat(40) },
+    jobs: [{ name: 'Prepare', conclusion: 'success' }, { name: 'Package (macos-x64)', conclusion: 'failure', steps: [
+      { name: 'Build desktop app', conclusion: 'success' },
+      { name: 'Verify Apple signature and notarization', conclusion: 'failure' },
+      { name: 'Upload macOS verification recovery bundles', conclusion: 'success' },
+    ] }],
+    artifacts: [{ name: 'openbitfun-v1.0.6-macos-x64-verification-recovery', expired: false }],
+  };
+  assert.equal(planMacVerificationRecovery(repo, tag, platform, source).runId, '123');
+  const mutations = [
+    (s) => { s.run.head_repository.full_name = 'untrusted/fork'; },
+    (s) => { s.run.repository.full_name = 'untrusted/fork'; },
+    (s) => { s.run.path = '.github/workflows/ci.yml'; },
+    (s) => { s.run.event = 'pull_request'; }, (s) => { s.run.head_sha = 'invalid'; },
+    (s) => { s.jobs[0].conclusion = 'failure'; },
+    (s) => { s.jobs[1].conclusion = 'success'; },
+    (s) => { s.jobs[1].steps[0].conclusion = 'failure'; },
+    (s) => { s.jobs[1].steps[1].conclusion = 'skipped'; },
+    (s) => { s.jobs[1].steps[2].conclusion = 'failure'; },
+    (s) => { s.jobs[1].steps.push({ name: 'Other stage', conclusion: 'failure' }); },
+    (s) => { s.artifacts[0].expired = true; }, (s) => { s.artifacts = []; },
+    (s) => { s.artifacts.push({ ...s.artifacts[0] }); },
+  ];
+  for (const mutate of mutations) {
+    const next = structuredClone(source); mutate(next);
+    assert.throws(() => planMacVerificationRecovery(repo, tag, platform, next));
+  }
+  assert.throws(() => planMacVerificationRecovery(repo, tag, 'macos-arm64', source));
+  const resumed = structuredClone(source);
+  resumed.jobs[1].steps.splice(0, 1, { name: 'Restore prior macOS bundles', conclusion: 'success' },
+    { name: 'Repair missing application ticket', conclusion: 'success' });
+  assert.equal(planMacVerificationRecovery(repo, tag, platform, resumed).runId, '123');
+});
+
+test('Mac ticket repair repacks both distributions and fails closed on unrelated errors', (t) => {
+  const root = mkdtempSync(path.join(tmpdir(), 'openbitfun-staple-repair-'));
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+  const bundle = path.join(root, 'target/x86_64-apple-darwin/release/bundle');
+  const app = path.join(bundle, 'macos/OpenBitFun.app');
+  const dmg = path.join(bundle, 'dmg/OpenBitFun_1.0.6_x64.dmg');
+  mkdirSync(path.join(root, 'bin'));
+  mkdirSync(path.join(root, 'scripts'));
+  mkdirSync(path.join(app, 'Contents'), { recursive: true });
+  mkdirSync(path.dirname(dmg), { recursive: true });
+  copyFileSync(path.join(repoRoot, 'scripts/write-minisign-public-key.mjs'), path.join(root, 'scripts/write-minisign-public-key.mjs'));
+  function stub(name, source) {
+    writeFileSync(path.join(root, 'bin', name), `#!/usr/bin/env bash\nset -euo pipefail\n${source}\n`, { mode: 0o755 });
+  }
+  stub('codesign', 'if [[ "$1" == -dv ]]; then echo "Authority=Developer ID Application: Fixture"; fi');
+  stub('spctl', 'if [[ "${REJECT_GATEKEEPER:-}" == true ]]; then exit 1; fi');
+  stub('minisign', 'if [[ "${REJECT_SIGNATURE:-}" == true ]]; then exit 1; fi');
+  stub('xcrun', `
+    printf '%s\\n' "$*" >> "$FIXTURE/calls"
+    app="$3"
+    if [[ "$2" == staple ]]; then
+      printf stapled > "$app/Contents/stapled-fixture"
+    elif [[ "\${TICKET_ERROR:-}" == network ]]; then
+      echo 'Error Domain=NSURLErrorDomain Code=-1005'; exit 68
+    elif [[ "\${TICKET_ERROR:-}" == invalid ]]; then
+      echo 'Invalid ticket'; exit 65
+    elif [[ "\${VALID_TICKET:-}" != true && ! -f "$app/Contents/stapled-fixture" ]]; then
+      echo 'OpenBitFun.app does not have a ticket stapled to it.'; exit 65
+    fi
+  `);
+  stub('hdiutil', `
+    case "$1" in
+      convert)
+        format="$3"; output="$5"; input="$6"
+        if [[ "$format" == UDRW ]]; then cp "$input" "$output";
+        else mount="$(cat "$FIXTURE/mount-path")"; cat "$mount/OpenBitFun.app/Contents/stapled-fixture" > "$output"; fi
+        ;;
+      attach)
+        mount="$5"; printf '%s' "$mount" > "$FIXTURE/mount-path"
+        mkdir -p "$mount/OpenBitFun.app"
+        ;;
+      detach) ;;
+    esac
+  `);
+  stub('ditto', 'cp -R "$1" "$2"');
+  writeFileSync(path.join(root, 'scripts/sign-release-assets.sh'), '#!/usr/bin/env bash\nset -euo pipefail\nprintf signed | base64 > "$1.sig"\n');
+  const env = { ...process.env, PATH: `${path.join(root, 'bin')}:${process.env.PATH}`, FIXTURE: root,
+    OPENBITFUN_APPLE_SIGNING_CONFIGURED: 'true', APPLE_SIGNING_IDENTITY: 'Fixture Developer ID',
+    OPENBITFUN_SIGNING_KEY: 'fixture', OPENBITFUN_SIGNING_PUBKEY: 'untrusted comment: fixture\nfixture-key' };
+  function reset() {
+    rmSync(path.join(app, 'Contents/stapled-fixture'), { force: true });
+    rmSync(`${dmg}.notary-submission.json`, { force: true });
+    writeFileSync(dmg, 'original dmg');
+    writeFileSync(`${app}.tar.gz`, 'original updater');
+    writeFileSync(`${app}.tar.gz.sig`, Buffer.from('original signature').toString('base64'));
+    writeFileSync(path.join(root, 'calls'), '');
+  }
+  function execute(overrides = {}) {
+    return spawnSync('bash', [path.join(repoRoot, 'scripts/ci/repair-macos-stapling.sh'), 'x86_64-apple-darwin'], {
+      cwd: root, env: { ...env, ...overrides }, encoding: 'utf8', windowsHide: true,
+    });
+  }
+  for (const overrides of [{ TICKET_ERROR: 'network' }, { TICKET_ERROR: 'invalid' }, { REJECT_SIGNATURE: 'true' }]) {
+    reset(); const result = execute(overrides);
+    assert.notEqual(result.status, 0);
+    assert.equal(readFileSync(dmg, 'utf8'), 'original dmg');
+    assert.equal(existsSync(path.join(app, 'Contents/stapled-fixture')), false);
+  }
+  reset();
+  writeFileSync(`${dmg}.notary-submission.json`, 'retained receipt');
+  assert.notEqual(execute().status, 0);
+  assert.equal(readFileSync(dmg, 'utf8'), 'original dmg');
+  assert.equal(execute({ VALID_TICKET: 'true' }).status, 0, 'existing valid ticket retains the submission and bytes');
+  assert.equal(readFileSync(`${dmg}.notary-submission.json`, 'utf8'), 'retained receipt');
+  reset();
+  writeFileSync(`${dmg}.sig`, 'obsolete signature for original bytes');
+  const repaired = execute();
+  assert.equal(repaired.status, 0, repaired.stderr);
+  assert.equal(readFileSync(dmg, 'utf8'), 'stapled');
+  assert.equal(existsSync(`${dmg}.sig`), false);
+  assert.equal(Buffer.from(readFileSync(`${app}.tar.gz.sig`, 'utf8'), 'base64').toString(), 'signed');
+  const contents = spawnSync('tar', ['-tzf', `${app}.tar.gz`], { encoding: 'utf8', windowsHide: true });
+  assert.equal(contents.status, 0, contents.stderr);
+  assert.match(contents.stdout, /OpenBitFun.app\/Contents\/stapled-fixture/);
+  assert.equal(readFileSync(path.join(root, 'calls'), 'utf8').match(/stapler staple/g)?.length, 1);
 });