CI: version native helpers independently and split preview/release native pipelines

.bumpy/_config.json

@@ -9,9 +9,11 @@
       "internalAuthors": ["theoephraim", "philmillman", "@app/copilot-swe-agent", "claude"]
     }
   ],
-  // the per-platform native binary packages must always publish in lockstep with
-  // varlock, which pins them as exact-version optionalDependencies
-  "fixed": [["varlock", "@varlock/native-helper-*"]],
+  // The native helper packages are versioned independently of varlock and only
+  // released when their binaries change (CI fails a PR that changes native sources
+  // without a helper bump file; see scripts/native-sources.ts). The three Rust
+  // helpers are built from one crate, so they always release together.
+  "fixed": [["@varlock/native-helper-linux-*", "@varlock/native-helper-win32-x64"]],
   "packages": {
     // this is the vscode extension
     "env-spec-language": {
@@ -30,6 +32,12 @@
     },
     "varlock": {
       "cascadeFrom": {
+        // varlock pins the helpers as exact-version optionalDependencies, so a
+        // helper release must ship a varlock release carrying the new pin
+        "@varlock/native-helper-*": {
+          "trigger": "patch",
+          "bumpAs": "patch"
+        },
         "@env-spec/parser": {
           "trigger": "patch",
           "bumpAs": "patch"

.bumpy/darwin-adhoc-build-flag.md

@@ -0,0 +1,3 @@
+---
+"@varlock/native-helper-darwin": none
+---

.bumpy/native-helpers-independent-versions.md

@@ -0,0 +1,5 @@
+---
+varlock: patch
+---
+
+Native helper packages (`@varlock/native-helper-*`) are now versioned independently and only re-released when their binaries change. varlock still pins exact helper versions.

.github/workflows/binary-preview.yaml

@@ -23,34 +23,53 @@ jobs:
       (github.event.action == 'labeled' && github.event.label.name == 'preview:standalone') ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'preview:standalone'))
     runs-on: depot-ubuntu-24.04
+    outputs:
+      macos-pending: ${{ steps.native.outputs.macos-pending }}
+      rust-pending: ${{ steps.native.outputs.rust-pending }}
+      macos-key: ${{ steps.native.outputs.macos-key }}
+      rust-key: ${{ steps.native.outputs.rust-key }}
     steps:
-      - run: echo "preview:standalone label detected — building preview binaries"
+      - run: echo "preview:standalone label detected, building preview binaries"
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0 # native-sources.ts diffs against the merge base with main
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+      # installs the repo's pinned bumpy, which native-sources.ts calls
+      - name: Install node deps
+        run: bun install
+      - name: Native helper sources
+        id: native
+        run: bun run scripts/native-sources.ts
 
-  # Build and sign the macOS native binary (cache-aware)
+  # Native helpers: when a side's helpers have no pending release, the published
+  # @varlock/native-helper-* binaries at their pinned version are used as-is (see
+  # build-sea-binaries). Otherwise build unsigned preview binaries; no secrets.
   build-native-macos:
     needs: check-label
+    if: needs.check-label.outputs.macos-pending == 'true'
     uses: ./.github/workflows/build-native-macos.yaml
     with:
       mode: preview
       artifact-name: native-bin-macos-preview
-    secrets:
-      OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }}
-      DEPOT_TURBO_TOKEN: ${{ secrets.DEPOT_TURBO_TOKEN }}
+      cache-key: ${{ needs.check-label.outputs.macos-key }}
 
-  # Build Rust native binaries for Linux and Windows
   build-native-rust:
     needs: check-label
-    # Preview builds never sign, so no id-token is granted (signing is gated on
-    # the `sign` input, which only the release workflows pass).
+    if: needs.check-label.outputs.rust-pending == 'true'
     permissions:
       contents: read
     uses: ./.github/workflows/build-native-rust.yaml
     with:
       artifact-name: native-bin-rust-preview
+      cache-key: ${{ needs.check-label.outputs.rust-key }}
 
   # Build SEA binaries for all platforms and upload as artifacts
   build-sea-binaries:
-    needs: [build-native-macos, build-native-rust]
+    needs: [check-label, build-native-macos, build-native-rust]
+    if: >-
+      always() && !failure() && !cancelled()
+      && needs.check-label.result == 'success'
     runs-on: depot-ubuntu-24.04-8
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -67,27 +86,41 @@ jobs:
         with:
           token: ${{ secrets.DEPOT_TURBO_TOKEN }}
 
-      # Download macOS native binary
+      # Published helpers for sides with no pending helper release
+      - name: Fetch published native helpers from npm
+        env:
+          MACOS: ${{ needs.check-label.outputs.macos-pending }}
+          RUST: ${{ needs.check-label.outputs.rust-pending }}
+        run: |
+          PLATFORMS=""
+          if [ "$MACOS" != "true" ]; then PLATFORMS="darwin"; fi
+          if [ "$RUST" != "true" ]; then PLATFORMS="$PLATFORMS,linux-x64,linux-arm64,win32-x64"; fi
+          if [ -n "${PLATFORMS#,}" ]; then bun run scripts/fetch-native-helpers.ts --platforms "${PLATFORMS#,}"; fi
+
+      # Preview builds from this run for sides with a pending helper release
       - name: Download macOS native binary
+        if: needs.check-label.outputs.macos-pending == 'true'
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: native-bin-macos-preview
           path: packages/varlock/native-bins/darwin/VarlockEnclave.app
       - name: Restore macOS binary permissions
         run: chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt
 
-      # Download Rust native binaries
       - name: Download Linux x64 native binary
+        if: needs.check-label.outputs.rust-pending == 'true'
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: native-bin-rust-preview-linux-x64
           path: packages/varlock/native-bins/linux-x64
       - name: Download Linux arm64 native binary
+        if: needs.check-label.outputs.rust-pending == 'true'
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: native-bin-rust-preview-linux-arm64
           path: packages/varlock/native-bins/linux-arm64
       - name: Download Windows x64 native binary
+        if: needs.check-label.outputs.rust-pending == 'true'
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: native-bin-rust-preview-win32-x64

.github/workflows/binary-release.yaml

@@ -6,9 +6,9 @@ name: Re-cut varlock CLI binaries
 # The normal release flow (release.yaml) builds and uploads these in the same run
 # as the npm publish, reusing the in-run signed/notarized native binaries. This
 # workflow is only for re-cutting binaries of an existing version. It pulls the
-# signed native binaries from the published @varlock/native-helper-* npm packages
-# rather than rebuilding + re-signing them, so it needs no Windows runner and no
-# Azure. The macOS CLI binary itself is not published anywhere it could be reused
+# signed native binaries from the @varlock/native-helper-* versions that this
+# varlock version pins on npm rather than rebuilding + re-signing them, so it
+# needs no Windows runner and no Azure. The macOS CLI binary itself is not published anywhere it could be reused
 # from, so it is rebuilt, Developer ID signed, and notarized on a macOS runner,
 # which needs the Apple credentials.
 #
@@ -65,41 +65,10 @@ jobs:
           cosign-release: v3.0.6
 
       # Pull the signed/notarized native binaries from the published npm packages
-      # instead of rebuilding + re-signing them.
+      # (the helper versions that varlock@<version> pins) instead of rebuilding
+      # + re-signing them.
       - name: Fetch native binaries from published npm packages
-        run: |
-          set -euo pipefail
-          TMP="$RUNNER_TEMP/varlock-npm"
-          mkdir -p "$TMP" && cd "$TMP"
-          # wait for registry propagation of every package we are about to pack
-          wait_for() { # <pkg spec> <max attempts>
-            for i in $(seq 1 "$2"); do
-              if npm view "$1" version >/dev/null 2>&1; then return 0; fi
-              echo "waiting for $1 on npm ($i)..."; sleep 10
-            done
-            echo "::error::$1 not found on npm. Note: versions <= 1.16.x predate the per-platform @varlock/native-helper-* packages and cannot be re-cut by this workflow."
-            return 1
-          }
-          wait_for "varlock@${RELEASE_VERSION}" 30
-          for SUFFIX in darwin linux-x64 linux-arm64 win32-x64; do
-            # helpers publish before varlock, so short extra wait once varlock is visible
-            wait_for "@varlock/native-helper-${SUFFIX}@${RELEASE_VERSION}" 12
-          done
-          DEST="$GITHUB_WORKSPACE/packages/varlock/native-bins"
-          rm -rf "$DEST"
-          # platform suffix -> tarball entry mapping; source of truth is
-          # packages/native-helpers/platforms.ts - keep in sync
-          fetch() { # <platform suffix> <tarball entry>
-            rm -rf package
-            npm pack "@varlock/native-helper-$1@${RELEASE_VERSION}"
-            tar -xzf "varlock-native-helper-$1-${RELEASE_VERSION}.tgz"
-            mkdir -p "$DEST/$1"
-            cp -R "package/$2" "$DEST/$1/$2"
-          }
-          fetch darwin VarlockEnclave.app
-          fetch linux-x64 varlock-local-encrypt
-          fetch linux-arm64 varlock-local-encrypt
-          fetch win32-x64 varlock-local-encrypt.exe
+        run: bun run scripts/fetch-native-helpers.ts --varlock-version "$RELEASE_VERSION" --wait
       - name: Restore native binary execute permissions
         run: |
           chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt

.github/workflows/build-cli-binaries-macos.yaml

@@ -77,26 +77,13 @@ jobs:
           name: ${{ inputs.native-bins-artifact }}
           path: packages/varlock/native-bins
 
-      # Re-cut path: reuse the signed + notarized helper from the published
-      # @varlock/native-helper-darwin package instead of rebuilding it
+      # Re-cut path: reuse the signed + notarized darwin helper that the published
+      # varlock pins instead of rebuilding it
       - name: Fetch darwin native binary from npm
         if: inputs.native-bins-npm-version != ''
         env:
           RELEASE_VERSION: ${{ inputs.native-bins-npm-version }}
-        run: |
-          set -euo pipefail
-          PKG="@varlock/native-helper-darwin@${RELEASE_VERSION}"
-          TMP="$RUNNER_TEMP/varlock-npm"
-          mkdir -p "$TMP" && cd "$TMP"
-          for i in $(seq 1 30); do
-            if npm view "$PKG" version >/dev/null 2>&1; then break; fi
-            echo "waiting for $PKG on npm ($i)..."; sleep 10
-          done
-          npm pack "$PKG"
-          tar -xzf "varlock-native-helper-darwin-${RELEASE_VERSION}.tgz"
-          DEST="$GITHUB_WORKSPACE/packages/varlock/native-bins/darwin"
-          rm -rf "$DEST" && mkdir -p "$DEST"
-          cp -R package/VarlockEnclave.app "$DEST/VarlockEnclave.app"
+        run: bun run scripts/fetch-native-helpers.ts --platforms darwin --varlock-version "$RELEASE_VERSION" --wait
 
       - name: Restore native binary execute permissions
         run: chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt

.github/workflows/build-native-macos.yaml

@@ -1,14 +1,17 @@
 name: Build macOS native binary
 
-# Reusable workflow that compiles, bundles, and Developer ID signs the
-# VarlockEnclave Swift binary on a macOS runner.
+# Reusable workflow that compiles and bundles the VarlockEnclave Swift binary on
+# a macOS runner, and either Developer ID signs it (`sign: true`, releases only)
+# or ad-hoc signs it (previews, no secrets needed).
 #
-# The Swift .build directory is cached by source hash, so the compile
-# step (~minutes) is near-instant on cache hit. The .app bundle wrapping
-# (plist, icon, signing) always runs since it varies by mode.
+# Ad-hoc is enough for previews: npm installs set no quarantine attribute, so
+# Gatekeeper never checks them, and Secure Enclave keys are not bound to the
+# signing identity. The one difference users notice is the keychain() resolver:
+# keychain grants are recorded per signing identity, and an ad-hoc identity is
+# the exact binary hash, so each preview build asks for keychain access again.
 #
-# Notarization is intentionally NOT included here — it's a separate
-# workflow for production releases.
+# Notarization is intentionally NOT included here; it's a separate workflow
+# for production releases.
 
 permissions:
   contents: read
@@ -24,9 +27,22 @@ on:
         description: 'Name for the uploaded artifact'
         type: string
         default: 'native-bin-macos'
+      sign:
+        description: 'Developer ID sign the bundle. Only release builds pass this, along with OP_CI_TOKEN.'
+        type: boolean
+        default: false
+      use-build-cache:
+        description: 'Restore/save the Swift .build directory. Release builds turn this off so a published binary never contains cached build output.'
+        type: boolean
+        default: true
+      cache-key:
+        description: 'If set, the built .app is saved to the Actions cache under this key (preview builds only)'
+        type: string
+        default: ''
     secrets:
       OP_CI_TOKEN:
-        required: true
+        description: '1Password service-account token for the Apple signing credentials. Required only when sign is true.'
+        required: false
       # PR callers pass this so the varlock libs build reuses Depot's turbo cache.
       # Release callers leave it out, so release builds always run from source.
       DEPOT_TURBO_TOKEN:
@@ -54,23 +70,27 @@ jobs:
       # Cache the Swift .build directory so compilation is fast on unchanged source
       - name: Compute Swift source hash
         id: swift-hash
+        if: inputs.use-build-cache
         run: |
           HASH=$(find packages/encryption-binary-swift/swift -type f | sort | xargs shasum -a 256 | shasum -a 256 | cut -d' ' -f1)
           echo "hash=$HASH" >> $GITHUB_OUTPUT
           echo "Swift source hash: $HASH"
 
       - name: Cache Swift build artifacts
+        if: inputs.use-build-cache
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: packages/encryption-binary-swift/swift/.build
           key: varlock-swift-build-${{ steps.swift-hash.outputs.hash }}
 
       # Build varlock JS so we can use it to resolve secrets from 1Password
       - name: Build varlock libs
+        if: inputs.sign
         run: bun run build:libs
 
       # Load secrets from 1Password via varlock (scoped to the Swift package)
       - name: Load signing secrets
+        if: inputs.sign
         uses: dmno-dev/varlock-action@7141115d0060616e398c5fa9035e67d7820ab2b6 # v1.0.5
         with:
           working-directory: packages/encryption-binary-swift
@@ -79,6 +99,7 @@ jobs:
 
       # Import signing certificate into a temporary keychain
       - name: Import signing certificate
+        if: inputs.sign
         run: |
           KEYCHAIN_PATH=$RUNNER_TEMP/signing.keychain-db
           KEYCHAIN_PASSWORD=$(openssl rand -base64 24)
@@ -101,11 +122,17 @@ jobs:
 
           echo "APPLE_SIGNING_IDENTITY=$APPLE_SIGNING_IDENTITY" >> $GITHUB_ENV
 
-      # Compile (cached), bundle with mode-specific metadata, and sign
+      # Compile, bundle with mode-specific metadata, and sign
       - name: Build, bundle, and sign
+        env:
+          SIGN: ${{ inputs.sign }}
+          MODE: ${{ inputs.mode }}
         run: |
-          bun run --filter @varlock/encryption-binary-swift build:universal \
-            -- --mode ${{ inputs.mode }} --sign "$APPLE_SIGNING_IDENTITY"
+          if [ "$SIGN" = "true" ]; then
+            bun run --filter @varlock/encryption-binary-swift build:universal -- --mode "$MODE" --sign "$APPLE_SIGNING_IDENTITY"
+          else
+            bun run --filter @varlock/encryption-binary-swift build:universal -- --mode "$MODE" --adhoc
+          fi
 
       - name: Verify binary
         run: |
@@ -209,17 +236,18 @@ jobs:
           path: packages/varlock/native-bins/darwin/VarlockEnclave.app
           retention-days: 7
 
-      # Cache the signed .app so other jobs (e.g. release-preview) can restore
-      # it on a Linux runner without needing a macOS build
-      - name: Cache signed .app bundle
+      # Preview builds only: lets later runs (other pushes, other PRs once saved
+      # from main) reuse an identical build. Releases never pass a key.
+      - name: Cache built .app bundle
+        if: inputs.cache-key != ''
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         continue-on-error: true  # cache key is immutable; fails harmlessly if already saved by a concurrent run
         with:
           path: packages/varlock/native-bins/darwin/VarlockEnclave.app
-          key: native-bin-macos-signed-${{ hashFiles('packages/encryption-binary-swift/swift/Package.swift', 'packages/encryption-binary-swift/swift/Sources/**') }}
+          key: ${{ inputs.cache-key }}
 
       - name: Cleanup signing keychain
-        if: always()
+        if: always() && inputs.sign
         run: |
           KEYCHAIN_PATH=$RUNNER_TEMP/signing.keychain-db
           if [ -f "$KEYCHAIN_PATH" ]; then