diff --git a/.bumpy/_config.json b/.bumpy/_config.json index 5aebe9d36..709ae5367 100644 --- a/.bumpy/_config.json +++ b/.bumpy/_config.json @@ -9,9 +9,11 @@ "internalAuthors": ["theoephraim", "philmillman", "@app/copilot-swe-agent", "claude"] } ], - // the per-platform native binary packages must always publish in lockstep with - // varlock, which pins them as exact-version optionalDependencies - "fixed": [["varlock", "@varlock/native-helper-*"]], + // The native helper packages are versioned independently of varlock and only + // released when their binaries change (CI fails a PR that changes native sources + // without a helper bump file; see scripts/native-sources.ts). The three Rust + // helpers are built from one crate, so they always release together. + "fixed": [["@varlock/native-helper-linux-*", "@varlock/native-helper-win32-x64"]], "packages": { // this is the vscode extension "env-spec-language": { @@ -30,6 +32,12 @@ }, "varlock": { "cascadeFrom": { + // varlock pins the helpers as exact-version optionalDependencies, so a + // helper release must ship a varlock release carrying the new pin + "@varlock/native-helper-*": { + "trigger": "patch", + "bumpAs": "patch" + }, "@env-spec/parser": { "trigger": "patch", "bumpAs": "patch" diff --git a/.bumpy/darwin-adhoc-build-flag.md b/.bumpy/darwin-adhoc-build-flag.md new file mode 100644 index 000000000..7b7e9ca72 --- /dev/null +++ b/.bumpy/darwin-adhoc-build-flag.md @@ -0,0 +1,3 @@ +--- +"@varlock/native-helper-darwin": none +--- diff --git a/.bumpy/native-helpers-independent-versions.md b/.bumpy/native-helpers-independent-versions.md new file mode 100644 index 000000000..1a5c22d3f --- /dev/null +++ b/.bumpy/native-helpers-independent-versions.md @@ -0,0 +1,5 @@ +--- +varlock: patch +--- + +Native helper packages (`@varlock/native-helper-*`) are now versioned independently and only re-released when their binaries change. varlock still pins exact helper versions. diff --git a/.github/workflows/binary-preview.yaml b/.github/workflows/binary-preview.yaml index 0b08fdccf..6ee88a7cb 100644 --- a/.github/workflows/binary-preview.yaml +++ b/.github/workflows/binary-preview.yaml @@ -23,34 +23,53 @@ jobs: (github.event.action == 'labeled' && github.event.label.name == 'preview:standalone') || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'preview:standalone')) runs-on: depot-ubuntu-24.04 + outputs: + macos-pending: ${{ steps.native.outputs.macos-pending }} + rust-pending: ${{ steps.native.outputs.rust-pending }} + macos-key: ${{ steps.native.outputs.macos-key }} + rust-key: ${{ steps.native.outputs.rust-key }} steps: - - run: echo "preview:standalone label detected — building preview binaries" + - run: echo "preview:standalone label detected, building preview binaries" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # native-sources.ts diffs against the merge base with main + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + # installs the repo's pinned bumpy, which native-sources.ts calls + - name: Install node deps + run: bun install + - name: Native helper sources + id: native + run: bun run scripts/native-sources.ts - # Build and sign the macOS native binary (cache-aware) + # Native helpers: when a side's helpers have no pending release, the published + # @varlock/native-helper-* binaries at their pinned version are used as-is (see + # build-sea-binaries). Otherwise build unsigned preview binaries; no secrets. build-native-macos: needs: check-label + if: needs.check-label.outputs.macos-pending == 'true' uses: ./.github/workflows/build-native-macos.yaml with: mode: preview artifact-name: native-bin-macos-preview - secrets: - OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }} - DEPOT_TURBO_TOKEN: ${{ secrets.DEPOT_TURBO_TOKEN }} + cache-key: ${{ needs.check-label.outputs.macos-key }} - # Build Rust native binaries for Linux and Windows build-native-rust: needs: check-label - # Preview builds never sign, so no id-token is granted (signing is gated on - # the `sign` input, which only the release workflows pass). + if: needs.check-label.outputs.rust-pending == 'true' permissions: contents: read uses: ./.github/workflows/build-native-rust.yaml with: artifact-name: native-bin-rust-preview + cache-key: ${{ needs.check-label.outputs.rust-key }} # Build SEA binaries for all platforms and upload as artifacts build-sea-binaries: - needs: [build-native-macos, build-native-rust] + needs: [check-label, build-native-macos, build-native-rust] + if: >- + always() && !failure() && !cancelled() + && needs.check-label.result == 'success' runs-on: depot-ubuntu-24.04-8 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -67,8 +86,20 @@ jobs: with: token: ${{ secrets.DEPOT_TURBO_TOKEN }} - # Download macOS native binary + # Published helpers for sides with no pending helper release + - name: Fetch published native helpers from npm + env: + MACOS: ${{ needs.check-label.outputs.macos-pending }} + RUST: ${{ needs.check-label.outputs.rust-pending }} + run: | + PLATFORMS="" + if [ "$MACOS" != "true" ]; then PLATFORMS="darwin"; fi + if [ "$RUST" != "true" ]; then PLATFORMS="$PLATFORMS,linux-x64,linux-arm64,win32-x64"; fi + if [ -n "${PLATFORMS#,}" ]; then bun run scripts/fetch-native-helpers.ts --platforms "${PLATFORMS#,}"; fi + + # Preview builds from this run for sides with a pending helper release - name: Download macOS native binary + if: needs.check-label.outputs.macos-pending == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-macos-preview @@ -76,18 +107,20 @@ jobs: - name: Restore macOS binary permissions run: chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt - # Download Rust native binaries - name: Download Linux x64 native binary + if: needs.check-label.outputs.rust-pending == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-rust-preview-linux-x64 path: packages/varlock/native-bins/linux-x64 - name: Download Linux arm64 native binary + if: needs.check-label.outputs.rust-pending == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-rust-preview-linux-arm64 path: packages/varlock/native-bins/linux-arm64 - name: Download Windows x64 native binary + if: needs.check-label.outputs.rust-pending == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-rust-preview-win32-x64 diff --git a/.github/workflows/binary-release.yaml b/.github/workflows/binary-release.yaml index 8fb8b2f38..1aa90d109 100644 --- a/.github/workflows/binary-release.yaml +++ b/.github/workflows/binary-release.yaml @@ -6,9 +6,9 @@ name: Re-cut varlock CLI binaries # The normal release flow (release.yaml) builds and uploads these in the same run # as the npm publish, reusing the in-run signed/notarized native binaries. This # workflow is only for re-cutting binaries of an existing version. It pulls the -# signed native binaries from the published @varlock/native-helper-* npm packages -# rather than rebuilding + re-signing them, so it needs no Windows runner and no -# Azure. The macOS CLI binary itself is not published anywhere it could be reused +# signed native binaries from the @varlock/native-helper-* versions that this +# varlock version pins on npm rather than rebuilding + re-signing them, so it +# needs no Windows runner and no Azure. The macOS CLI binary itself is not published anywhere it could be reused # from, so it is rebuilt, Developer ID signed, and notarized on a macOS runner, # which needs the Apple credentials. # @@ -65,41 +65,10 @@ jobs: cosign-release: v3.0.6 # Pull the signed/notarized native binaries from the published npm packages - # instead of rebuilding + re-signing them. + # (the helper versions that varlock@ pins) instead of rebuilding + # + re-signing them. - name: Fetch native binaries from published npm packages - run: | - set -euo pipefail - TMP="$RUNNER_TEMP/varlock-npm" - mkdir -p "$TMP" && cd "$TMP" - # wait for registry propagation of every package we are about to pack - wait_for() { # - for i in $(seq 1 "$2"); do - if npm view "$1" version >/dev/null 2>&1; then return 0; fi - echo "waiting for $1 on npm ($i)..."; sleep 10 - done - echo "::error::$1 not found on npm. Note: versions <= 1.16.x predate the per-platform @varlock/native-helper-* packages and cannot be re-cut by this workflow." - return 1 - } - wait_for "varlock@${RELEASE_VERSION}" 30 - for SUFFIX in darwin linux-x64 linux-arm64 win32-x64; do - # helpers publish before varlock, so short extra wait once varlock is visible - wait_for "@varlock/native-helper-${SUFFIX}@${RELEASE_VERSION}" 12 - done - DEST="$GITHUB_WORKSPACE/packages/varlock/native-bins" - rm -rf "$DEST" - # platform suffix -> tarball entry mapping; source of truth is - # packages/native-helpers/platforms.ts - keep in sync - fetch() { # - rm -rf package - npm pack "@varlock/native-helper-$1@${RELEASE_VERSION}" - tar -xzf "varlock-native-helper-$1-${RELEASE_VERSION}.tgz" - mkdir -p "$DEST/$1" - cp -R "package/$2" "$DEST/$1/$2" - } - fetch darwin VarlockEnclave.app - fetch linux-x64 varlock-local-encrypt - fetch linux-arm64 varlock-local-encrypt - fetch win32-x64 varlock-local-encrypt.exe + run: bun run scripts/fetch-native-helpers.ts --varlock-version "$RELEASE_VERSION" --wait - name: Restore native binary execute permissions run: | chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt diff --git a/.github/workflows/build-cli-binaries-macos.yaml b/.github/workflows/build-cli-binaries-macos.yaml index c453f19f8..3ebe6e605 100644 --- a/.github/workflows/build-cli-binaries-macos.yaml +++ b/.github/workflows/build-cli-binaries-macos.yaml @@ -77,26 +77,13 @@ jobs: name: ${{ inputs.native-bins-artifact }} path: packages/varlock/native-bins - # Re-cut path: reuse the signed + notarized helper from the published - # @varlock/native-helper-darwin package instead of rebuilding it + # Re-cut path: reuse the signed + notarized darwin helper that the published + # varlock pins instead of rebuilding it - name: Fetch darwin native binary from npm if: inputs.native-bins-npm-version != '' env: RELEASE_VERSION: ${{ inputs.native-bins-npm-version }} - run: | - set -euo pipefail - PKG="@varlock/native-helper-darwin@${RELEASE_VERSION}" - TMP="$RUNNER_TEMP/varlock-npm" - mkdir -p "$TMP" && cd "$TMP" - for i in $(seq 1 30); do - if npm view "$PKG" version >/dev/null 2>&1; then break; fi - echo "waiting for $PKG on npm ($i)..."; sleep 10 - done - npm pack "$PKG" - tar -xzf "varlock-native-helper-darwin-${RELEASE_VERSION}.tgz" - DEST="$GITHUB_WORKSPACE/packages/varlock/native-bins/darwin" - rm -rf "$DEST" && mkdir -p "$DEST" - cp -R package/VarlockEnclave.app "$DEST/VarlockEnclave.app" + run: bun run scripts/fetch-native-helpers.ts --platforms darwin --varlock-version "$RELEASE_VERSION" --wait - name: Restore native binary execute permissions run: chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt diff --git a/.github/workflows/build-native-macos.yaml b/.github/workflows/build-native-macos.yaml index 00e3c767b..d12266e59 100644 --- a/.github/workflows/build-native-macos.yaml +++ b/.github/workflows/build-native-macos.yaml @@ -1,14 +1,17 @@ name: Build macOS native binary -# Reusable workflow that compiles, bundles, and Developer ID signs the -# VarlockEnclave Swift binary on a macOS runner. +# Reusable workflow that compiles and bundles the VarlockEnclave Swift binary on +# a macOS runner, and either Developer ID signs it (`sign: true`, releases only) +# or ad-hoc signs it (previews, no secrets needed). # -# The Swift .build directory is cached by source hash, so the compile -# step (~minutes) is near-instant on cache hit. The .app bundle wrapping -# (plist, icon, signing) always runs since it varies by mode. +# Ad-hoc is enough for previews: npm installs set no quarantine attribute, so +# Gatekeeper never checks them, and Secure Enclave keys are not bound to the +# signing identity. The one difference users notice is the keychain() resolver: +# keychain grants are recorded per signing identity, and an ad-hoc identity is +# the exact binary hash, so each preview build asks for keychain access again. # -# Notarization is intentionally NOT included here — it's a separate -# workflow for production releases. +# Notarization is intentionally NOT included here; it's a separate workflow +# for production releases. permissions: contents: read @@ -24,9 +27,22 @@ on: description: 'Name for the uploaded artifact' type: string default: 'native-bin-macos' + sign: + description: 'Developer ID sign the bundle. Only release builds pass this, along with OP_CI_TOKEN.' + type: boolean + default: false + use-build-cache: + description: 'Restore/save the Swift .build directory. Release builds turn this off so a published binary never contains cached build output.' + type: boolean + default: true + cache-key: + description: 'If set, the built .app is saved to the Actions cache under this key (preview builds only)' + type: string + default: '' secrets: OP_CI_TOKEN: - required: true + description: '1Password service-account token for the Apple signing credentials. Required only when sign is true.' + required: false # PR callers pass this so the varlock libs build reuses Depot's turbo cache. # Release callers leave it out, so release builds always run from source. DEPOT_TURBO_TOKEN: @@ -54,12 +70,14 @@ jobs: # Cache the Swift .build directory so compilation is fast on unchanged source - name: Compute Swift source hash id: swift-hash + if: inputs.use-build-cache run: | HASH=$(find packages/encryption-binary-swift/swift -type f | sort | xargs shasum -a 256 | shasum -a 256 | cut -d' ' -f1) echo "hash=$HASH" >> $GITHUB_OUTPUT echo "Swift source hash: $HASH" - name: Cache Swift build artifacts + if: inputs.use-build-cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/encryption-binary-swift/swift/.build @@ -67,10 +85,12 @@ jobs: # Build varlock JS so we can use it to resolve secrets from 1Password - name: Build varlock libs + if: inputs.sign run: bun run build:libs # Load secrets from 1Password via varlock (scoped to the Swift package) - name: Load signing secrets + if: inputs.sign uses: dmno-dev/varlock-action@7141115d0060616e398c5fa9035e67d7820ab2b6 # v1.0.5 with: working-directory: packages/encryption-binary-swift @@ -79,6 +99,7 @@ jobs: # Import signing certificate into a temporary keychain - name: Import signing certificate + if: inputs.sign run: | KEYCHAIN_PATH=$RUNNER_TEMP/signing.keychain-db KEYCHAIN_PASSWORD=$(openssl rand -base64 24) @@ -101,11 +122,17 @@ jobs: echo "APPLE_SIGNING_IDENTITY=$APPLE_SIGNING_IDENTITY" >> $GITHUB_ENV - # Compile (cached), bundle with mode-specific metadata, and sign + # Compile, bundle with mode-specific metadata, and sign - name: Build, bundle, and sign + env: + SIGN: ${{ inputs.sign }} + MODE: ${{ inputs.mode }} run: | - bun run --filter @varlock/encryption-binary-swift build:universal \ - -- --mode ${{ inputs.mode }} --sign "$APPLE_SIGNING_IDENTITY" + if [ "$SIGN" = "true" ]; then + bun run --filter @varlock/encryption-binary-swift build:universal -- --mode "$MODE" --sign "$APPLE_SIGNING_IDENTITY" + else + bun run --filter @varlock/encryption-binary-swift build:universal -- --mode "$MODE" --adhoc + fi - name: Verify binary run: | @@ -209,17 +236,18 @@ jobs: path: packages/varlock/native-bins/darwin/VarlockEnclave.app retention-days: 7 - # Cache the signed .app so other jobs (e.g. release-preview) can restore - # it on a Linux runner without needing a macOS build - - name: Cache signed .app bundle + # Preview builds only: lets later runs (other pushes, other PRs once saved + # from main) reuse an identical build. Releases never pass a key. + - name: Cache built .app bundle + if: inputs.cache-key != '' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 continue-on-error: true # cache key is immutable; fails harmlessly if already saved by a concurrent run with: path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: native-bin-macos-signed-${{ hashFiles('packages/encryption-binary-swift/swift/Package.swift', 'packages/encryption-binary-swift/swift/Sources/**') }} + key: ${{ inputs.cache-key }} - name: Cleanup signing keychain - if: always() + if: always() && inputs.sign run: | KEYCHAIN_PATH=$RUNNER_TEMP/signing.keychain-db if [ -f "$KEYCHAIN_PATH" ]; then diff --git a/.github/workflows/build-native-rust.yaml b/.github/workflows/build-native-rust.yaml index 746d53850..a7e5bcc5d 100644 --- a/.github/workflows/build-native-rust.yaml +++ b/.github/workflows/build-native-rust.yaml @@ -3,8 +3,10 @@ name: Build Rust native binaries # Reusable workflow that compiles the varlock-local-encrypt Rust binary # for Linux and Windows targets. # -# Builds are cached by Cargo.lock + source hash. Each platform builds -# natively on its own runner for maximum compatibility. +# Each platform builds natively on its own runner for maximum compatibility. +# Preview builds reuse the Cargo build cache and can save the built binaries +# for later runs (cache-key); release builds use neither, so a published binary +# never contains cached build output. # # Output: native binaries uploaded as artifacts, ready to be bundled # into the varlock npm package and CLI release archives. @@ -23,8 +25,12 @@ on: description: 'Base name for uploaded artifacts (suffixed with platform)' type: string default: 'native-bin-rust' - source-hash: - description: 'Pre-computed hash of Rust source files (from Ubuntu runner). When provided, used for the binary cache key instead of local hashFiles to ensure cross-OS consistency.' + use-build-cache: + description: 'Restore/save the Cargo registry + target cache. Release builds turn this off.' + type: boolean + default: true + cache-key: + description: 'If set, each built binary is saved to the Actions cache under - (preview builds only)' type: string default: '' sign: @@ -72,6 +78,7 @@ jobs: # Cache Cargo registry + build artifacts by lockfile hash - name: Cache Cargo + if: inputs.use-build-cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | @@ -268,14 +275,20 @@ jobs: # Cache the built binary so preview releases on other runners can restore it. # Path must match what test.yaml's release-preview-packages restores to. - name: Prepare binary for cache + if: inputs.cache-key != '' shell: bash run: | mkdir -p "packages/varlock/native-bins/${{ matrix.native-bin-subdir }}" cp "native-bins/${{ matrix.native-bin-subdir }}/${{ matrix.binary-name }}" \ "packages/varlock/native-bins/${{ matrix.native-bin-subdir }}/" + # enableCrossOsArchive: the win32 binary is saved on Windows but probed and + # restored on Ubuntu; without it, the cache version gets a Windows-only + # component and Ubuntu lookups never match. - name: Cache built binary + if: inputs.cache-key != '' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 continue-on-error: true # cache key is immutable; fails harmlessly if already saved by a concurrent run with: path: packages/varlock/native-bins/${{ matrix.native-bin-subdir }}/ - key: native-bin-rust-${{ matrix.native-bin-subdir }}-${{ inputs.source-hash || hashFiles('packages/encryption-binary-rust/Cargo.lock', 'packages/encryption-binary-rust/src/**') }} + enableCrossOsArchive: true + key: ${{ inputs.cache-key }}-${{ matrix.native-bin-subdir }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index b1cb28d77..24ce5fd6d 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -56,8 +56,9 @@ jobs: echo "js-changed=$(check_pattern '\.(js|ts|jsx|tsx|mjs|cjs)$')" >> $GITHUB_OUTPUT echo "actions-changed=$(check_pattern '^\.github/')" >> $GITHUB_OUTPUT - echo "swift-changed=$(check_pattern '^packages/encryption-binary-swift/')" >> $GITHUB_OUTPUT - echo "rust-changed=$(check_pattern '^packages/encryption-binary-rust/')" >> $GITHUB_OUTPUT + # only the sources CodeQL compiles; build scripts are TypeScript (analyze-js) + echo "swift-changed=$(check_pattern '^packages/encryption-binary-swift/swift/')" >> $GITHUB_OUTPUT + echo "rust-changed=$(check_pattern '^packages/encryption-binary-rust/(src/|Cargo\.(toml|lock)$)')" >> $GITHUB_OUTPUT analyze-js: name: Analyze JavaScript/TypeScript diff --git a/.github/workflows/notarize-native-macos.yaml b/.github/workflows/notarize-native-macos.yaml index 826cdec12..349eda224 100644 --- a/.github/workflows/notarize-native-macos.yaml +++ b/.github/workflows/notarize-native-macos.yaml @@ -18,10 +18,6 @@ on: description: 'Name for the notarized artifact' type: string default: 'native-bin-macos-notarized' - cache-key: - description: 'If set, the notarized .app is saved to the Actions cache under this key for cross-run reuse' - type: string - default: '' secrets: OP_CI_TOKEN: required: true @@ -91,23 +87,3 @@ jobs: name: ${{ inputs.artifact-name }} path: VarlockEnclave.app retention-days: 7 - - # Stage the notarized bundle at the canonical path and cache it so future - # releases with unchanged source can reuse it (skipping build + notarize). - # ditto preserves the bundle's signature and stapled notarization ticket. - # The cache key is content-addressed (source + scripts + workflows), so a - # restored bundle always corresponds to the source that produced it; the - # release re-verifies it on macOS before publishing. - - name: Stage notarized .app for caching - if: inputs.cache-key != '' - run: | - mkdir -p packages/varlock/native-bins/darwin - rm -rf packages/varlock/native-bins/darwin/VarlockEnclave.app - ditto VarlockEnclave.app packages/varlock/native-bins/darwin/VarlockEnclave.app - - name: Cache notarized .app for cross-run reuse - if: inputs.cache-key != '' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - continue-on-error: true # immutable key; harmless if a concurrent run saved it first - with: - path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: ${{ inputs.cache-key }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ea2ab96e3..3d8a97465 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,10 +38,9 @@ jobs: echo "::error::Timed out waiting for CI test suite" exit 1 - # Determine what bumpy will do — only run expensive build/notarize when publishing. - # Also probe the native-binary caches: if this exact source (+ build scripts + - # workflows) already produced a signed/notarized binary on a previous run, we - # reuse it instead of rebuilding and — for macOS — re-notarizing. + # Determine what bumpy will do. Native helpers are versioned independently of + # varlock and only built when this publish releases them, always from source in + # this same run (see build-native-macos). plan: needs: wait-for-tests runs-on: ubuntu-latest @@ -49,11 +48,10 @@ jobs: mode: ${{ steps.plan.outputs.mode }} bumpy-version: ${{ steps.bumpy-version.outputs.version }} includes-varlock: ${{ contains(fromJSON(steps.plan.outputs.json).packageNames, 'varlock') }} + includes-macos-native: ${{ contains(fromJSON(steps.plan.outputs.json).packageNames, '@varlock/native-helper-darwin') }} + # the three Rust helpers are a bumpy fixed group, so one stands for all + includes-rust-native: ${{ contains(fromJSON(steps.plan.outputs.json).packageNames, '@varlock/native-helper-linux-x64') }} includes-vscode: ${{ contains(fromJSON(steps.plan.outputs.json).packageNames, 'env-spec-language') }} - macos-cache-key: ${{ steps.keys.outputs.macos-cache-key }} - rust-source-hash: ${{ steps.keys.outputs.rust-source-hash }} - macos-cache-hit: ${{ steps.macos-cache.outputs.cache-hit }} - rust-cache-hit: ${{ steps.rust-cache.outputs.cache-hit }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Bun @@ -70,70 +68,6 @@ jobs: env: GH_TOKEN: ${{ github.token }} - # Comprehensive content hashes for the native binaries. These cover the - # native source, the build/bundle scripts, AND the workflows that drive - # them — anything that can change the produced binary. The leading `v1` - # salt forces a full rebuild when something hashFiles can't see changes - # (runner image / Xcode / Rust toolchain) — bump it to invalidate. - # - # Note: the macOS .app bundle embeds a fixed constant version (hardcoded in - # build-swift.ts, not the npm version), so the notarized bundle is - # byte-stable across releases — which is what makes cross-release reuse - # valid. Do not reintroduce a per-release version there. - - id: keys - run: | - MACOS_HASH=${{ hashFiles('packages/encryption-binary-swift/swift/**', 'packages/encryption-binary-swift/scripts/**', 'packages/encryption-binary-swift/package.json', '.github/workflows/build-native-macos.yaml', '.github/workflows/notarize-native-macos.yaml') }} - RUST_HASH=${{ hashFiles('packages/encryption-binary-rust/Cargo.lock', 'packages/encryption-binary-rust/Cargo.toml', 'packages/encryption-binary-rust/src/**', 'packages/encryption-binary-rust/scripts/**', '.github/workflows/build-native-rust.yaml') }} - echo "macos-cache-key=native-bin-macos-notarized-v1-$MACOS_HASH" >> "$GITHUB_OUTPUT" - echo "rust-source-hash=v1-$RUST_HASH" >> "$GITHUB_OUTPUT" - - # Probe caches (lookup-only, no download). cache-hit is 'true' only on an - # exact key match within this branch's scope (main + its base) — feature - # branch caches are not visible here, so a release can only reuse a binary - # produced by a prior run on main. - - name: Probe macOS notarized binary cache - id: macos-cache - if: steps.plan.outputs.mode == 'publish' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: ${{ steps.keys.outputs.macos-cache-key }} - lookup-only: true - - name: Probe Rust cache - linux-x64 - id: rust-cache-linux-x64 - if: steps.plan.outputs.mode == 'publish' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/linux-x64/ - key: native-bin-rust-linux-x64-${{ steps.keys.outputs.rust-source-hash }} - lookup-only: true - - name: Probe Rust cache - linux-arm64 - id: rust-cache-linux-arm64 - if: steps.plan.outputs.mode == 'publish' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/linux-arm64/ - key: native-bin-rust-linux-arm64-${{ steps.keys.outputs.rust-source-hash }} - lookup-only: true - - name: Probe Rust cache - win32-x64 - id: rust-cache-win32-x64 - if: steps.plan.outputs.mode == 'publish' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/win32-x64/ - key: native-bin-rust-win32-x64-${{ steps.keys.outputs.rust-source-hash }} - lookup-only: true - - name: Aggregate Rust cache status - id: rust-cache - if: steps.plan.outputs.mode == 'publish' - run: | - if [[ "${{ steps.rust-cache-linux-x64.outputs.cache-hit }}" == "true" \ - && "${{ steps.rust-cache-linux-arm64.outputs.cache-hit }}" == "true" \ - && "${{ steps.rust-cache-win32-x64.outputs.cache-hit }}" == "true" ]]; then - echo "cache-hit=true" >> "$GITHUB_OUTPUT" - else - echo "cache-hit=false" >> "$GITHUB_OUTPUT" - fi # Open/update the "version packages" PR. This is the non-publish path: when # bump files are pending on main, bumpy computes versions + changelogs and @@ -179,83 +113,55 @@ jobs: - name: Audit dependencies run: bun run audit:critical - # Build + sign the macOS native binary. Skipped on a cache hit — the previously - # notarized bundle is reused and re-verified (see verify-native-macos) instead. + # Native helpers being released are always built, signed, and (macOS) + # notarized from source in this run. Nothing a release ships comes from the + # Actions cache: the keys are predictable and any job in main's cache scope + # (schedule, workflow_run, a compromised dependency in a push run) can write + # them, so neither binary caches nor build caches (Cargo, Swift .build) are used. build-native-macos: needs: plan - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.macos-cache-hit != 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-macos-native == 'true' uses: ./.github/workflows/build-native-macos.yaml with: mode: release artifact-name: native-bin-macos-signed + sign: true + use-build-cache: false secrets: OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }} - # Notarize for production npm distribution and cache the notarized bundle so - # future releases with unchanged source can reuse it. + # Notarize for production npm distribution notarize-native-macos: needs: [plan, build-native-macos] - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.macos-cache-hit != 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-macos-native == 'true' uses: ./.github/workflows/notarize-native-macos.yaml with: source-artifact-name: native-bin-macos-signed artifact-name: native-bin-macos-npm - cache-key: ${{ needs.plan.outputs.macos-cache-key }} secrets: OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }} - # When the notarized bundle is reused from cache (build + notarize skipped), - # re-verify it on a macOS runner before we trust it in the publish step. This - # is the safety net for "never publish the wrong thing": a corrupted or - # un-notarized cached bundle fails here and blocks the release. - verify-native-macos: - needs: plan - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.macos-cache-hit == 'true' - runs-on: macos-latest - steps: - - name: Restore notarized macOS binary from cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: ${{ needs.plan.outputs.macos-cache-key }} - fail-on-cache-miss: true - - name: Verify signature, notarization, and function - run: | - APP="packages/varlock/native-bins/darwin/VarlockEnclave.app" - BIN="$APP/Contents/MacOS/varlock-local-encrypt" - echo "=== codesign verify ===" - codesign --verify --deep --strict --verbose=2 "$APP" - codesign -dvv "$APP" 2>&1 || true - echo "=== notarization staple ===" - xcrun stapler validate "$APP" - echo "=== gatekeeper assessment (non-fatal) ===" - spctl -a -t exec -vv "$APP" || true - echo "=== functional smoke ===" - chmod +x "$BIN" - "$BIN" status | python3 -c "import sys,json; d=json.load(sys.stdin); assert d['ok'], 'status not ok'" - echo "Cached notarized binary verified" - - # Build Rust native binaries for Linux and Windows. Skipped on a cache hit. + # Build + sign Rust native binaries for Linux and Windows build-native-rust: needs: plan permissions: contents: read id-token: write # Azure OIDC signing in the reusable workflow - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit != 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-rust-native == 'true' uses: ./.github/workflows/build-native-rust.yaml with: artifact-name: native-bin-rust - source-hash: ${{ needs.plan.outputs.rust-source-hash }} sign: true + use-build-cache: false secrets: OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }} - # Publish path: native binaries are ready (built this run or restored from - # cache), so build the npm packages and publish. Gated on `mode == 'publish'`; + # Publish path: native binaries are ready (built this run, or skipped when the + # publish releases no helpers), so build the npm packages and publish. Gated on `mode == 'publish'`; # the version-PR path is handled by the `version-pr` job above. release: name: Release - needs: [plan, audit, notarize-native-macos, verify-native-macos, build-native-rust] + needs: [plan, audit, notarize-native-macos, build-native-rust] if: always() && !failure() && !cancelled() && needs.plan.outputs.mode == 'publish' runs-on: ubuntu-latest # Publishing runs in the `publish` environment so the OIDC tokens minted in @@ -292,71 +198,44 @@ jobs: - name: Update npm run: npm install -g npm@latest - # macOS native binary: download the freshly-notarized artifact (cache miss), - # or restore the previously-notarized bundle from cache (cache hit, already - # re-verified by verify-native-macos). + # Native helpers: binaries built, signed, and notarized earlier in this run + # for helpers this publish releases; the published version pinned by varlock + # for the rest (needed for SHA256SUMS and the CLI binaries below). - name: Download macOS native binary (built this run) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.macos-cache-hit != 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-macos-native == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-macos-npm path: packages/varlock/native-bins/darwin/VarlockEnclave.app - - name: Restore macOS native binary (from cache) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.macos-cache-hit == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: ${{ needs.plan.outputs.macos-cache-key }} - fail-on-cache-miss: true - name: Restore native binary execute permission - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-macos-native == 'true' run: chmod +x packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt - # Rust native binaries: download artifacts (cache miss) or restore from cache (hit) - name: Download Rust native binaries (built this run) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit != 'true' - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: native-bin-rust-linux-x64 - path: packages/varlock/native-bins/linux-x64 - - name: Download Rust native binaries - linux-arm64 (built this run) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit != 'true' - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: native-bin-rust-linux-arm64 - path: packages/varlock/native-bins/linux-arm64 - - name: Download Rust native binaries - win32-x64 (built this run) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit != 'true' + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-rust-native == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: native-bin-rust-win32-x64 - path: packages/varlock/native-bins/win32-x64 - - name: Restore Rust native binaries (from cache) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/linux-x64/ - key: native-bin-rust-linux-x64-${{ needs.plan.outputs.rust-source-hash }} - fail-on-cache-miss: true - - name: Restore Rust native binaries - linux-arm64 (from cache) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/linux-arm64/ - key: native-bin-rust-linux-arm64-${{ needs.plan.outputs.rust-source-hash }} - fail-on-cache-miss: true - - name: Restore Rust native binaries - win32-x64 (from cache) - if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' && needs.plan.outputs.rust-cache-hit == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: packages/varlock/native-bins/win32-x64/ - key: native-bin-rust-win32-x64-${{ needs.plan.outputs.rust-source-hash }} - fail-on-cache-miss: true - - name: Restore Rust binary execute permissions + pattern: native-bin-rust-* + path: packages/varlock/native-bins + - name: Stage Rust native binaries + if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-rust-native == 'true' + working-directory: packages/varlock/native-bins + run: | + for SUBDIR in linux-x64 linux-arm64 win32-x64; do + rm -rf "$SUBDIR" && mv "native-bin-rust-$SUBDIR" "$SUBDIR" + done + chmod +x linux-x64/varlock-local-encrypt linux-arm64/varlock-local-encrypt + + - name: Fetch published native helpers (not released this run) if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' + env: + MACOS: ${{ needs.plan.outputs.includes-macos-native }} + RUST: ${{ needs.plan.outputs.includes-rust-native }} run: | - chmod +x packages/varlock/native-bins/linux-x64/varlock-local-encrypt - chmod +x packages/varlock/native-bins/linux-arm64/varlock-local-encrypt + PLATFORMS="" + if [ "$MACOS" != "true" ]; then PLATFORMS="darwin"; fi + if [ "$RUST" != "true" ]; then PLATFORMS="$PLATFORMS,linux-x64,linux-arm64,win32-x64"; fi + if [ -n "${PLATFORMS#,}" ]; then bun run scripts/fetch-native-helpers.ts --platforms "${PLATFORMS#,}"; fi # Fail loudly if any native binary is missing before we build the npm package, # rather than silently publishing an incomplete package. @@ -437,8 +316,8 @@ jobs: # VS Code Marketplace auth comes from the Azure OIDC login step above; # Open VSX (OVSX_PAT) is loaded into the job env by the varlock step above. - # Publish the in-run signed/notarized native binaries so the binary - # distribution jobs below reuse them instead of rebuilding + re-signing. + # Publish the staged native binaries (built this run or fetched from npm) so + # the binary distribution jobs below reuse them instead of rebuilding + re-signing. - name: Upload staged native binaries if: needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -453,8 +332,8 @@ jobs: # --- varlock CLI binary distribution --------------------------------------- # The macOS CLI archives are built, Developer ID signed, and notarized on a # macOS runner; release-binaries builds every other target on linux and merges - # them in. Both reuse the signed/notarized native binaries from earlier in THIS - # run. Same skip-propagation guard as release-binaries below. + # them in. Both reuse the native binaries staged by the release job. Same + # skip-propagation guard as release-binaries below. build-cli-macos: needs: [plan, release] if: always() && !failure() && !cancelled() && needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' @@ -470,11 +349,10 @@ jobs: release-binaries: name: Release varlock CLI binaries needs: [plan, release, build-cli-macos] - # `release` always has a by-design skipped dependency (the native-binary cache - # logic skips either verify-native-macos on a miss, or build/notarize on a hit), - # and GitHub propagates that skip through `release`'s always() into the implicit - # success() of any downstream job — which would skip this job on every release. - # Mirror the `release` job's guard to break that propagation. + # `release` can have skipped dependencies (the native-binary jobs, on a publish that + # releases no helpers), and GitHub propagates a skip through `release`'s always() + # into the implicit success() of downstream jobs. Mirror the `release` job's + # guard so this job never depends on that propagation. if: always() && !failure() && !cancelled() && needs.plan.outputs.mode == 'publish' && needs.plan.outputs.includes-varlock == 'true' runs-on: ubuntu-latest permissions: @@ -506,7 +384,7 @@ jobs: echo "version=$V" >> "$GITHUB_OUTPUT" echo "tag=varlock@$V" >> "$GITHUB_OUTPUT" - # Reuse the native binaries built + signed + notarized earlier this run + # Reuse the native binaries staged by the release job - name: Download staged native binaries uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f0d6ac27..67f241791 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,11 +15,20 @@ jobs: build-and-test: runs-on: depot-ubuntu-24.04-8 outputs: - includes-varlock: ${{ steps.check-release.outputs.includes-varlock }} release-packages: ${{ steps.check-release.outputs.packages }} - swift-changed: ${{ steps.check-swift.outputs.changed }} - rust-changed: ${{ steps.check-rust.outputs.changed }} - rust-source-hash: ${{ steps.rust-hash.outputs.hash }} + # Which unsigned preview natives this run needs. On PRs: the preview includes + # that side's @varlock/native-helper-* packages, which only happens when they + # have a pending release (see check-release-packages.ts). On main: a helper + # release is pending, so build them once to warm the main-scoped cache that + # PR previews restore. + needs-macos-native: ${{ steps.check-release.outputs.needs-macos-native || steps.native.outputs.macos-pending }} + needs-rust-native: ${{ steps.check-release.outputs.needs-rust-native || steps.native.outputs.rust-pending }} + # PR changes that side's sources or build workflow: test-build it even when + # the preview uses the published helpers + macos-native-touched: ${{ steps.native.outputs.macos-touched }} + rust-native-touched: ${{ steps.native.outputs.rust-touched }} + macos-native-key: ${{ steps.native.outputs.macos-key }} + rust-native-key: ${{ steps.native.outputs.rust-key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -33,6 +42,11 @@ jobs: node-version: "24.x" - name: Install js deps (w/ bun) run: bun install + # Also fails when this branch changes native sources without a helper bump + # file (see scripts/native-sources.ts) + - name: Native helper sources + id: native + run: bun run scripts/native-sources.ts # Blocks only when the PR changes deps; pre-existing advisories are warnings # (the nightly audit handles those). The version PR always blocks, so a # release can't be merged with a known-critical advisory. @@ -75,136 +89,109 @@ jobs: id: check-release run: bun run scripts/check-release-packages.ts - # Check if native binary source changed (used to gate native builds) - - name: Check for Swift source changes - id: check-swift - if: steps.check-release.outputs.includes-varlock == 'true' - run: | - if git diff --name-only origin/main...HEAD | grep -qE '^(packages/encryption-binary-swift/|\.github/workflows/build-native-macos\.yaml)'; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - name: Check for Rust source changes - id: check-rust - if: steps.check-release.outputs.includes-varlock == 'true' - run: | - if git diff --name-only origin/main...HEAD | grep -qE '^(packages/encryption-binary-rust/|\.github/workflows/build-native-rust\.yaml)'; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - # Compute Rust source hash once on Ubuntu — used for cache keys everywhere - # (hashFiles can differ across OSes, so we normalize it here) - - name: Compute Rust source hash - id: rust-hash - if: steps.check-release.outputs.includes-varlock == 'true' - run: | - HASH=${{ hashFiles('packages/encryption-binary-rust/Cargo.lock', 'packages/encryption-binary-rust/src/**') }} - echo "hash=$HASH" >> $GITHUB_OUTPUT - - # Native binary caches are saved by the native build jobs, which run on GitHub - # runners, so they live in GitHub's cache. Depot runners redirect actions/cache - # to Depot Cache, where these entries never exist, so lookups and restores of - # native binaries must run on GitHub runners. + # Unsigned preview native binaries. Releases never use these (release.yaml always + # builds natives from source), so they need no signing secrets: macOS is ad-hoc + # signed, Rust is unsigned. They are cached by native source hash, so an identical + # build is reused across pushes and, once saved from main, across PRs. + # + # Depot runners redirect actions/cache to Depot Cache, where these entries never + # exist, so lookups and restores of native binaries must run on GitHub runners. native-cache-check: needs: build-and-test - if: needs.build-and-test.outputs.includes-varlock == 'true' + if: >- + needs.build-and-test.outputs.needs-macos-native == 'true' + || needs.build-and-test.outputs.needs-rust-native == 'true' + || needs.build-and-test.outputs.macos-native-touched == 'true' + || needs.build-and-test.outputs.rust-native-touched == 'true' runs-on: ubuntu-latest permissions: contents: read outputs: - swift-cache-hit: ${{ steps.swift-cache-check.outputs.cache-hit }} - rust-cache-hit: ${{ steps.rust-cache-final.outputs.cache-hit }} + macos-cache-hit: ${{ steps.macos-cache.outputs.cache-hit }} + rust-cache-hit: ${{ steps.rust-cache.outputs.cache-hit }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - sparse-checkout: packages/encryption-binary-swift # Check if native binary caches exist (lookup-only, no download) - - name: Check Swift binary cache - id: swift-cache-check - if: needs.build-and-test.outputs.swift-changed != 'true' + - name: Check macOS binary cache + id: macos-cache + if: needs.build-and-test.outputs.needs-macos-native == 'true' || needs.build-and-test.outputs.macos-native-touched == 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: native-bin-macos-signed-${{ hashFiles('packages/encryption-binary-swift/swift/Package.swift', 'packages/encryption-binary-swift/swift/Sources/**') }} + key: ${{ needs.build-and-test.outputs.macos-native-key }} lookup-only: true - name: Check Rust cache - linux-x64 - id: rust-cache-check-linux-x64 - if: needs.build-and-test.outputs.rust-changed != 'true' + id: rust-cache-linux-x64 + if: needs.build-and-test.outputs.needs-rust-native == 'true' || needs.build-and-test.outputs.rust-native-touched == 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/varlock/native-bins/linux-x64/ - key: native-bin-rust-linux-x64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-linux-x64 lookup-only: true - name: Check Rust cache - linux-arm64 - id: rust-cache-check-linux-arm64 - if: needs.build-and-test.outputs.rust-changed != 'true' + id: rust-cache-linux-arm64 + if: needs.build-and-test.outputs.needs-rust-native == 'true' || needs.build-and-test.outputs.rust-native-touched == 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/varlock/native-bins/linux-arm64/ - key: native-bin-rust-linux-arm64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-linux-arm64 lookup-only: true - name: Check Rust cache - win32-x64 - id: rust-cache-check-win32-x64 - if: needs.build-and-test.outputs.rust-changed != 'true' + id: rust-cache-win32-x64 + if: needs.build-and-test.outputs.needs-rust-native == 'true' || needs.build-and-test.outputs.rust-native-touched == 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/varlock/native-bins/win32-x64/ - key: native-bin-rust-win32-x64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-win32-x64 lookup-only: true - name: Determine Rust cache status - id: rust-cache-final - if: needs.build-and-test.outputs.rust-changed != 'true' + id: rust-cache + if: needs.build-and-test.outputs.needs-rust-native == 'true' || needs.build-and-test.outputs.rust-native-touched == 'true' run: | - if [[ "${{ steps.rust-cache-check-linux-x64.outputs.cache-hit }}" == "true" \ - && "${{ steps.rust-cache-check-linux-arm64.outputs.cache-hit }}" == "true" \ - && "${{ steps.rust-cache-check-win32-x64.outputs.cache-hit }}" == "true" ]]; then - echo "cache-hit=true" >> $GITHUB_OUTPUT + if [[ "${{ steps.rust-cache-linux-x64.outputs.cache-hit }}" == "true" \ + && "${{ steps.rust-cache-linux-arm64.outputs.cache-hit }}" == "true" \ + && "${{ steps.rust-cache-win32-x64.outputs.cache-hit }}" == "true" ]]; then + echo "cache-hit=true" >> "$GITHUB_OUTPUT" else - echo "cache-hit=false" >> $GITHUB_OUTPUT - echo "::warning::Some Rust binary caches missing — will trigger rebuild" + echo "cache-hit=false" >> "$GITHUB_OUTPUT" fi - # Build + sign the macOS native binary if varlock is being released AND (source changed or cache missing) + # Build the ad-hoc signed macOS preview binary on a cache miss, when the preview + # needs it or the PR changes its inputs. No secrets. build-native-macos: needs: [build-and-test, native-cache-check] if: >- - needs.build-and-test.outputs.includes-varlock == 'true' - && (needs.build-and-test.outputs.swift-changed == 'true' || needs.native-cache-check.outputs.swift-cache-hit != 'true') + (needs.build-and-test.outputs.needs-macos-native == 'true' || needs.build-and-test.outputs.macos-native-touched == 'true') + && needs.native-cache-check.outputs.macos-cache-hit != 'true' uses: ./.github/workflows/build-native-macos.yaml with: + mode: preview artifact-name: native-bin-macos-ci - secrets: - OP_CI_TOKEN: ${{ secrets.OP_CI_TOKEN }} - DEPOT_TURBO_TOKEN: ${{ secrets.DEPOT_TURBO_TOKEN }} + cache-key: ${{ needs.build-and-test.outputs.macos-native-key }} - # Build Rust native binaries if varlock is being released AND (source changed or cache missing) + # Build the unsigned Rust preview binaries, same conditions as macOS. No secrets. build-native-rust: needs: [build-and-test, native-cache-check] - # CI builds never sign, so no id-token is granted here (signing is gated on - # the `sign` input, which only the release workflows pass). permissions: contents: read if: >- - needs.build-and-test.outputs.includes-varlock == 'true' - && (needs.build-and-test.outputs.rust-changed == 'true' || needs.native-cache-check.outputs.rust-cache-hit != 'true') + (needs.build-and-test.outputs.needs-rust-native == 'true' || needs.build-and-test.outputs.rust-native-touched == 'true') + && needs.native-cache-check.outputs.rust-cache-hit != 'true' uses: ./.github/workflows/build-native-rust.yaml with: artifact-name: native-bin-rust-ci - source-hash: ${{ needs.build-and-test.outputs.rust-source-hash }} + cache-key: ${{ needs.build-and-test.outputs.rust-native-key }} # Publish preview packages via pkg-pr-new release-preview-packages: - needs: [build-and-test, build-native-macos, build-native-rust] - # Run even if native builds were skipped (source unchanged), but not if anything failed or was cancelled. + needs: [build-and-test, native-cache-check, build-native-macos, build-native-rust] + # Run even if native builds were skipped (no helper release pending, or cached), but not if anything failed or was cancelled. # `!cancelled()` only covers whole-workflow cancellation, so build-and-test must explicitly be a success # (a cancelled job leaves its outputs empty). Skip on main, and when there is nothing to release. if: >- always() && !failure() && !cancelled() && github.ref_name != 'main' && needs.build-and-test.result == 'success' + && needs.native-cache-check.result != 'cancelled' && needs.build-native-macos.result != 'cancelled' && needs.build-native-rust.result != 'cancelled' && needs.build-and-test.outputs.release-packages != '' @@ -228,23 +215,22 @@ jobs: with: token: ${{ secrets.DEPOT_TURBO_TOKEN }} - # Get signed macOS .app if varlock is being released - # If the macOS build ran this run, download the artifact directly - # Otherwise, restore from cross-run cache + # Preview natives, only for helper packages in this preview. Download the + # artifact if the build ran this run, otherwise restore the cached build. - name: Download macOS native binary (from this run) - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-macos.result == 'success' + if: needs.build-and-test.outputs.needs-macos-native == 'true' && needs.build-native-macos.result == 'success' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: native-bin-macos-ci path: packages/varlock/native-bins/darwin/VarlockEnclave.app - - name: Restore cached macOS native binary (from prior run) - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-macos.result != 'success' + - name: Restore cached macOS native binary + if: needs.build-and-test.outputs.needs-macos-native == 'true' && needs.build-native-macos.result != 'success' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: packages/varlock/native-bins/darwin/VarlockEnclave.app - key: native-bin-macos-signed-${{ hashFiles('packages/encryption-binary-swift/swift/Package.swift', 'packages/encryption-binary-swift/swift/Sources/**') }} + key: ${{ needs.build-and-test.outputs.macos-native-key }} - name: Verify and fix native binary permissions - if: needs.build-and-test.outputs.includes-varlock == 'true' + if: needs.build-and-test.outputs.needs-macos-native == 'true' run: | BINARY=packages/varlock/native-bins/darwin/VarlockEnclave.app/Contents/MacOS/varlock-local-encrypt if [ ! -f "$BINARY" ]; then @@ -253,10 +239,9 @@ jobs: fi chmod +x "$BINARY" - # Get Rust native binaries if varlock is being released - # If the Rust build ran this run, download the artifacts; otherwise restore from cache + # Rust preview natives, same as macOS above - name: Download Rust binaries (from this run) - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-rust.result == 'success' + if: needs.build-and-test.outputs.needs-rust-native == 'true' && needs.build-native-rust.result == 'success' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: native-bin-rust-ci-* @@ -264,7 +249,7 @@ jobs: merge-multiple: false # Flatten: download-artifact creates subdirs per artifact name, but we need linux-x64/ etc. - name: Flatten Rust artifact directories (from this run) - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-rust.result == 'success' + if: needs.build-and-test.outputs.needs-rust-native == 'true' && needs.build-native-rust.result == 'success' run: | cd packages/varlock/native-bins for dir in native-bin-rust-ci-*/; do @@ -273,23 +258,23 @@ jobs: done - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-rust.result != 'success' + if: needs.build-and-test.outputs.needs-rust-native == 'true' && needs.build-native-rust.result != 'success' with: path: packages/varlock/native-bins/linux-x64/ - key: native-bin-rust-linux-x64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-linux-x64 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-rust.result != 'success' + if: needs.build-and-test.outputs.needs-rust-native == 'true' && needs.build-native-rust.result != 'success' with: path: packages/varlock/native-bins/linux-arm64/ - key: native-bin-rust-linux-arm64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-linux-arm64 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: needs.build-and-test.outputs.includes-varlock == 'true' && needs.build-native-rust.result != 'success' + if: needs.build-and-test.outputs.needs-rust-native == 'true' && needs.build-native-rust.result != 'success' with: path: packages/varlock/native-bins/win32-x64/ - key: native-bin-rust-win32-x64-${{ needs.build-and-test.outputs.rust-source-hash }} + key: ${{ needs.build-and-test.outputs.rust-native-key }}-win32-x64 - name: Verify and fix Rust binary permissions - if: needs.build-and-test.outputs.includes-varlock == 'true' + if: needs.build-and-test.outputs.needs-rust-native == 'true' run: | MISSING=() for SUBDIR in linux-x64 linux-arm64 win32-x64; do @@ -306,7 +291,6 @@ jobs: done if [ ${#MISSING[@]} -gt 0 ]; then echo "::error::Rust native binaries not found: ${MISSING[*]}" - echo "Cache may not be seeded yet — ensure release.yaml has run on main" exit 1 fi diff --git a/AGENTS.md b/AGENTS.md index 8871277a2..865202838 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ This is a monorepo managed with bun workspaces and Turborepo: - `packages/varlock-website` — docs site (Astro); docs content lives in `src/content/docs/` - `packages/vscode-plugin` — VSCode extension for @env-spec language support - `packages/integrations/*` — framework integrations (nextjs, vite, astro, ...) -- `packages/native-helpers/*` — per-platform npm publishing shells for the native helper binaries (published as `@varlock/native-helper-*`, versioned in lockstep with `varlock`); the binaries themselves are built from `packages/encryption-binary-swift` (darwin) and `packages/encryption-binary-rust` (linux/win32) +- `packages/native-helpers/*` — per-platform npm publishing shells for the native helper binaries (published as `@varlock/native-helper-*`, versioned independently and only released when their binaries change; changing their sources needs a helper bump file, see `packages/native-helpers/README.md`); the binaries themselves are built from `packages/encryption-binary-swift` (darwin) and `packages/encryption-binary-rust` (linux/win32) - `packages/utils`, `packages/plugins` — shared internals - `packages/varlock-docs-mcp` — docs MCP server for external varlock users; do **not** use it to look things up while working on this repo — read the docs source directly diff --git a/packages/encryption-binary-swift/scripts/build-swift.ts b/packages/encryption-binary-swift/scripts/build-swift.ts index 15804fe9d..4618e3f50 100644 --- a/packages/encryption-binary-swift/scripts/build-swift.ts +++ b/packages/encryption-binary-swift/scripts/build-swift.ts @@ -15,18 +15,25 @@ * bun run scripts/build-swift.ts --universal # universal binary (CI) * bun run scripts/build-swift.ts --mode release # production bundle metadata * bun run scripts/build-swift.ts --sign "Developer ID Application: ..." + * bun run scripts/build-swift.ts --adhoc # ad-hoc sign, never load signing env + * + * Without --sign or --adhoc, the signing identity comes from APPLE_SIGNING_IDENTITY + * (loaded from .env.schema via varlock), falling back to ad-hoc. */ import { execSync } from 'node:child_process'; import path from 'node:path'; import fs from 'node:fs'; -import 'varlock/auto-load'; - // ── CLI args ──────────────────────────────────────────────────── const args = process.argv.slice(2); +// --adhoc skips loading .env.schema, whose signing values come from 1Password, so +// CI preview builds need no secrets at all +const adhoc = args.includes('--adhoc'); +if (!adhoc) await import('varlock/auto-load'); + function getArg(flag: string): string | undefined { const idx = args.indexOf(flag); return idx >= 0 ? args[idx + 1] : undefined; @@ -39,8 +46,8 @@ const mode = (getArg('--mode') ?? 'dev') as 'dev' | 'preview' | 'release'; // version. Nothing reads CFBundleVersion at runtime — not the Swift binary, the // Rust binary, the varlock CLI, or notarization — so coupling it to a // per-release version served no purpose, and it actively broke binary reuse: -// CI caches and reuses the signed/notarized .app keyed by source hash, which -// only holds if the bundle is byte-stable across releases. Keep this constant. +// CI caches and reuses built .app bundles keyed by source hash, which only holds +// if the bundle is byte-stable across releases. Keep this constant. const version = '1.0.0'; // ── Paths ─────────────────────────────────────────────────────── @@ -186,7 +193,7 @@ console.log(`Built app bundle: ${appDir}`); const entitlementsPath = path.resolve(import.meta.dir, '..', 'VarlockEnclave.entitlements'); // Resolve signing identity: explicit flag > APPLE_SIGNING_IDENTITY env var > ad-hoc -const resolvedIdentity = signingIdentity ?? process.env.APPLE_SIGNING_IDENTITY; +const resolvedIdentity = adhoc ? undefined : (signingIdentity ?? process.env.APPLE_SIGNING_IDENTITY); if (resolvedIdentity) { run(`codesign --force --deep --options runtime --entitlements "${entitlementsPath}" --sign "${resolvedIdentity}" "${appDir}"`); diff --git a/packages/native-helpers/README.md b/packages/native-helpers/README.md index fab2f8b03..d902f3de1 100644 --- a/packages/native-helpers/README.md +++ b/packages/native-helpers/README.md @@ -12,5 +12,7 @@ CI builds, signs, and stages the binaries into `packages/varlock/native-bins//`; compare it against the matching `native-bins//...` line for your release version: +**Verify the download.** Each [GitHub release](https://github.com/dmno-dev/varlock/releases) includes `SHA256SUMS.txt` with hashes for all native helpers. In npm installs the helper lives under `node_modules/@varlock/native-helper-/`; compare it against the matching `native-bins//...` line for your varlock version. The helper packages are only re-released when the binary changes, so their version can be older than varlock's; the checksums are listed under the varlock release either way: ```powershell # Windows PowerShell diff --git a/scripts/check-release-packages.ts b/scripts/check-release-packages.ts index 1d8ab06b9..2b6a72544 100644 --- a/scripts/check-release-packages.ts +++ b/scripts/check-release-packages.ts @@ -8,120 +8,110 @@ * Usage: * bun run scripts/check-release-packages.ts * + * Native helper packages (@varlock/native-helper-*) are only in the list when + * they have a pending release (they are versioned independently of varlock, see + * native-sources.ts). Otherwise the preview's varlock depends on the published + * helpers at their pinned version. + * * Outputs (via GITHUB_OUTPUT if available): * packages=["path1","path2"] - * includes-varlock=true|false + * needs-macos-native=true|false (preview includes the darwin helper) + * needs-rust-native=true|false (preview includes a linux/win32 helper) */ -import { execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { listWorkspaces } from './list-workspaces'; +import { NATIVE_HELPER_PACKAGES, readBumpyStatus, type NativeSide } from './native-sources'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const MONOREPO_ROOT = path.resolve(__dirname, '..'); -let bumpyStatusRaw: string; -try { - bumpyStatusRaw = execSync('bunx @varlock/bumpy status --json 2>/dev/null', { cwd: MONOREPO_ROOT }).toString(); -} catch (execErr: any) { - // bumpy may exit non-zero with warnings but still output valid JSON - bumpyStatusRaw = execErr.stdout?.toString() ?? ''; -} - let releasePackagePaths: Array = []; -if (bumpyStatusRaw) { - // stdout may contain warning lines before the JSON — extract just the JSON - const jsonMatch = bumpyStatusRaw.match(/\{[\s\S]*\}/); - if (jsonMatch) { - const bumpyStatus = JSON.parse(jsonMatch[0]); - const isPR = !!process.env.GITHUB_HEAD_REF || !!process.env.GITHUB_BASE_REF; - - const bumpyReleases = bumpyStatus.releases - .filter((r: any) => r.publishTargets?.some((t: any) => t.type === 'npm')); - - if (isPR) { - // On PRs, only include packages bumped in the current branch - // (ignore pending bumps already on main that aren't part of this PR) - console.log('PR detected — filtering to packages bumped in this branch'); - const branchReleases = bumpyReleases.filter((r: any) => r.inCurrentBranch); - const branchReleasePaths = branchReleases - .map((r: any) => path.resolve(MONOREPO_ROOT, r.dir)); - - const workspaces = await listWorkspaces(MONOREPO_ROOT); - const workspacesByName = new Map(workspaces.map((ws) => [ws.name, ws])); - const allReleasePaths = new Set(bumpyReleases.map((r: any) => path.resolve(MONOREPO_ROOT, r.dir))); - - function getWorkspaceDeps(pkgPath: string): Array { - const pkgJsonPath = path.join(pkgPath, 'package.json'); - try { - const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')); - const allDeps = { - ...pkgJson.dependencies, - ...pkgJson.devDependencies, - ...pkgJson.peerDependencies, - // pulls the @varlock/native-helper-* platform packages into varlock previews - ...pkgJson.optionalDependencies, - }; - return Object.entries(allDeps) - .filter(([, version]) => typeof version === 'string' && (version as string).startsWith('workspace:')) - .map(([name]) => workspacesByName.get(name)?.path) - .filter((p): p is string => !!p); - } catch { - return []; - } - } - - const neededPackages = new Set(branchReleasePaths); - - // Include cascade-bumped dependents — packages bumpy wants to release - // because a package bumped in this branch is in their dependency tree - // (e.g. varlock cascades when @env-spec/parser is bumped) - const depsByPath = new Map( - [...allReleasePaths].map((p) => [p, getWorkspaceDeps(p)]), - ); - let addedDependent = true; - while (addedDependent) { - addedDependent = false; - for (const [pkg, deps] of depsByPath) { - if (!neededPackages.has(pkg) && deps.some((dep) => neededPackages.has(dep))) { - neededPackages.add(pkg); - addedDependent = true; - } - } - } +const bumpyStatus = readBumpyStatus(); +const isPR = !!process.env.GITHUB_HEAD_REF || !!process.env.GITHUB_BASE_REF; + +const bumpyReleases = bumpyStatus.releases + .filter((r: any) => r.publishTargets?.some((t: any) => t.type === 'npm')); + +if (isPR) { + // On PRs, only include packages bumped in the current branch + // (ignore pending bumps already on main that aren't part of this PR) + console.log('PR detected — filtering to packages bumped in this branch'); + const branchReleases = bumpyReleases.filter((r: any) => r.inCurrentBranch); + const branchReleasePaths = branchReleases + .map((r: any) => path.resolve(MONOREPO_ROOT, r.dir)); + + const workspaces = await listWorkspaces(MONOREPO_ROOT); + const workspacesByName = new Map(workspaces.map((ws) => [ws.name, ws])); + const allReleasePaths = new Set(bumpyReleases.map((r: any) => path.resolve(MONOREPO_ROOT, r.dir))); + + function getWorkspaceDeps(pkgPath: string): Array { + const pkgJsonPath = path.join(pkgPath, 'package.json'); + try { + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')); + const allDeps = { + ...pkgJson.dependencies, + ...pkgJson.devDependencies, + ...pkgJson.peerDependencies, + // pulls the @varlock/native-helper-* platform packages into varlock previews + ...pkgJson.optionalDependencies, + }; + return Object.entries(allDeps) + .filter(([, version]) => typeof version === 'string' && (version as string).startsWith('workspace:')) + .map(([name]) => workspacesByName.get(name)?.path) + .filter((p): p is string => !!p); + } catch { + return []; + } + } - // Also include transitive workspace dependencies of included packages - // (so preview packages that reference each other have consistent versions) - const queue = [...neededPackages]; - while (queue.length > 0) { - const pkg = queue.pop()!; + const neededPackages = new Set(branchReleasePaths); + + // Include cascade-bumped dependents — packages bumpy wants to release + // because a package bumped in this branch is in their dependency tree + // (e.g. varlock cascades when @env-spec/parser is bumped) + const depsByPath = new Map( + [...allReleasePaths].map((p) => [p, getWorkspaceDeps(p)]), + ); + let addedDependent = true; + while (addedDependent) { + addedDependent = false; + for (const [pkg, deps] of depsByPath) { + if (!neededPackages.has(pkg) && deps.some((dep) => neededPackages.has(dep))) { neededPackages.add(pkg); - for (const dep of getWorkspaceDeps(pkg)) { - // Only include deps that bumpy also wants to release - if (!neededPackages.has(dep) && allReleasePaths.has(dep)) { - queue.push(dep); - } - } + addedDependent = true; } + } + } - releasePackagePaths = [...neededPackages]; - console.log('Packages bumped in this branch:', branchReleasePaths); - console.log('Packages to preview (+ cascades + dependencies):', releasePackagePaths); - } else { - releasePackagePaths = bumpyReleases - .map((r: any) => path.resolve(MONOREPO_ROOT, r.dir)); + // Also include transitive workspace dependencies of included packages + // (so preview packages that reference each other have consistent versions) + const queue = [...neededPackages]; + while (queue.length > 0) { + const pkg = queue.pop()!; + neededPackages.add(pkg); + for (const dep of getWorkspaceDeps(pkg)) { + // Only include deps that bumpy also wants to release + if (!neededPackages.has(dep) && allReleasePaths.has(dep)) { + queue.push(dep); + } } } + + releasePackagePaths = [...neededPackages]; + console.log('Packages bumped in this branch:', branchReleasePaths); + console.log('Packages to preview (+ cascades + dependencies):', releasePackagePaths); +} else { + releasePackagePaths = bumpyReleases + .map((r: any) => path.resolve(MONOREPO_ROOT, r.dir)); } // filter out vscode extension which is not released via npm releasePackagePaths = releasePackagePaths.filter((p: string) => !p.endsWith('packages/vscode-plugin')); -// The platform binary packages also need the native binaries staged in CI, -// so releasing any of them counts as "includes varlock" for workflow gating. // Match on package names (not directory paths) so relocating packages cannot // silently change release gating. function readPackageName(pkgPath: string): string | undefined { @@ -131,17 +121,20 @@ function readPackageName(pkgPath: string): string | undefined { return undefined; } } -const releaseNames = releasePackagePaths.map(readPackageName); -const includesVarlock = releaseNames.some( - (name) => name === 'varlock' || name?.startsWith('@varlock/native-helper-'), -); + +const needsNative: Record = { macos: false, rust: false }; +for (const side of Object.keys(NATIVE_HELPER_PACKAGES) as Array) { + const helperNames = NATIVE_HELPER_PACKAGES[side]; + needsNative[side] = releasePackagePaths.some((p) => helperNames.includes(readPackageName(p) ?? '')); +} console.log('Packages to release:', releasePackagePaths); -console.log('Includes varlock:', includesVarlock); +console.log('Needs native builds:', needsNative); // Write to GITHUB_OUTPUT if running in CI const githubOutput = process.env.GITHUB_OUTPUT; if (githubOutput) { fs.appendFileSync(githubOutput, `packages=${JSON.stringify(releasePackagePaths)}\n`); - fs.appendFileSync(githubOutput, `includes-varlock=${includesVarlock}\n`); + fs.appendFileSync(githubOutput, `needs-macos-native=${needsNative.macos}\n`); + fs.appendFileSync(githubOutput, `needs-rust-native=${needsNative.rust}\n`); } diff --git a/scripts/fetch-native-helpers.ts b/scripts/fetch-native-helpers.ts new file mode 100644 index 000000000..28cfa9d30 --- /dev/null +++ b/scripts/fetch-native-helpers.ts @@ -0,0 +1,86 @@ +/** + * Stages published @varlock/native-helper-* binaries into + * packages/varlock/native-bins//, for builds that bundle the helpers + * without rebuilding them (release CLI binaries when a helper is not being + * released, re-cuts, previews). + * + * Helper versions are exact and npm versions are immutable, so a pinned version + * is always the binary our release workflow published for it. + * + * Usage: + * bun run scripts/fetch-native-helpers.ts [--platforms darwin,linux-x64] [--varlock-version 1.2.3] [--wait] + * + * --platforms helper suffixes to fetch (default: all) + * --varlock-version use the helper versions pinned by that published varlock + * (default: the versions in packages/native-helpers//package.json) + * --wait poll the registry until each version is visible (right after a publish) + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { SUBDIR_CONTENTS } from '../packages/native-helpers/platforms'; + +const MONOREPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const DEST = path.join(MONOREPO_ROOT, 'packages/varlock/native-bins'); + +const args = process.argv.slice(2); +function getArg(name: string) { + const i = args.indexOf(name); + return i === -1 ? undefined : args[i + 1]; +} +const platforms = getArg('--platforms')?.split(',').filter(Boolean) ?? Object.keys(SUBDIR_CONTENTS); +const varlockVersion = getArg('--varlock-version'); +const wait = args.includes('--wait'); + +for (const p of platforms) { + if (!SUBDIR_CONTENTS[p]) throw new Error(`unknown helper platform: ${p}`); +} + +function npm(npmArgs: Array, cwd?: string) { + return execFileSync('npm', npmArgs, { cwd, encoding: 'utf-8' }).trim(); +} + +function waitForPublished(spec: string, attempts: number) { + for (let i = 1; i <= attempts; i++) { + if (spawnSync('npm', ['view', spec, 'version'], { encoding: 'utf-8' }).stdout.trim()) return; + if (!wait) break; + console.log(`waiting for ${spec} on npm (${i})...`); + execFileSync('sleep', ['10']); + } + throw new Error(`${spec} not found on npm`); +} + +let pinned: Record | undefined; +if (varlockVersion) { + waitForPublished(`varlock@${varlockVersion}`, 30); + pinned = JSON.parse(npm(['view', `varlock@${varlockVersion}`, 'optionalDependencies', '--json']) || '{}'); +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'varlock-native-helpers-')); +for (const suffix of platforms) { + const name = `@varlock/native-helper-${suffix}`; + const version = pinned + ? pinned[name] + : JSON.parse(fs.readFileSync(path.join(MONOREPO_ROOT, 'packages/native-helpers', suffix, 'package.json'), 'utf-8')).version; + if (!version) throw new Error(`varlock@${varlockVersion} does not pin ${name} (versions <= 1.16.x bundled the binaries in the varlock tarball)`); + const spec = `${name}@${version}`; + waitForPublished(spec, 30); + + const workDir = path.join(tmp, suffix); + fs.mkdirSync(workDir); + const [packed] = JSON.parse(npm(['pack', spec, '--json'], workDir)); + execFileSync('tar', ['-xzf', packed.filename], { cwd: workDir }); + + const entry = SUBDIR_CONTENTS[suffix]; + const destDir = path.join(DEST, suffix); + fs.rmSync(destDir, { recursive: true, force: true }); + fs.mkdirSync(destDir, { recursive: true }); + fs.cpSync(path.join(workDir, 'package', entry), path.join(destDir, entry), { recursive: true, verbatimSymlinks: true }); + // npm tarballs keep the mode bits, but be explicit about the executables + const exe = suffix === 'darwin' ? path.join(destDir, entry, 'Contents/MacOS/varlock-local-encrypt') : path.join(destDir, entry); + fs.chmodSync(exe, 0o755); + console.log(`staged ${spec} -> ${path.relative(MONOREPO_ROOT, destDir)}/${entry}`); +} +fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/scripts/native-sources.ts b/scripts/native-sources.ts new file mode 100644 index 000000000..28291a161 --- /dev/null +++ b/scripts/native-sources.ts @@ -0,0 +1,182 @@ +/** + * Native helper binary inputs, shared by PR preview gating, preview caching, and + * the native bump check. + * + * The @varlock/native-helper-* packages are versioned independently of varlock + * and only released when their binaries change. varlock pins them as exact + * versions, so a varlock version always maps to one set of released binaries, + * and a helper with no pending release is exactly what is on npm at its + * package.json version. + * + * - `pending`: the side's helpers have a pending release (a bump file on this + * branch or on main, or a checked-out version with no release tag yet, as on + * the version PR). Previews of such a branch need preview builds of that side; + * everything else uses the published helpers. + * - `touched`: whether this branch changes the sources or the build workflows + * relative to its merge base with origin/main. CI then test-builds that side, + * so a broken build is caught in its own PR instead of at release time. + * - `key`: content hash of the sources and build workflows (git blob ids, so + * identical on every OS), used as the cache key for unsigned preview builds. + * + * Bump check: when this branch changes a side's sources, a bump file on this + * branch must name one of that side's helpers (bump type `none` if the binary is + * unaffected), otherwise the script fails. bumpy cannot enforce this itself: it + * only maps changes inside a package's own directory, and the Swift / Rust + * sources live in separate private packages. + * + * Releases never use the preview builds: they always build natives from source. + * + * Usage: + * bun run scripts/native-sources.ts + * + * Outputs (via GITHUB_OUTPUT if available), for each of `macos` and `rust`: + * -pending=true|false -touched=true|false + * -key=native-preview--v1- + */ +import { execFileSync, execSync, spawnSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const MONOREPO_ROOT = path.resolve(path.dirname(__filename), '..'); + +// Everything that can change a helper binary or its npm package. Bump the `v1` +// in the key prefix to invalidate preview caches when something outside these +// paths changes the output (runner image, Xcode, Rust toolchain). +const SHARED_INPUTS = [ + 'packages/native-helpers/platforms.ts', + 'packages/native-helpers/prepack.ts', + 'packages/native-helpers/postpack.ts', +]; +const EXCLUDES = [':(exclude,glob)**/*.md']; + +export const NATIVE_SOURCES = { + macos: [ + 'packages/encryption-binary-swift', + 'packages/native-helpers/darwin', + ...SHARED_INPUTS, + ], + rust: [ + 'packages/encryption-binary-rust', + 'packages/native-helpers/linux-x64', + 'packages/native-helpers/linux-arm64', + 'packages/native-helpers/win32-x64', + ...SHARED_INPUTS, + ], +} as const; + +export type NativeSide = keyof typeof NATIVE_SOURCES; + +const BUILD_WORKFLOWS: Record> = { + macos: ['.github/workflows/build-native-macos.yaml'], + rust: ['.github/workflows/build-native-rust.yaml'], +}; + +function buildInputs(side: NativeSide) { + return [...NATIVE_SOURCES[side], ...BUILD_WORKFLOWS[side]]; +} + +/** npm package suffixes (packages/native-helpers/) of the helpers each side produces */ +export const NATIVE_HELPER_SUFFIXES: Record> = { + macos: ['darwin'], + rust: ['linux-x64', 'linux-arm64', 'win32-x64'], +}; + +/** npm package names of the helpers each side produces */ +export const NATIVE_HELPER_PACKAGES = Object.fromEntries( + Object.entries(NATIVE_HELPER_SUFFIXES).map(([side, suffixes]) => [side, suffixes.map((s) => `@varlock/native-helper-${s}`)]), +) as Record>; + +function git(args: Array) { + return execFileSync('git', args, { cwd: MONOREPO_ROOT, encoding: 'utf-8' }); +} + +export function nativeSourceKey(side: NativeSide) { + const files = git(['ls-files', '-s', '--', ...buildInputs(side), ...EXCLUDES]); + if (!files.trim()) throw new Error(`no tracked native ${side} inputs found`); + const hash = crypto.createHash('sha256').update(files).digest('hex'); + return `native-preview-${side}-v1-${hash}`; +} + +function refExists(ref: string) { + return spawnSync('git', ['rev-parse', '--verify', '--quiet', ref], { cwd: MONOREPO_ROOT }).status === 0; +} + +function differs(fromRef: string, paths: Array) { + const diff = spawnSync('git', ['diff', '--quiet', fromRef, 'HEAD', '--', ...paths, ...EXCLUDES], { cwd: MONOREPO_ROOT }); + if (diff.status !== 0 && diff.status !== 1) throw new Error(`git diff failed: ${diff.stderr}`); + return diff.status === 1; +} + +function mergeBase() { + if (!refExists('refs/remotes/origin/main')) return undefined; + return git(['merge-base', 'refs/remotes/origin/main', 'HEAD']).trim(); +} + +/** `bumpy status --json`, which may exit non-zero with warnings but still print valid JSON */ +export function readBumpyStatus(): { bumpFiles: Array, releases: Array } { + let raw: string; + try { + raw = execSync('bunx @varlock/bumpy status --json 2>/dev/null', { cwd: MONOREPO_ROOT }).toString(); + } catch (execErr: any) { + raw = execErr.stdout?.toString() ?? ''; + } + // stdout may contain warning lines before the JSON, extract just the JSON + const jsonMatch = raw.match(/\{[\s\S]*\}/); + return jsonMatch ? JSON.parse(jsonMatch[0]) : { bumpFiles: [], releases: [] }; +} + +function helperVersion(suffix: string, ref?: string) { + const relPath = `packages/native-helpers/${suffix}/package.json`; + const raw = ref ? git(['show', `${ref}:${relPath}`]) : fs.readFileSync(path.join(MONOREPO_ROOT, relPath), 'utf-8'); + return JSON.parse(raw).version as string; +} + +// run as a script (not when imported by check-release-packages.ts) +if (process.argv[1] && path.resolve(process.argv[1]) === __filename) { + const status = readBumpyStatus(); + const base = mergeBase(); + const lines: Array = []; + const missingBumps: Array = []; + for (const side of Object.keys(NATIVE_SOURCES) as Array) { + const helpers = NATIVE_HELPER_PACKAGES[side]; + // also pending when the checked-out version was never released: on the version + // PR (and main right after it merges) the bump files are already consumed, but + // that version is not on npm yet, so it cannot be fetched + const unreleased = NATIVE_HELPER_SUFFIXES[side].some( + (s) => !refExists(`refs/tags/@varlock/native-helper-${s}@${helperVersion(s)}`), + ); + const pending = unreleased || status.releases.some((r) => helpers.includes(r.name)); + const touched = base ? differs(base, buildInputs(side)) : false; + const key = nativeSourceKey(side); + console.log(`native ${side}: pending release=${pending} touched by branch=${touched} key=${key}`); + lines.push(`${side}-pending=${pending}`, `${side}-touched=${touched}`, `${side}-key=${key}`); + + if (base && differs(base, [...NATIVE_SOURCES[side]])) { + // only this branch's bump files count: one already on main (possibly `none`) + // covers the change that added it, not this one + const covered = status.bumpFiles.some( + (bf) => bf.inCurrentBranch && bf.releases.some((r: any) => helpers.includes(r.name)), + ); + // the version PR consumes the bump files and bumps the helper versions itself + const versioning = NATIVE_HELPER_SUFFIXES[side].some((s) => helperVersion(s) !== helperVersion(s, base)); + if (!covered && !versioning) missingBumps.push(side); + } + } + if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); + + if (missingBumps.length) { + for (const side of missingBumps) { + const pkg = NATIVE_HELPER_PACKAGES[side][0]; + console.error([ + `Native ${side} sources changed on this branch, but no bump file names ${NATIVE_HELPER_PACKAGES[side].join(' / ')}.`, + 'The helpers are only released when bumped, so without one this change never ships. Add a bump file:', + ` bunx @varlock/bumpy add --packages "${pkg}:patch" --message "" --name "${side}-native-"`, + `Use "${pkg}:none" instead if the change does not affect the shipped binary.`, + ].join('\n')); + } + process.exit(1); + } +}