fix!: require Node.js 22.22 and repair CI

.github/workflows/release-please.yml

@@ -31,7 +31,8 @@ jobs:
         id: release
         if: github.event_name == 'push'
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          # A dedicated token lets the resulting pull_request run start without manual workflow approval.
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN || github.token }}
 
   build:
     if: |
@@ -54,7 +55,15 @@ jobs:
         run: uv python pin 3.12
 
       - name: Install dependencies
-        run: uv sync --extra dev
+        env:
+          REPUBLISH_TAG: ${{ inputs.tag }}
+        run: |
+          if [[ -n "$REPUBLISH_TAG" ]]; then
+            # Historical tags predate keeping the package version in uv.lock in sync.
+            uv sync --extra dev
+          else
+            uv sync --locked --extra dev
+          fi
 
       - name: Run unit tests
         run: uv run pytest -m 'not smoke' -q
@@ -67,7 +76,7 @@ jobs:
           TAG: ${{ needs.release-please.outputs.tag_name }}
         run: |
           EXPECTED_VERSION="${TAG#promptfoo-v}"
-          if ls dist/*-${EXPECTED_VERSION}-*.whl 1> /dev/null 2>&1; then
+          if compgen -G "dist/*-${EXPECTED_VERSION}-*.whl" > /dev/null; then
             echo "✓ Package version ${EXPECTED_VERSION} matches release tag ${TAG}"
           else
             echo "ERROR: Package version mismatch!"
@@ -82,6 +91,7 @@ jobs:
         with:
           name: dist
           path: dist/
+          if-no-files-found: error
 
   publish-pypi:
     if: |

.github/workflows/test.yml

@@ -2,9 +2,6 @@ name: Python CI
 
 on:
   pull_request:
-    paths-ignore:
-      - "**.md"
-      - "LICENSE"
   push:
     branches:
       - main
@@ -13,11 +10,38 @@ on:
 permissions:
   contents: read
 
+env:
+  PROMPTFOO_DISABLE_TELEMETRY: "1"
+  PROMPTFOO_DISABLE_UPDATE: "1"
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  resolve-promptfoo:
+    name: Resolve promptfoo version
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      version: ${{ steps.promptfoo.outputs.version }}
+    steps:
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: "24"
+          package-manager-cache: false
+
+      - name: Resolve the published promptfoo version once for every test job
+        id: promptfoo
+        shell: bash
+        run: |
+          version=$(npm view promptfoo@latest version)
+          if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "Unexpected promptfoo version: $version" >&2
+            exit 1
+          fi
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
   lint:
     name: Lint and Format
     runs-on: ubuntu-latest
@@ -33,14 +57,52 @@ jobs:
         run: uv python pin 3.12
 
       - name: Install dependencies
-        run: uv sync --extra dev
+        run: uv sync --locked --extra dev
 
       - name: Lint with Ruff
         run: uv run ruff check src/
 
       - name: Check formatting
         run: uv run ruff format --check src/
 
+      - name: Check GitHub workflows
+        run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12
+
+      - name: Verify Alpine installation instructions
+        run: |
+          alpine_dir="$RUNNER_TEMP/promptfoo-alpine"
+          mkdir -p "$alpine_dir"
+          uv run python - <<'PY' > "$alpine_dir/Dockerfile"
+          from promptfoo.environment import Environment
+          from promptfoo.instructions import get_installation_instructions
+
+          instructions = get_installation_instructions(
+              Environment(os_type="linux", linux_distro="alpine", is_docker=True)
+          )
+          for line in instructions.splitlines():
+              command = line.strip()
+              if command.startswith(("FROM ", "RUN ", "ENV ")):
+                  print(command)
+          PY
+          cat >> "$alpine_dir/Dockerfile" <<'EOF'
+          RUN node --version | grep -E '^v24\.' \
+              && npm --version && npx --version && python --version && pip --version \
+              && python -c 'import sys; assert sys.prefix == "/opt/venv"'
+          EOF
+          docker build --progress=plain "$alpine_dir"
+          docker run --rm alpine:3.20 sh -ec '
+            apk update
+            apk add --no-cache nodejs npm
+            case "$(node --version)" in
+              v20.*) ;;
+              *) echo "Expected Alpine 3.20 to supply Node.js 20" >&2; exit 1 ;;
+            esac
+            if apk add --no-cache "nodejs~24" npm; then
+              echo "Alpine 3.20 unexpectedly installed Node.js 24" >&2
+              exit 1
+            fi
+          '
+
   type-check:
     name: Type Check (${{ matrix.type-checker }})
     runs-on: ubuntu-latest
@@ -59,7 +121,7 @@ jobs:
         run: uv python pin 3.12
 
       - name: Install dependencies
-        run: uv sync --extra dev
+        run: uv sync --locked --extra dev
 
       - name: Type check with mypy
         if: matrix.type-checker == 'mypy'
@@ -70,60 +132,47 @@ jobs:
         run: uv run pyright src/promptfoo/
 
   test:
-    name: Test (py${{ matrix.python-version }}, ${{ matrix.os }})
+    name: Test (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }})
+    needs: resolve-promptfoo
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+    env:
+      PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }}
     strategy:
+      fail-fast: false
       matrix:
         # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
         # causing BlockingIOError [Errno 35] when spawning subprocess
         os: [ubuntu-latest, windows-latest]
         # Test only min and max supported Python versions for efficiency
         python-version: ["3.10", "3.14"]
+        include:
+          - python-version: "3.10"
+            node-version: "22.22.0"
+          - python-version: "3.14"
+            node-version: "24"
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
         with:
-          node-version: "24"
+          node-version: ${{ matrix.node-version }}
+          package-manager-cache: false
 
       - name: Configure npm on Windows
         if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
-          # Configure cache location (applies immediately to this step)
           $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
-          New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
-          npm config set cache $cacheDir --location=user
-
-          # Configure prefix location (applies immediately to this step)
-          $globalPrefix = npm config get prefix
-          if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
-            $globalPrefix = Join-Path $env:APPDATA "npm"
-          }
-          $globalPrefix = $globalPrefix.Trim()
-          npm config set prefix $globalPrefix --location=user
-
-          # NOW clean and verify cache (cleans the correctly-configured cache)
-          npm cache clean --force
-          npm cache verify
-
-          # Export settings for future steps
+          $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global"
+          New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null
           "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
           "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-
-          # Add global bin directories to PATH
-          $binPaths = @($globalPrefix, (Join-Path $globalPrefix "bin")) | Where-Object { Test-Path $_ }
-          foreach ($binPath in $binPaths) {
-            $binPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          }
-
-          Write-Host "npm cache: $cacheDir"
-          Write-Host "npm prefix: $globalPrefix"
+          $globalPrefix | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
       - name: Install promptfoo globally
-        run: npm install -g promptfoo@latest
+        shell: bash
+        run: npm install --global "promptfoo@$PROMPTFOO_VERSION"
         env:
           NODE_OPTIONS: --max-old-space-size=4096
 
@@ -135,7 +184,10 @@ jobs:
         run: uv python pin ${{ matrix.python-version }}
 
       - name: Install package with dev dependencies
-        run: uv sync --extra dev
+        run: uv sync --locked --extra dev
+
+      - name: Verify the wrapper can find the global npm installation
+        run: uv run python -c "from promptfoo.cli import _find_external_promptfoo; path = _find_external_promptfoo(); print(path); assert path is not None"
 
       - name: Run unit tests
         run: uv run pytest tests/ -v -m 'not smoke'
@@ -150,51 +202,38 @@ jobs:
         run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"
 
   test-npx-fallback:
-    name: Test npx fallback (py${{ matrix.python-version }}, ${{ matrix.os }})
+    name: Test npx fallback (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }})
+    needs: resolve-promptfoo
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+    env:
+      PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }}
     strategy:
+      fail-fast: false
       matrix:
         # Test npx fallback (without global install)
         # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
         os: [ubuntu-latest, windows-latest]
         # Use middle-version Python for this test
         python-version: ["3.12"]
+        node-version: ["22.22.0", "24"]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
         with:
-          node-version: "24"
+          node-version: ${{ matrix.node-version }}
+          package-manager-cache: false
 
       - name: Configure npm on Windows
         if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
-          # Configure cache location (applies immediately to this step)
           $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
-          New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
-          npm config set cache $cacheDir --location=user
-
-          # Configure prefix location (applies immediately to this step)
-          $globalPrefix = npm config get prefix
-          if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
-            $globalPrefix = Join-Path $env:APPDATA "npm"
-          }
-          $globalPrefix = $globalPrefix.Trim()
-          npm config set prefix $globalPrefix --location=user
-
-          # NOW clean and verify cache (cleans the correctly-configured cache)
-          npm cache clean --force
-          npm cache verify
-
-          # Export settings for future steps
+          $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global"
+          New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null
           "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
           "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-
-          Write-Host "npm cache: $cacheDir"
-          Write-Host "npm prefix: $globalPrefix"
 
       # Intentionally skip installing promptfoo globally
       # This tests the npx fallback path
@@ -207,7 +246,10 @@ jobs:
         run: uv python pin ${{ matrix.python-version }}
 
       - name: Install package with dev dependencies
-        run: uv sync --extra dev
+        run: uv sync --locked --extra dev
+
+      - name: Verify the wrapper must fall back to npx
+        run: uv run python -c "from promptfoo.cli import _find_external_promptfoo, check_npx_installed; path = _find_external_promptfoo(); print(path); assert path is None; assert check_npx_installed()"
 
       - name: Run unit tests
         run: uv run pytest tests/ -v -m 'not smoke'
@@ -243,36 +285,17 @@ jobs:
         with:
           name: dist
           path: dist/
+          if-no-files-found: error
 
   ci-success:
     name: CI Success
-    needs: [lint, type-check, test, test-npx-fallback, build]
+    needs: [resolve-promptfoo, lint, type-check, test, test-npx-fallback, build]
     if: always()
     runs-on: ubuntu-latest
     steps:
       - name: Check if all jobs succeeded
+        env:
+          JOB_RESULTS: ${{ join(needs.*.result, ',') }}
         run: |
-          LINT_RESULT="${{ needs.lint.result }}"
-          TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
-          TEST_RESULT="${{ needs.test.result }}"
-          TEST_NPX_FALLBACK_RESULT="${{ needs.test-npx-fallback.result }}"
-          BUILD_RESULT="${{ needs.build.result }}"
-
-          echo "Job results:"
-          echo "  lint: $LINT_RESULT"
-          echo "  type-check: $TYPE_CHECK_RESULT"
-          echo "  test: $TEST_RESULT"
-          echo "  test-npx-fallback: $TEST_NPX_FALLBACK_RESULT"
-          echo "  build: $BUILD_RESULT"
-
-          if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
-                "$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
-                "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
-                "$TEST_NPX_FALLBACK_RESULT" == "failure" || "$TEST_NPX_FALLBACK_RESULT" == "cancelled" ||
-                "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
-            echo "Some CI checks failed!"
-            exit 1
-          else
-            echo "All CI checks passed!"
-            exit 0
-          fi
+          echo "Job results: $JOB_RESULTS"
+          [[ "$JOB_RESULTS" =~ ^success(,success)*$ ]]