test.yml

name: Python CI

on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

env:
  PROMPTFOO_DISABLE_TELEMETRY: "1"
  PROMPTFOO_DISABLE_UPDATE: "1"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  resolve-promptfoo:
    name: Resolve promptfoo version
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      version: ${{ steps.promptfoo.outputs.version }}
    steps:
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: "24"
          package-manager-cache: false

      - name: Resolve the published promptfoo version once for every test job
        id: promptfoo
        shell: bash
        run: |
          version=$(npm view promptfoo@latest version)
          if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            echo "Unexpected promptfoo version: $version" >&2
            exit 1
          fi
          echo "version=$version" >> "$GITHUB_OUTPUT"

  lint:
    name: Lint and Format
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.12.17"
          enable-cache: auto

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --locked --extra dev

      - name: Lint with Ruff
        run: uv run ruff check src/

      - name: Check formatting
        run: uv run ruff format --check src/

      - name: Check GitHub workflows
        run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12

      - name: Verify Alpine installation instructions
        run: |
          alpine_dir="$RUNNER_TEMP/promptfoo-alpine"
          mkdir -p "$alpine_dir"
          uv run python - <<'PY' > "$alpine_dir/Dockerfile"
          from promptfoo.environment import Environment
          from promptfoo.instructions import get_installation_instructions

          instructions = get_installation_instructions(
              Environment(os_type="linux", linux_distro="alpine", is_docker=True)
          )
          for line in instructions.splitlines():
              command = line.strip()
              if command.startswith(("FROM ", "RUN ", "ENV ")):
                  print(command)
          PY
          cat >> "$alpine_dir/Dockerfile" <<'EOF'
          RUN node --version | grep -E '^v24\.' \
              && npm --version && npx --version && python --version && pip --version \
              && python -c 'import sys; assert sys.prefix == "/opt/venv"'
          EOF
          docker build --progress=plain "$alpine_dir"
          docker run --rm alpine:3.20 sh -ec '
            apk update
            apk add --no-cache nodejs npm
            case "$(node --version)" in
              v20.*) ;;
              *) echo "Expected Alpine 3.20 to supply Node.js 20" >&2; exit 1 ;;
            esac
            if apk add --no-cache "nodejs~24" npm; then
              echo "Alpine 3.20 unexpectedly installed Node.js 24" >&2
              exit 1
            fi
          '

  type-check:
    name: Type Check (${{ matrix.type-checker }})
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      matrix:
        type-checker: [mypy, pyright]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.12.17"
          enable-cache: auto

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --locked --extra dev

      - name: Type check with mypy
        if: matrix.type-checker == 'mypy'
        run: uv run mypy src/promptfoo/

      - name: Type check with pyright
        if: matrix.type-checker == 'pyright'
        run: uv run pyright src/promptfoo/

  test:
    name: Test (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }})
    needs: resolve-promptfoo
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    env:
      PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }}
    strategy:
      fail-fast: false
      matrix:
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        # causing BlockingIOError [Errno 35] when spawning subprocess
        os: [ubuntu-latest, windows-latest]
        # Test only min and max supported Python versions for efficiency
        python-version: ["3.10", "3.14"]
        include:
          - python-version: "3.10"
            node-version: "22.22.0"
          - python-version: "3.14"
            node-version: "24"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: ${{ matrix.node-version }}
          package-manager-cache: false

      - name: Configure npm on Windows
        if: matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
          $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global"
          New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null
          "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          $globalPrefix | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

      - name: Install promptfoo globally
        shell: bash
        run: npm install --global "promptfoo@$PROMPTFOO_VERSION"
        env:
          NODE_OPTIONS: --max-old-space-size=4096

      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.12.17"
          enable-cache: auto

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package with dev dependencies
        run: uv sync --locked --extra dev

      - name: Verify the wrapper can find the global npm installation
        run: uv run python -c "from promptfoo.cli import _find_external_promptfoo; path = _find_external_promptfoo(); print(path); assert path is not None"

      - name: Run unit tests
        run: uv run pytest tests/ -v -m 'not smoke'

      - name: Run smoke tests
        run: uv run pytest tests/smoke/ -v

      - name: Test CLI can be invoked
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  test-npx-fallback:
    name: Test npx fallback (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }})
    needs: resolve-promptfoo
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    env:
      PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }}
    strategy:
      fail-fast: false
      matrix:
        # Test npx fallback (without global install)
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        os: [ubuntu-latest, windows-latest]
        # Use middle-version Python for this test
        python-version: ["3.12"]
        node-version: ["22.22.0", "24"]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: ${{ matrix.node-version }}
          package-manager-cache: false

      - name: Configure npm on Windows
        if: matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
          $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global"
          New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null
          "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

      # Intentionally skip installing promptfoo globally
      # This tests the npx fallback path

      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.12.17"
          enable-cache: auto

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package with dev dependencies
        run: uv sync --locked --extra dev

      - name: Verify the wrapper must fall back to npx
        run: uv run python -c "from promptfoo.cli import _find_external_promptfoo, check_npx_installed; path = _find_external_promptfoo(); print(path); assert path is None; assert check_npx_installed()"

      - name: Run unit tests
        run: uv run pytest tests/ -v -m 'not smoke'

      - name: Run smoke tests (with npx fallback)
        run: uv run pytest tests/smoke/ -v

      - name: Test CLI fallback to npx (no global install)
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  build:
    name: Build Package
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.12.17"
          enable-cache: auto

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Build package
        run: uv build

      - name: Check distribution metadata with Twine
        run: uvx --from twine --constraints .github/requirements-twine.txt twine check --strict dist/*

      - name: Upload artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: dist
          path: dist/
          if-no-files-found: error

  ci-success:
    name: CI Success
    needs: [resolve-promptfoo, lint, type-check, test, test-npx-fallback, build]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check if all jobs succeeded
        env:
          JOB_RESULTS: ${{ join(needs.*.result, ',') }}
        run: |
          echo "Job results: $JOB_RESULTS"
          [[ "$JOB_RESULTS" =~ ^success(,success)*$ ]]