test(ci): enforce the setup-node cache policy - abandoned #199
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| env: | |
| PROMPTFOO_DISABLE_TELEMETRY: "1" | |
| PROMPTFOO_DISABLE_UPDATE: "1" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| resolve-promptfoo: | |
| name: Resolve promptfoo version | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| version: ${{ steps.promptfoo.outputs.version }} | |
| steps: | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version: "24" | |
| package-manager-cache: false | |
| - name: Resolve the published promptfoo version once for every test job | |
| id: promptfoo | |
| shell: bash | |
| run: | | |
| version=$(npm view promptfoo@latest version) | |
| if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "Unexpected promptfoo version: $version" >&2 | |
| exit 1 | |
| fi | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| lint: | |
| name: Lint and Format | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: uv python pin 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --extra dev | |
| - name: Lint with Ruff | |
| run: uv run ruff check src/ | |
| - name: Check formatting | |
| run: uv run ruff format --check src/ | |
| - name: Check GitHub workflows | |
| run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 | |
| - name: Verify Alpine installation instructions | |
| run: | | |
| alpine_dir="$RUNNER_TEMP/promptfoo-alpine" | |
| mkdir -p "$alpine_dir" | |
| uv run python - <<'PY' > "$alpine_dir/Dockerfile" | |
| from promptfoo.environment import Environment | |
| from promptfoo.instructions import get_installation_instructions | |
| instructions = get_installation_instructions( | |
| Environment(os_type="linux", linux_distro="alpine", is_docker=True) | |
| ) | |
| for line in instructions.splitlines(): | |
| command = line.strip() | |
| if command.startswith(("FROM ", "RUN ", "ENV ")): | |
| print(command) | |
| PY | |
| cat >> "$alpine_dir/Dockerfile" <<'EOF' | |
| RUN node --version | grep -E '^v24\.' \ | |
| && npm --version && npx --version && python --version && pip --version \ | |
| && python -c 'import sys; assert sys.prefix == "/opt/venv"' | |
| EOF | |
| docker build --progress=plain "$alpine_dir" | |
| docker run --rm alpine:3.20 sh -ec ' | |
| apk update | |
| apk add --no-cache nodejs npm | |
| case "$(node --version)" in | |
| v20.*) ;; | |
| *) echo "Expected Alpine 3.20 to supply Node.js 20" >&2; exit 1 ;; | |
| esac | |
| if apk add --no-cache "nodejs~24" npm; then | |
| echo "Alpine 3.20 unexpectedly installed Node.js 24" >&2 | |
| exit 1 | |
| fi | |
| ' | |
| type-check: | |
| name: Type Check (${{ matrix.type-checker }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| strategy: | |
| matrix: | |
| type-checker: [mypy, pyright] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: uv python pin 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --extra dev | |
| - name: Type check with mypy | |
| if: matrix.type-checker == 'mypy' | |
| run: uv run mypy src/promptfoo/ | |
| - name: Type check with pyright | |
| if: matrix.type-checker == 'pyright' | |
| run: uv run pyright src/promptfoo/ | |
| test: | |
| name: Test (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }}) | |
| needs: resolve-promptfoo | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| env: | |
| PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints | |
| # causing BlockingIOError [Errno 35] when spawning subprocess | |
| os: [ubuntu-latest, windows-latest] | |
| # Test only min and max supported Python versions for efficiency | |
| python-version: ["3.10", "3.14"] | |
| include: | |
| - python-version: "3.10" | |
| node-version: "22.22.0" | |
| - python-version: "3.14" | |
| node-version: "24" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| package-manager-cache: false | |
| - name: Configure npm on Windows | |
| if: matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache" | |
| $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global" | |
| New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null | |
| "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| $globalPrefix | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| - name: Install promptfoo globally | |
| shell: bash | |
| run: npm install --global "promptfoo@$PROMPTFOO_VERSION" | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=4096 | |
| - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: uv python pin ${{ matrix.python-version }} | |
| - name: Install package with dev dependencies | |
| run: uv sync --locked --extra dev | |
| - name: Verify the wrapper can find the global npm installation | |
| run: uv run python -c "from promptfoo.cli import _find_external_promptfoo; path = _find_external_promptfoo(); print(path); assert path is not None" | |
| - name: Run unit tests | |
| run: uv run pytest tests/ -v -m 'not smoke' | |
| - name: Run smoke tests | |
| run: uv run pytest tests/smoke/ -v | |
| - name: Test CLI can be invoked | |
| run: uv run promptfoo --version | |
| - name: Test Node.js detection | |
| run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()" | |
| test-npx-fallback: | |
| name: Test npx fallback (py${{ matrix.python-version }}, node${{ matrix.node-version }}, ${{ matrix.os }}) | |
| needs: resolve-promptfoo | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| env: | |
| PROMPTFOO_VERSION: ${{ needs.resolve-promptfoo.outputs.version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Test npx fallback (without global install) | |
| # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints | |
| os: [ubuntu-latest, windows-latest] | |
| # Use middle-version Python for this test | |
| python-version: ["3.12"] | |
| node-version: ["22.22.0", "24"] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| package-manager-cache: false | |
| - name: Configure npm on Windows | |
| if: matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache" | |
| $globalPrefix = Join-Path $env:RUNNER_TEMP "npm-global" | |
| New-Item -ItemType Directory -Force -Path $cacheDir, $globalPrefix | Out-Null | |
| "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| # Intentionally skip installing promptfoo globally | |
| # This tests the npx fallback path | |
| - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: uv python pin ${{ matrix.python-version }} | |
| - name: Install package with dev dependencies | |
| run: uv sync --locked --extra dev | |
| - name: Verify the wrapper must fall back to npx | |
| run: uv run python -c "from promptfoo.cli import _find_external_promptfoo, check_npx_installed; path = _find_external_promptfoo(); print(path); assert path is None; assert check_npx_installed()" | |
| - name: Run unit tests | |
| run: uv run pytest tests/ -v -m 'not smoke' | |
| - name: Run smoke tests (with npx fallback) | |
| run: uv run pytest tests/smoke/ -v | |
| - name: Test CLI fallback to npx (no global install) | |
| run: uv run promptfoo --version | |
| - name: Test Node.js detection | |
| run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()" | |
| build: | |
| name: Build Package | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: uv python pin 3.12 | |
| - name: Build package | |
| run: uv build | |
| - name: Check distribution metadata with Twine | |
| run: uvx --from twine --constraints .github/requirements-twine.txt twine check --strict dist/* | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: dist | |
| path: dist/ | |
| if-no-files-found: error | |
| ci-success: | |
| name: CI Success | |
| needs: [resolve-promptfoo, lint, type-check, test, test-npx-fallback, build] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if all jobs succeeded | |
| env: | |
| JOB_RESULTS: ${{ join(needs.*.result, ',') }} | |
| run: | | |
| echo "Job results: $JOB_RESULTS" | |
| [[ "$JOB_RESULTS" =~ ^success(,success)*$ ]] |