diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 988518a070..7c08103c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,7 +157,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "24.22.1" - - run: node --test packages/junior-evals/scripts/report.test.mjs + - run: node --test packages/junior-evals/scripts/*.test.mjs junior-tests: name: test / junior ${{ matrix.suite }} diff --git a/.github/workflows/evals-behavioral-sample.yml b/.github/workflows/evals-behavioral-sample.yml new file mode 100644 index 0000000000..74c5e7c69d --- /dev/null +++ b/.github/workflows/evals-behavioral-sample.yml @@ -0,0 +1,217 @@ +name: Evals sample +run-name: "behavioral sample: ${{ inputs.ref }} x${{ inputs.samples }}" + +# Runs chosen behavioral eval files live several times, without replay. Run it +# one time for `main` and one time for a branch to compare their pass counts. +# See "Live Samples" in `packages/junior-evals/README.md`. + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + ref: + description: Branch, tag, or commit to run + required: true + default: main + files: + description: Space-separated eval files, relative to packages/junior-evals (for example evals/conversation/output.eval.ts) + required: true + filter: + description: Optional case name filter (vitest -t) + required: false + default: "" + samples: + description: Number of live runs (1-10) + required: true + default: "5" + +jobs: + plan: + name: behavioral sample / plan + runs-on: blacksmith-4vcpu-ubuntu-2404 + outputs: + samples: ${{ steps.plan.outputs.samples }} + sha: ${{ steps.plan.outputs.sha }} + steps: + # Resolve the ref one time, so that every sample runs the same commit. + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + - id: plan + env: + EVAL_FILES: ${{ inputs.files }} + SAMPLES: ${{ inputs.samples }} + run: | + set -euo pipefail + if [[ ! "$SAMPLES" =~ ^[1-9][0-9]*$ ]] || (( SAMPLES > 10 )); then + echo "::error::samples must be a number from 1 to 10, got '$SAMPLES'" + exit 1 + fi + read -r -a files <<< "$EVAL_FILES" + if [[ "${#files[@]}" -eq 0 ]]; then + echo "::error::files must name at least one eval file" + exit 1 + fi + for file in "${files[@]}"; do + if [[ ! "$file" =~ ^evals/[A-Za-z0-9_/.-]+\.eval\.ts$ || "$file" == *..* ]]; then + echo "::error::files must be paths like evals//.eval.ts, got '$file'" + exit 1 + fi + done + { + echo "samples=$(jq -cn --argjson n "$SAMPLES" '[range(1; $n + 1)]')" + echo "sha=$(git rev-parse HEAD)" + } >> "$GITHUB_OUTPUT" + + sample: + name: behavioral sample / ${{ matrix.sample }} + needs: plan + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + sample: ${{ fromJSON(needs.plan.outputs.samples) }} + # Keep services and setup the same as `evals-behavioral.yml`. + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: junior + POSTGRES_PASSWORD: junior + POSTGRES_DB: junior + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U junior -d junior" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + # Relative to `packages/junior-evals`. + RESULT_FILE: sample-results-${{ matrix.sample }}.json + JUNIOR_EVAL_REDIS_URL: redis://127.0.0.1:6379 + DATABASE_URL: postgres://junior:junior@localhost:5432/junior + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.plan.outputs.sha }} + persist-credentials: false + - uses: ./.github/actions/setup-node-pnpm + - name: Install cloudflared + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + --retry 3 --retry-max-time 120 --connect-timeout 10 --max-time 60 \ + https://github.com/cloudflare/cloudflared/releases/download/2026.10.4/cloudflared-linux-amd64 \ + --output "$RUNNER_TEMP/cloudflared" + echo "ae5e8add7884b0531d66bbca4570d1de491ff6da51e4ebe7eb160cdf241d9e42 $RUNNER_TEMP/cloudflared" | sha256sum --check + chmod +x "$RUNNER_TEMP/cloudflared" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + "$RUNNER_TEMP/cloudflared" version + - name: Keep sandbox snapshot ids + uses: actions/cache@v4 + with: + path: ~/.cache/junior-evals/sandbox-snapshots.json + key: eval-sandbox-snapshots-${{ github.run_id }} + restore-keys: eval-sandbox-snapshots- + - name: Run behavioral evals live + continue-on-error: true + env: + VITEST_EVALS_OUTPUT_FILE: ${{ env.RESULT_FILE }} + VITEST_EVALS_REPORT_LEVEL: info + # `off` sends every request live and records nothing. It needs the + # write token of Roach. + VITEST_EVALS_REPLAY_MODE: "off" + ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} + EVAL_FILES: ${{ inputs.files }} + FILTER: ${{ inputs.filter }} + run: | + set -euo pipefail + read -r -a files <<< "$EVAL_FILES" + args=("${files[@]}") + if [[ -n "$FILTER" ]]; then + args+=(-t "$FILTER") + fi + packages/junior-evals/scripts/network-jail.sh pnpm --filter @sentry/junior-evals evals:behavioral "${args[@]}" + - name: Require eval results + working-directory: packages/junior-evals + run: | + jq -e '.numTotalTests > 0 and (.testResults | length > 0)' "$RESULT_FILE" >/dev/null || { + echo "::error::sample ${{ matrix.sample }} produced no eval results ($RESULT_FILE)" + exit 1 + } + - name: Publish sample summary + uses: getsentry/vitest-evals@v0.16.1 + with: + results: packages/junior-evals/${{ env.RESULT_FILE }} + - name: Upload sample results + uses: actions/upload-artifact@v4 + with: + name: sample-results-${{ matrix.sample }} + path: packages/junior-evals/${{ env.RESULT_FILE }} + if-no-files-found: error + retention-days: 14 + + summary: + name: behavioral sample / summary + needs: [plan, sample] + if: ${{ always() && !cancelled() && needs.plan.result == 'success' }} + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + # Use the scripts of this workflow, not of the sampled ref. + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "24.22.1" + - name: Download sample results + uses: actions/download-artifact@v4 + with: + pattern: sample-results-* + path: sample-results + merge-multiple: true + - name: Write pass counts + env: + REF: ${{ inputs.ref }} + SHA: ${{ needs.plan.outputs.sha }} + EVAL_FILES: ${{ inputs.files }} + FILTER: ${{ inputs.filter }} + SAMPLES: ${{ inputs.samples }} + run: | + set -euo pipefail + shopt -s nullglob + files=(sample-results/sample-results-*.json) + if [[ "${#files[@]}" -eq 0 ]]; then + echo "::error::no sample produced results" + exit 1 + fi + { + echo "## Behavioral eval samples" + echo + echo "- ref: \`$REF\` (\`$SHA\`)" + echo "- files: \`$EVAL_FILES\`" + echo "- filter: \`${FILTER:-none}\`" + echo "- requested samples: $SAMPLES" + echo "- replay: off (every request is live)" + echo + node packages/junior-evals/scripts/sample-summary.mjs "${files[@]}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index d893e4fcd4..ac92f2c16c 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -528,6 +528,48 @@ Pass eval file paths, `-t` filters, and shard options directly after the suite s Behavioral and integration evals require real Vercel Sandbox access and public Quick Tunnel connectivity. If either bootstrap fails, the eval fails immediately with no local fallback path. Guardian and Router evals only need AI Gateway access. +## Live Samples + +A pull request run does not measure a prompt change well: + +- A passed test writes its recordings. Each later run with the same requests + replays that pass. Only a failed test goes live again. +- Each case gets one result in each run. + +To measure a prompt change, use the `Evals sample` workflow +(`.github/workflows/evals-behavioral-sample.yml`). It runs the behavioral +eval files that you select with `VITEST_EVALS_REPLAY_MODE=off`. Each request +goes live, and the run records nothing. It runs the files one time for each +sample, in parallel jobs. + +- `ref`: the branch, tag, or commit to run. The plan job resolves it to one + commit, so all samples run the same code. For a pull request from a fork, + use `refs/pull//head`. +- `files`: eval files, relative to `packages/junior-evals`, with a space + between them. +- `filter`: optional. A case name filter, the same as `-t`. +- `samples`: the number of runs, from 1 to 10. The default is 5. + +To compare a branch with `main`, start two runs with the same `files`, +`filter`, and `samples`: + +```sh +gh workflow run evals-behavioral-sample.yml --repo getsentry/junior \ + -f ref=main -f files="evals/conversation/output.eval.ts" -f samples=5 +gh workflow run evals-behavioral-sample.yml --repo getsentry/junior \ + -f ref=my-branch -f files="evals/conversation/output.eval.ts" -f samples=5 +``` + +The `behavioral sample / summary` job shows the passed runs of each case, for +example `3/5`. Compare the two summaries. Each sample job also shows its +`vitest-evals` summary with judge scores. The workflow does not upload to +`evals.sentry.dev` and does not publish a check. + +The ref must have `scripts/network-jail.sh` and the +`.github/actions/setup-node-pnpm` action. The summary uses the scripts of the +workflow branch. Live runs cost model and Sandbox usage for each sample, so +select only the files that the change can affect. + ## Authoring Rules - Write cases that run the agent with the agent test fixture (see **Agent Test Fixture**). diff --git a/packages/junior-evals/scripts/report.mjs b/packages/junior-evals/scripts/report.mjs index c9aed2ddbf..0bac7d627f 100644 --- a/packages/junior-evals/scripts/report.mjs +++ b/packages/junior-evals/scripts/report.mjs @@ -82,7 +82,7 @@ function scenarioResult(file, assertion) { } /** Exclude intentional skips. Report file setup failures as separate errors. */ -function scenariosFromReport(report) { +export function scenariosFromReport(report) { if (!Array.isArray(report.testResults)) { throw new Error("Expected a Vitest JSON report with testResults"); } diff --git a/packages/junior-evals/scripts/sample-summary.mjs b/packages/junior-evals/scripts/sample-summary.mjs new file mode 100644 index 0000000000..d2e3f32ad0 --- /dev/null +++ b/packages/junior-evals/scripts/sample-summary.mjs @@ -0,0 +1,43 @@ +import { readFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { scenariosFromReport } from "./report.mjs"; + +// Each report is one live sample of the same eval cases. A case that a sample +// did not finish counts as a run that did not pass. + +/** Return a Markdown table with the passed runs of each case across samples. */ +export function summarizeSamples(reports) { + const cases = new Map(); + for (const report of reports) { + for (const { name, status } of scenariosFromReport(report)) { + const counts = cases.get(name) ?? { passed: 0, runs: 0 }; + counts.runs += 1; + if (status === "passed") counts.passed += 1; + cases.set(name, counts); + } + } + return [ + `Sample reports: ${reports.length}`, + "", + "| Case | Passed |", + "| --- | --- |", + ...[...cases].map( + ([name, { passed, runs }]) => + `| ${name.replaceAll("|", "\\|")} | ${passed}/${runs} |`, + ), + ].join("\n"); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + const files = process.argv.slice(2); + if (files.length === 0) { + throw new Error("Usage: sample-summary.mjs [...]"); + } + const reports = await Promise.all( + files.map(async (file) => JSON.parse(await readFile(file, "utf8"))), + ); + console.log(summarizeSamples(reports)); +} diff --git a/packages/junior-evals/scripts/sample-summary.test.mjs b/packages/junior-evals/scripts/sample-summary.test.mjs new file mode 100644 index 0000000000..6a9e6adfc1 --- /dev/null +++ b/packages/junior-evals/scripts/sample-summary.test.mjs @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { summarizeSamples } from "./sample-summary.mjs"; + +const file = + "/home/runner/work/junior/junior/packages/junior-evals/evals/conversation/output.eval.ts"; + +function sample(statuses) { + return { + testResults: [ + { + name: file, + status: Object.values(statuses).includes("failed") + ? "failed" + : "passed", + message: "", + assertionResults: Object.entries(statuses).map( + ([fullName, status]) => ({ + fullName, + status, + failureMessages: status === "failed" ? ["judge score 0.4"] : [], + }), + ), + }, + ], + }; +} + +test("counts the passed runs of each case across samples", () => { + const summary = summarizeSamples([ + sample({ "Replies | short": "passed", "Replies cite": "passed" }), + sample({ "Replies | short": "failed", "Replies cite": "passed" }), + sample({ "Replies | short": "pending", "Replies cite": "skipped" }), + ]); + + assert.equal( + summary, + [ + "Sample reports: 3", + "", + "| Case | Passed |", + "| --- | --- |", + "| evals/conversation/output.eval.ts > Replies \\| short | 1/3 |", + "| evals/conversation/output.eval.ts > Replies cite | 2/2 |", + ].join("\n"), + ); +});