From a0c3dd58136c49d7894a0454b13d5fe4fcbcc444 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sat, 26 Sep 2026 19:53:37 +0000 Subject: [PATCH 1/5] Add an end-to-end harness for bare metal job callbacks Runs the job callback scenarios against the dev API from a test pull request, and the push scenario from a push to a dedicated branch. The submit jobs start the detached runs and exit, then a runner on a KVM machine picks up the queued Jobs, and the attach workflow on main reports each one when its callback arrives. Each submit job checks the CLI's raw output for the tokens it was given, so a leak fails the job before the log is masked. This is test tooling only and must never be merged. --- .github/e2e/image/Dockerfile | 2 + .github/e2e/image/e2e-bench | 21 ++ .github/e2e/install-bencher/action.yml | 63 +++++ .github/e2e/runner.sh | 85 ++++++ .github/e2e/setup.sh | 50 ++++ .github/e2e/submit.sh | 64 +++++ .github/workflows/bare_metal_e2e.yml | 367 +++++++++++++++++++++++++ 7 files changed, 652 insertions(+) create mode 100644 .github/e2e/image/Dockerfile create mode 100755 .github/e2e/image/e2e-bench create mode 100644 .github/e2e/install-bencher/action.yml create mode 100755 .github/e2e/runner.sh create mode 100755 .github/e2e/setup.sh create mode 100755 .github/e2e/submit.sh create mode 100644 .github/workflows/bare_metal_e2e.yml diff --git a/.github/e2e/image/Dockerfile b/.github/e2e/image/Dockerfile new file mode 100644 index 0000000..ab1c33e --- /dev/null +++ b/.github/e2e/image/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +COPY --chmod=755 e2e-bench /usr/local/bin/e2e-bench diff --git a/.github/e2e/image/e2e-bench b/.github/e2e/image/e2e-bench new file mode 100755 index 0000000..6f3f7ab --- /dev/null +++ b/.github/e2e/image/e2e-bench @@ -0,0 +1,21 @@ +#!/bin/sh +# A stand-in benchmark: `run` prints one Bencher Metric Format result, +# `fail` exits with an error, and `sleep ` runs long enough to cancel. +set -eu + +case "${1:-}" in +run) ;; +fail) + echo "e2e-bench: failing on purpose" >&2 + exit 1 + ;; +sleep) + sleep "${2:?sleep needs a number of seconds}" + ;; +*) + echo "usage: e2e-bench run | fail | sleep " >&2 + exit 2 + ;; +esac + +echo '{"e2e::callback": {"latency": {"value": 1000.0}}}' diff --git a/.github/e2e/install-bencher/action.yml b/.github/e2e/install-bencher/action.yml new file mode 100644 index 0000000..b28ba73 --- /dev/null +++ b/.github/e2e/install-bencher/action.yml @@ -0,0 +1,63 @@ +name: Install Bencher from the job callbacks branch +description: Install the Bencher CLI or runner built from the tip of the job callbacks branch, cached by its commit. + +inputs: + component: + description: "`cli` for the `bencher` CLI, or `runner` for the runner daemon" + required: true + +runs: + using: composite + steps: + - name: Resolve the Bencher ${{ inputs.component }} commit + id: commit + shell: bash + env: + COMPONENT: ${{ inputs.component }} + run: | + case "$COMPONENT" in + cli | runner) ;; + *) + echo "::error::Unknown component: $COMPONENT" + exit 1 + ;; + esac + sha="$(git ls-remote https://github.com/bencherdev/bencher refs/heads/u/ep/callback/smoke | cut -f1)" + test -n "$sha" + echo "Bencher $COMPONENT from bencherdev/bencher@$sha" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "key=bencher-$COMPONENT-${ImageOS:?}-$RUNNER_ARCH-$sha" >> "$GITHUB_OUTPUT" + - name: Restore the Bencher ${{ inputs.component }} + id: restore + uses: actions/cache/restore@v5 + with: + path: ~/.bencher-${{ inputs.component }} + key: ${{ steps.commit.outputs.key }} + - name: Install the Bencher CLI + if: inputs.component == 'cli' && steps.restore.outputs.cache-hit != 'true' + shell: bash + env: + BENCHER_SHA: ${{ steps.commit.outputs.sha }} + run: cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --root ~/.bencher-cli bencher_cli + - name: Install the Bencher runner + if: inputs.component == 'runner' && steps.restore.outputs.cache-hit != 'true' + shell: bash + env: + BENCHER_SHA: ${{ steps.commit.outputs.sha }} + run: | + sudo apt-get update && sudo apt-get install -y musl-tools + rustup target add x86_64-unknown-linux-musl + # The release runner embeds a static `bencher-init` for its VMs. + cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --target x86_64-unknown-linux-musl --root "$RUNNER_TEMP/bencher-init" bencher_init + BENCHER_INIT_PATH="$RUNNER_TEMP/bencher-init/bin/bencher-init" cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --root ~/.bencher-runner bencher_runner_cli + - name: Save the Bencher ${{ inputs.component }} + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: ~/.bencher-${{ inputs.component }} + key: ${{ steps.commit.outputs.key }} + - name: Add the Bencher ${{ inputs.component }} to the path + shell: bash + env: + COMPONENT: ${{ inputs.component }} + run: echo "$HOME/.bencher-$COMPONENT/bin" >> "$GITHUB_PATH" diff --git a/.github/e2e/runner.sh b/.github/e2e/runner.sh new file mode 100755 index 0000000..d00dc67 --- /dev/null +++ b/.github/e2e/runner.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Host a dev runner for the scenarios' Jobs, after every submit job has finished: +# +# runner.sh serve serve `test-spec` as `test-runner` for 30 minutes +# runner.sh cancel serve `no-sandbox-spec` as `test-runner-no-sandbox`, and kill the runner +# once the cancel scenario's Job is running, so the server cancels that Job +# when its timeout and grace period pass +set -euo pipefail + +now() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +rotate_key() { + local key + key="$(bencher runner key --token "$BENCHER_ADMIN_API_TOKEN" "$1" | jq -r '.key // empty')" + if [ -z "$key" ]; then + echo "::error::Rotating the key of $1 returned no key" + return 1 + fi + echo "::add-mask::$key" + export BENCHER_RUNNER_KEY="$key" +} + +serve() { + rotate_key test-runner + echo "Runner up at $(now)" + local status=0 + timeout --kill-after 60s 30m runner up --runner test-runner --no-auto-update || status=$? + # `timeout` exits 124 after it stops the runner, or 137 if the runner needed a SIGKILL. + if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then + echo "Runner down at $(now)" + return 0 + fi + return "$status" +} + +cancel() { + rotate_key test-runner-no-sandbox + local jobs count job + jobs="$(bencher job list "$PAID_PROJECT" --status pending --sort created --direction desc --per-page 255)" + count="$(jq '[.[] | select(.spec.slug == "no-sandbox-spec")] | length' <<< "$jobs")" + job="$(jq -r '[.[] | select(.spec.slug == "no-sandbox-spec")][0].uuid // empty' <<< "$jobs")" + if [ -z "$job" ]; then + echo "::error::There is no pending no-sandbox-spec Job in $PAID_PROJECT to cancel" + return 1 + fi + if [ "$count" -gt 1 ]; then + echo "::warning::$count no-sandbox-spec Jobs are pending in $PAID_PROJECT, and the runner may run older ones before $job" + fi + + local log="$RUNNER_TEMP/runner.log" + echo "Runner up at $(now), waiting for Job $job to start" + runner up --runner test-runner-no-sandbox --danger-allow-no-sandbox --no-auto-update > "$log" 2>&1 & + local pid=$! + local deadline=$((SECONDS + 900)) + until grep -qF "Starting iteration 1/1 for job $job" "$log"; do + if ! kill -0 "$pid" 2> /dev/null; then + cat "$log" + echo "::error::The runner exited before Job $job started" + return 1 + fi + if [ "$SECONDS" -ge "$deadline" ]; then + kill -KILL "$pid" + cat "$log" + echo "::error::Job $job did not start within 15 minutes" + return 1 + fi + sleep 1 + done + # A SIGTERM would let the runner finish the Job first, so the runner gets no chance to. + kill -KILL "$pid" + wait "$pid" || true + cat "$log" + echo "Killed the runner at $(now) while Job $job was running" +} + +case "${1:-}" in +serve) serve ;; +cancel) cancel ;; +*) + echo "usage: runner.sh serve | cancel" >&2 + exit 2 + ;; +esac diff --git a/.github/e2e/setup.sh b/.github/e2e/setup.sh new file mode 100755 index 0000000..5924861 --- /dev/null +++ b/.github/e2e/setup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Make sure dev has what the scenarios need. Every step is idempotent, +# so it runs as is after every deploy wipes the dev database. +set -euo pipefail + +ensure_project() { + local organization="$1" slug="$2" name="$3" + if bencher project view "$slug" > /dev/null 2>&1; then + echo "Project $slug exists" + else + bencher project create "$organization" --name "$name" --slug "$slug" > /dev/null + echo "Created project $slug in $organization" + fi +} + +# The seed recreates the paid organization on every deploy, and only a plan makes it paid. +bencher organization view "$PAID_ORGANIZATION" > /dev/null +if bencher plan view --attempts 3 "$PAID_ORGANIZATION" > /dev/null 2>&1; then + echo "Organization $PAID_ORGANIZATION has a plan" +elif [ -z "${BENCHER_DEV_SUBSCRIPTION:-}" ]; then + echo "::warning::The BENCHER_DEV_SUBSCRIPTION repository variable is not set, so $PAID_ORGANIZATION has no plan and every callback is skipped" +else + # The level only matters to a licensed plan: a metered plan reads its level from the subscription. + bencher plan create "$PAID_ORGANIZATION" \ + --checkout "$BENCHER_DEV_SUBSCRIPTION" \ + --level "${BENCHER_DEV_PLAN_LEVEL:-team}" \ + --skip-remote > /dev/null + echo "Attached the subscription to $PAID_ORGANIZATION" +fi +ensure_project "$PAID_ORGANIZATION" "$PAID_PROJECT" "Callback E2E" + +if bencher organization view "$FREE_ORGANIZATION" > /dev/null 2>&1; then + echo "Organization $FREE_ORGANIZATION exists" +else + bencher organization create --name "Callback E2E Free" --slug "$FREE_ORGANIZATION" > /dev/null + echo "Created organization $FREE_ORGANIZATION" +fi +if bencher plan view --attempts 3 "$FREE_ORGANIZATION" > /dev/null 2>&1; then + echo "::error::Organization $FREE_ORGANIZATION has a plan, but the no plan scenario needs one without" + exit 1 +fi +ensure_project "$FREE_ORGANIZATION" "$FREE_PROJECT" "Callback E2E Free" + +docker build --tag e2e-bench "$(dirname "$0")/image" +printf '%s' "$BENCHER_API_TOKEN" | docker login "$BENCHER_REGISTRY" --username "$DEV_USER_EMAIL" --password-stdin +for project in "$PAID_PROJECT" "$FREE_PROJECT"; do + docker tag e2e-bench "$BENCHER_REGISTRY/$project:$IMAGE_TAG" + docker push "$BENCHER_REGISTRY/$project:$IMAGE_TAG" +done +docker logout "$BENCHER_REGISTRY" diff --git a/.github/e2e/submit.sh b/.github/e2e/submit.sh new file mode 100755 index 0000000..b2dcc02 --- /dev/null +++ b/.github/e2e/submit.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Run a detached `bencher run` and check its raw output, before GitHub masks the log: +# +# submit.sh bencher run ... +# +# REDACTED lists the environment variables whose values must never be printed, +# and EXPECT_SKIPPED says whether the server should skip the callback for want of a plan. +set -euo pipefail + +scenario="$1" +shift + +out="$RUNNER_TEMP/bencher-run.stdout" +err="$RUNNER_TEMP/bencher-run.stderr" +status=0 +"$@" > "$out" 2> "$err" || status=$? +cat "$out" +cat "$err" >&2 + +failed=0 +fail() { + echo "::error title=$scenario::$1" + failed=1 +} + +for name in $REDACTED; do + value="${!name:-}" + if [ -z "$value" ]; then + fail "$name is empty, so there is nothing to check its redaction against" + elif grep -qF -- "$value" "$out" "$err"; then + fail "the CLI printed the raw value of $name" + fi +done +if ! grep -qF '"authorization": "************"' "$out"; then + fail "the Bencher New Report echo does not show the CLI's mask for the authorization header" +fi +if grep -qF '/dispatches' "$out" "$err"; then + fail "the CLI printed the callback URL past its origin" +fi + +skipped=false +if grep -qxF 'callback skipped: requires a Bencher Plus plan' "$err"; then + skipped=true +fi +if [ "$skipped" != "$EXPECT_SKIPPED" ]; then + fail "expected a skipped callback to be $EXPECT_SKIPPED, but it was $skipped" +fi + +job="$(sed -n 's/^Remote job submitted successfully: //p' "$err" | head -n 1)" +check="$(grep -oE '"check": [0-9]+' "$out" | head -n 1 | grep -oE '[0-9]+' || true)" +if [ -z "$job" ]; then + fail "the CLI did not print the submitted Job" +fi +echo "::notice title=$scenario::Job ${job:-none}, check run ${check:-none}, commit $HEAD_SHA" +{ + echo "| Scenario | Job | Check run | Commit |" + echo "| --- | --- | --- | --- |" + echo "| $scenario | ${job:-none} | ${check:-none} | $HEAD_SHA |" +} >> "$GITHUB_STEP_SUMMARY" + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$failed" diff --git a/.github/workflows/bare_metal_e2e.yml b/.github/workflows/bare_metal_e2e.yml new file mode 100644 index 0000000..47bbc77 --- /dev/null +++ b/.github/workflows/bare_metal_e2e.yml @@ -0,0 +1,367 @@ +name: Bare Metal Callbacks E2E + +on: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: [u/ep/bare-metal-e2e-push] + +# Each run rotates the dev runners' keys, so two runs at once would lock out each other's runners. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: {} + +env: + BENCHER_HOST: https://dev.api.bencher.dev + BENCHER_REGISTRY: dev.registry.bencher.dev + DEV_USER_EMAIL: muriel.bagge@nowhere.com + PAID_ORGANIZATION: muriel-bagge + PAID_PROJECT: callback-e2e + FREE_ORGANIZATION: callback-e2e-free + FREE_PROJECT: callback-e2e-free + IMAGE_TAG: e2e + +jobs: + cli: + name: Build the Bencher CLI + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - run: bencher --version + + runner_build: + name: Build the Bencher runner + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: runner + + setup: + name: Set up dev + needs: cli + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 20 + env: + BENCHER_API_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + BENCHER_DEV_SUBSCRIPTION: ${{ vars.BENCHER_DEV_SUBSCRIPTION }} + BENCHER_DEV_PLAN_LEVEL: ${{ vars.BENCHER_DEV_PLAN_LEVEL }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - run: .github/e2e/setup.sh + + submit: + name: Submit (${{ matrix.scenario }}) + needs: setup + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + permissions: + checks: write + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + # Happy path, and the plain Job beside the build time Job + - scenario: happy path + plan: paid + spec: test-spec + command: run + - scenario: build time + plan: paid + spec: test-spec + command: run + build_time: true + - scenario: failure + plan: paid + spec: test-spec + command: fail + ci_id: failure + - scenario: cancel + plan: paid + spec: no-sandbox-spec + command: sleep 600 + ci_id: cancel + job_timeout: 60 + - scenario: matrix a + plan: paid + spec: test-spec + command: run + ci_id: matrix-a + - scenario: matrix b + plan: paid + spec: test-spec + command: run + ci_id: matrix-b + - scenario: no plan + plan: none + spec: test-spec + command: run + ci_id: no-plan + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - name: Submit PR Benchmarks with Bencher + env: + SCENARIO: ${{ matrix.scenario }} + PLAN: ${{ matrix.plan }} + SPEC: ${{ matrix.spec }} + COMMAND: ${{ matrix.command }} + CI_ID: ${{ matrix.ci_id }} + BUILD_TIME: ${{ matrix.build_time }} + JOB_TIMEOUT: ${{ matrix.job_timeout }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CALLBACK_TOKEN: ${{ secrets.BENCHER_CALLBACK_TOKEN }} + ACTIONS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + REDACTED: CALLBACK_TOKEN ACTIONS_TOKEN DEV_TOKEN + run: | + project="$PAID_PROJECT" + EXPECT_SKIPPED=false + if [ "$PLAN" = none ]; then + project="$FREE_PROJECT" + EXPECT_SKIPPED=true + fi + export EXPECT_SKIPPED + options=() + if [ -n "$CI_ID" ]; then options+=(--ci-id "$CI_ID"); fi + if [ "$BUILD_TIME" = true ]; then options+=(--build-time); fi + if [ -n "$JOB_TIMEOUT" ]; then options+=(--job-timeout "$JOB_TIMEOUT"); fi + read -ra command <<< "$COMMAND" + .github/e2e/submit.sh "$SCENARIO" \ + bencher run \ + --host "$BENCHER_HOST" \ + --project "$project" \ + --token '${{ secrets.DEV_BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-hash "$BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --adapter json \ + --image "$project:$IMAGE_TAG" \ + --spec "$SPEC" \ + --detach \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-callback-token '${{ secrets.BENCHER_CALLBACK_TOKEN }}' \ + "${options[@]}" \ + /usr/local/bin/e2e-bench "${command[@]}" + + submit_revoked: + name: Submit (revoked token) + needs: setup + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + # `contents: write` lets this token send a repository_dispatch, so its callback's 401 can only mean it was revoked. + permissions: + checks: write + contents: write + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - name: Submit PR Benchmarks with Bencher + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + ACTIONS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + REDACTED: ACTIONS_TOKEN DEV_TOKEN + EXPECT_SKIPPED: false + run: | + .github/e2e/submit.sh "revoked token" \ + bencher run \ + --host "$BENCHER_HOST" \ + --project "$PAID_PROJECT" \ + --token '${{ secrets.DEV_BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-hash "$BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --adapter json \ + --image "$PAID_PROJECT:$IMAGE_TAG" \ + --spec test-spec \ + --detach \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-callback-token '${{ secrets.GITHUB_TOKEN }}' \ + --ci-id revoked \ + /usr/local/bin/e2e-bench run + + submit_raw: + name: Submit (raw callback) + needs: setup + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - name: Submit PR Benchmarks with Bencher + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CALLBACK_TOKEN: ${{ secrets.BENCHER_CALLBACK_TOKEN }} + DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + REDACTED: CALLBACK_TOKEN DEV_TOKEN + EXPECT_SKIPPED: false + # `--ci-id` needs `--github-actions`, which conflicts with `--callback-url`, + # so the body carries the ID, and the head commit and pull request the attach reports to. + # It carries them twice: as top-level keys, and in the `bencher` and `github` objects. + run: | + body="$(jq -cn --arg sha "$HEAD_SHA" --argjson number "$PR_NUMBER" '{ + event_type: "bencher_run", + client_payload: { + job: "{{ job.uuid }}", + project: "{{ project.slug }}", + number: $number, + sha: $sha, + ci_id: "raw", + bencher: { + project: "{{ project.slug }}", + job: "{{ job.uuid }}", + ci_id: "raw", + ci_number: $number + }, + github: {sha: $sha}, + report: "{{ report }}" + } + }')" + .github/e2e/submit.sh "raw callback" \ + bencher run \ + --host "$BENCHER_HOST" \ + --project "$PAID_PROJECT" \ + --token '${{ secrets.DEV_BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-hash "$BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --adapter json \ + --image "$PAID_PROJECT:$IMAGE_TAG" \ + --spec test-spec \ + --detach \ + --callback-url "https://api.github.com/repos/$GITHUB_REPOSITORY/dispatches" \ + --callback-header "Accept: application/vnd.github+json" \ + --callback-header "Authorization: Bearer $CALLBACK_TOKEN" \ + --callback-body "$body" \ + /usr/local/bin/e2e-bench run + + submit_push: + name: Submit (push) + needs: setup + if: github.event_name == 'push' + permissions: + checks: write + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - name: Submit Benchmarks with Bencher + env: + HEAD_SHA: ${{ github.sha }} + CALLBACK_TOKEN: ${{ secrets.BENCHER_CALLBACK_TOKEN }} + ACTIONS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + REDACTED: CALLBACK_TOKEN ACTIONS_TOKEN DEV_TOKEN + EXPECT_SKIPPED: false + run: | + .github/e2e/submit.sh push \ + bencher run \ + --host "$BENCHER_HOST" \ + --project "$PAID_PROJECT" \ + --token '${{ secrets.DEV_BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_REF_NAME" \ + --adapter json \ + --image "$PAID_PROJECT:$IMAGE_TAG" \ + --spec test-spec \ + --detach \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-callback-token '${{ secrets.BENCHER_CALLBACK_TOKEN }}' \ + --ci-id push \ + /usr/local/bin/e2e-bench run + + # Starts only after every submit job has finished, so each Job waits in the queue first. + runner: + name: Run the Jobs on a dev runner + needs: [setup, runner_build, submit, submit_revoked, submit_raw, submit_push] + if: ${{ !cancelled() && needs.setup.result == 'success' && needs.runner_build.result == 'success' }} + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 45 + env: + BENCHER_API_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + BENCHER_ADMIN_API_TOKEN: ${{ secrets.DEV_BENCHER_ADMIN_API_TOKEN }} + steps: + - uses: actions/checkout@v6 + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - uses: ./.github/e2e/install-bencher + with: + component: cli + - uses: ./.github/e2e/install-bencher + with: + component: runner + - run: .github/e2e/runner.sh serve + + runner_cancel: + name: Cancel a Job mid-run + needs: [setup, runner_build, submit] + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.setup.result == 'success' && needs.runner_build.result == 'success' }} + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + BENCHER_API_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + BENCHER_ADMIN_API_TOKEN: ${{ secrets.DEV_BENCHER_ADMIN_API_TOKEN }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - uses: ./.github/e2e/install-bencher + with: + component: runner + - run: .github/e2e/runner.sh cancel From 898f70d26fdd8dcb713f8a5ad20b48765f13959f Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sat, 26 Sep 2026 21:25:15 +0000 Subject: [PATCH 2/5] Stop the dev runner after 10 minutes Every Job in a run finishes within about a minute of the runner starting, so the 30 minute timeout only held the runner, and the next run waiting on the concurrency group, idle. --- .github/e2e/runner.sh | 4 ++-- .github/workflows/bare_metal_e2e.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/e2e/runner.sh b/.github/e2e/runner.sh index d00dc67..5ff2330 100755 --- a/.github/e2e/runner.sh +++ b/.github/e2e/runner.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Host a dev runner for the scenarios' Jobs, after every submit job has finished: # -# runner.sh serve serve `test-spec` as `test-runner` for 30 minutes +# runner.sh serve serve `test-spec` as `test-runner` for 10 minutes # runner.sh cancel serve `no-sandbox-spec` as `test-runner-no-sandbox`, and kill the runner # once the cancel scenario's Job is running, so the server cancels that Job # when its timeout and grace period pass @@ -26,7 +26,7 @@ serve() { rotate_key test-runner echo "Runner up at $(now)" local status=0 - timeout --kill-after 60s 30m runner up --runner test-runner --no-auto-update || status=$? + timeout --kill-after 60s 10m runner up --runner test-runner --no-auto-update || status=$? # `timeout` exits 124 after it stops the runner, or 137 if the runner needed a SIGKILL. if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then echo "Runner down at $(now)" diff --git a/.github/workflows/bare_metal_e2e.yml b/.github/workflows/bare_metal_e2e.yml index 47bbc77..2de972a 100644 --- a/.github/workflows/bare_metal_e2e.yml +++ b/.github/workflows/bare_metal_e2e.yml @@ -326,7 +326,7 @@ jobs: permissions: contents: read runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 20 env: BENCHER_API_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} BENCHER_ADMIN_API_TOKEN: ${{ secrets.DEV_BENCHER_ADMIN_API_TOKEN }} From c7d293125159b186ef6b6f4e73287a4063dbabe4 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sat, 26 Sep 2026 23:36:37 +0000 Subject: [PATCH 3/5] Send only the two-object payload in the raw callback scenario The job callbacks branch now sends the dispatch payload as a bencher object and a github object, so the raw callback body no longer carries the old top-level keys. --- .github/workflows/bare_metal_e2e.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/bare_metal_e2e.yml b/.github/workflows/bare_metal_e2e.yml index 2de972a..0d8db41 100644 --- a/.github/workflows/bare_metal_e2e.yml +++ b/.github/workflows/bare_metal_e2e.yml @@ -241,16 +241,10 @@ jobs: EXPECT_SKIPPED: false # `--ci-id` needs `--github-actions`, which conflicts with `--callback-url`, # so the body carries the ID, and the head commit and pull request the attach reports to. - # It carries them twice: as top-level keys, and in the `bencher` and `github` objects. run: | body="$(jq -cn --arg sha "$HEAD_SHA" --argjson number "$PR_NUMBER" '{ event_type: "bencher_run", client_payload: { - job: "{{ job.uuid }}", - project: "{{ project.slug }}", - number: $number, - sha: $sha, - ci_id: "raw", bencher: { project: "{{ project.slug }}", job: "{{ job.uuid }}", From f9701e70ca688f2022e3ae0babb8c45a1097908f Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sun, 27 Sep 2026 01:00:54 +0000 Subject: [PATCH 4/5] Split the no plan scenario into a free dispatch and a free custom callback A repository_dispatch callback is now sealed on every plan, so the Job in the free organization that composes a dispatch expects it delivered like the happy path. A new submit job sends a custom callback from the same organization and expects it skipped, since every other callback still needs a paid plan. Its header carries a throwaway value for the redaction check, and the check for a printed callback URL path now takes that job's path. --- .github/e2e/submit.sh | 5 ++- .github/workflows/bare_metal_e2e.yml | 60 ++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.github/e2e/submit.sh b/.github/e2e/submit.sh index b2dcc02..da20ad9 100755 --- a/.github/e2e/submit.sh +++ b/.github/e2e/submit.sh @@ -4,7 +4,8 @@ # submit.sh bencher run ... # # REDACTED lists the environment variables whose values must never be printed, -# and EXPECT_SKIPPED says whether the server should skip the callback for want of a plan. +# EXPECT_SKIPPED says whether the server should skip the callback for want of a plan, +# and CALLBACK_PATH is the callback URL's path, which the CLI must never print (default `/dispatches`). set -euo pipefail scenario="$1" @@ -34,7 +35,7 @@ done if ! grep -qF '"authorization": "************"' "$out"; then fail "the Bencher New Report echo does not show the CLI's mask for the authorization header" fi -if grep -qF '/dispatches' "$out" "$err"; then +if grep -qF -- "${CALLBACK_PATH:-/dispatches}" "$out" "$err"; then fail "the CLI printed the callback URL past its origin" fi diff --git a/.github/workflows/bare_metal_e2e.yml b/.github/workflows/bare_metal_e2e.yml index 0d8db41..da979a9 100644 --- a/.github/workflows/bare_metal_e2e.yml +++ b/.github/workflows/bare_metal_e2e.yml @@ -115,7 +115,8 @@ jobs: spec: test-spec command: run ci_id: matrix-b - - scenario: no plan + # A `repository_dispatch` callback is sealed on every plan. + - scenario: free dispatch plan: none spec: test-spec command: run @@ -140,14 +141,10 @@ jobs: ACTIONS_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} REDACTED: CALLBACK_TOKEN ACTIONS_TOKEN DEV_TOKEN + EXPECT_SKIPPED: false run: | project="$PAID_PROJECT" - EXPECT_SKIPPED=false - if [ "$PLAN" = none ]; then - project="$FREE_PROJECT" - EXPECT_SKIPPED=true - fi - export EXPECT_SKIPPED + if [ "$PLAN" = none ]; then project="$FREE_PROJECT"; fi options=() if [ -n "$CI_ID" ]; then options+=(--ci-id "$CI_ID"); fi if [ "$BUILD_TIME" = true ]; then options+=(--build-time); fi @@ -275,6 +272,53 @@ jobs: --callback-body "$body" \ /usr/local/bin/e2e-bench run + # Any callback but a `repository_dispatch` still needs a paid plan. + submit_free_callback: + name: Submit (free custom callback) + needs: setup + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + - uses: ./.github/e2e/install-bencher + with: + component: cli + - name: Submit PR Benchmarks with Bencher + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DEV_TOKEN: ${{ secrets.DEV_BENCHER_API_TOKEN }} + REDACTED: RECEIVER_TOKEN DEV_TOKEN + EXPECT_SKIPPED: true + CALLBACK_PATH: /bencher/callback + # The receiver is on a reserved domain, and the skipped callback is never sent, + # so its header carries a throwaway value that only the redaction check reads. + run: | + RECEIVER_TOKEN="$(openssl rand -hex 16)" + echo "::add-mask::$RECEIVER_TOKEN" + export RECEIVER_TOKEN + .github/e2e/submit.sh "free custom callback" \ + bencher run \ + --host "$BENCHER_HOST" \ + --project "$FREE_PROJECT" \ + --token '${{ secrets.DEV_BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-hash "$BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --adapter json \ + --image "$FREE_PROJECT:$IMAGE_TAG" \ + --spec test-spec \ + --detach \ + --callback-url "https://receiver.example$CALLBACK_PATH" \ + --callback-header "Authorization: Bearer $RECEIVER_TOKEN" \ + /usr/local/bin/e2e-bench run + submit_push: name: Submit (push) needs: setup @@ -315,7 +359,7 @@ jobs: # Starts only after every submit job has finished, so each Job waits in the queue first. runner: name: Run the Jobs on a dev runner - needs: [setup, runner_build, submit, submit_revoked, submit_raw, submit_push] + needs: [setup, runner_build, submit, submit_revoked, submit_raw, submit_free_callback, submit_push] if: ${{ !cancelled() && needs.setup.result == 'success' && needs.runner_build.result == 'success' }} permissions: contents: read From 9905be4df8042e127afc5d9313f80fe6841f5f40 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sun, 27 Sep 2026 19:47:32 +0000 Subject: [PATCH 5/5] Render a placeholder inside a string in the raw callback scenario The raw callback body's ci_id embeds the project name placeholder inside text, so the attach's check name shows the server substituted it. --- .github/workflows/bare_metal_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bare_metal_e2e.yml b/.github/workflows/bare_metal_e2e.yml index da979a9..a00df2b 100644 --- a/.github/workflows/bare_metal_e2e.yml +++ b/.github/workflows/bare_metal_e2e.yml @@ -245,7 +245,7 @@ jobs: bencher: { project: "{{ project.slug }}", job: "{{ job.uuid }}", - ci_id: "raw", + ci_id: "raw {{ project.name }}", ci_number: $number }, github: {sha: $sha},