From f486d57335b323205b737b80eb9815b33a600181 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:52:32 +0000 Subject: [PATCH 1/4] feat: add weekly OSS benchmark workflow to benchmarks repo Runs weekly from computesdk/benchmarks (default branch) against the branch, committing results to results-weekly/ and weekly-*.svg so they don't overwrite the existing per-category results and charts. Includes the aggregate helper scripts required by the workflow: - benchmarks/src/ingest.ts - scripts/build-daily-matrix.mjs - scripts/oss-workflow-defaults.mjs Co-Authored-By: Noah Kiser --- .github/workflows/weekly-oss-benchmarks.yml | 510 ++++++++++++++++++++ benchmarks/src/ingest.ts | 220 +++++++++ scripts/build-daily-matrix.mjs | 89 ++++ scripts/oss-workflow-defaults.mjs | 75 +++ 4 files changed, 894 insertions(+) create mode 100644 .github/workflows/weekly-oss-benchmarks.yml create mode 100644 benchmarks/src/ingest.ts create mode 100644 scripts/build-daily-matrix.mjs create mode 100644 scripts/oss-workflow-defaults.mjs diff --git a/.github/workflows/weekly-oss-benchmarks.yml b/.github/workflows/weekly-oss-benchmarks.yml new file mode 100644 index 00000000..d8bc784f --- /dev/null +++ b/.github/workflows/weekly-oss-benchmarks.yml @@ -0,0 +1,510 @@ +name: Weekly OSS Benchmarks + +on: + schedule: + - cron: '0 10 * * 0' # Weekly on Sunday at 10am UTC (5am CDT) + workflow_dispatch: + inputs: + benchmark: + description: 'Benchmark category to run' + required: false + default: 'all' + type: choice + options: + - all + - sandbox + - dax + - storage + - snapshot-fork + - browser + - browser-throughput + - ai-gateway + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + iterations: + description: 'Override OSS iteration defaults for a manual run' + required: false + default: '' + sandbox_iterations: + description: 'Sandbox TTI iterations' + required: false + default: '' + dax_iterations: + description: 'DAX iterations' + required: false + default: '' + storage_iterations: + description: 'Storage iterations' + required: false + default: '' + snapshot_fork_iterations: + description: 'Snapshot/fork iterations' + required: false + default: '' + browser_iterations: + description: 'Browser session iterations' + required: false + default: '' + browser_throughput_iterations: + description: 'Browser throughput iterations' + required: false + default: '' + ai_gateway_iterations: + description: 'AI gateway iterations' + required: false + default: '' + dry_run: + description: 'Run without ingesting or committing results' + required: false + default: false + type: boolean + +concurrency: + group: weekly-oss-benchmarks + cancel-in-progress: false + +permissions: + contents: write + +env: + OSS_REPOSITORY: computesdk/benchmarks + OSS_REF: weekly + OSS_PATH: oss + +jobs: + matrix: + name: Discover OSS providers + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + outputs: + include: ${{ steps.matrix.outputs.include }} + oss_sha: ${{ steps.oss_sha.outputs.sha }} + throughput_iterations: ${{ steps.matrix.outputs.throughput_iterations }} + steps: + - uses: actions/checkout@v4 + with: + path: daily + - uses: actions/checkout@v4 + with: + repository: ${{ env.OSS_REPOSITORY }} + ref: ${{ env.OSS_REF }} + path: ${{ env.OSS_PATH }} + - uses: pnpm/action-setup@v4 + with: + package_json_file: oss/package.json + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: oss/pnpm-lock.yaml + - name: Install OSS dependencies + working-directory: oss + run: pnpm install --frozen-lockfile + - id: oss_sha + working-directory: oss + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - id: matrix + working-directory: oss + env: + PROVIDER_FILTER: ${{ github.event.inputs.provider || '' }} + BENCHMARK_FILTER: ${{ github.event.inputs.benchmark || 'all' }} + RUN_MODE: ${{ github.event_name == 'schedule' && 'schedule' || 'manual' }} + ITERATIONS_OVERRIDE: ${{ github.event.inputs.iterations || '' }} + SANDBOX_ITERATIONS: ${{ github.event.inputs.sandbox_iterations || '' }} + DAX_ITERATIONS: ${{ github.event.inputs.dax_iterations || '' }} + STORAGE_ITERATIONS: ${{ github.event.inputs.storage_iterations || '' }} + SNAPSHOT_FORK_ITERATIONS: ${{ github.event.inputs.snapshot_fork_iterations || '' }} + BROWSER_ITERATIONS: ${{ github.event.inputs.browser_iterations || '' }} + BROWSER_THROUGHPUT_ITERATIONS: ${{ github.event.inputs.browser_throughput_iterations || '' }} + AI_GATEWAY_ITERATIONS: ${{ github.event.inputs.ai_gateway_iterations || '' }} + run: | + include="$(pnpm exec tsx ../daily/scripts/build-daily-matrix.mjs)" + echo "include=$include" >> "$GITHUB_OUTPUT" + throughput_iterations="$(node -e "const x=JSON.parse(process.argv[1]); console.log(x.include.find(i => i.kind === 'browser-throughput')?.iterations || 0)" "$include")" + echo "throughput_iterations=$throughput_iterations" >> "$GITHUB_OUTPUT" + + throughput-pages: + name: Resolve throughput pages + needs: matrix + runs-on: ubuntu-latest + outputs: + urls: ${{ steps.pages.outputs.urls }} + steps: + - id: pages + run: | + count=${{ needs.matrix.outputs.throughput_iterations || '0' }} + if [ "$count" -eq 0 ]; then + echo 'urls<> "$GITHUB_OUTPUT" + echo 'URLS_EOF' >> "$GITHUB_OUTPUT" + exit 0 + fi + urls=() + for i in $(seq 1 "$count"); do + url=$(curl -sL -A 'ComputeSDK-Benchmark' -o /dev/null -w '%{url_effective}' \ + 'https://en.wikipedia.org/wiki/Special:Random') + urls+=("${url:-https://en.wikipedia.org/wiki/Special:Random}") + done + { + echo "urls<> "$GITHUB_OUTPUT" + + benchmark: + name: ${{ matrix.kind }} / ${{ matrix.provider }}${{ matrix.fileSize && format(' / {0}', matrix.fileSize) || '' }} + needs: [matrix, throughput-pages] + if: needs.matrix.result == 'success' + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.include) }} + steps: + - uses: actions/checkout@v4 + with: + path: daily + - uses: actions/checkout@v4 + with: + repository: ${{ env.OSS_REPOSITORY }} + ref: ${{ needs.matrix.outputs.oss_sha }} + path: ${{ env.OSS_PATH }} + - uses: pnpm/action-setup@v4 + with: + package_json_file: oss/package.json + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: oss/pnpm-lock.yaml + - name: Install OSS dependencies + working-directory: oss + run: pnpm install --frozen-lockfile + - name: Clear stale OSS results + working-directory: oss + run: rm -rf results/ + - name: Build vault key list from OSS provider manifest + working-directory: oss + run: | + case "${{ matrix.kind }}" in + dax) manifest_kind=sandbox ;; + browser-throughput) manifest_kind=throughput ;; + *) manifest_kind="${{ matrix.kind }}" ;; + esac + + if [ "${{ matrix.provider }}" = "all" ]; then + jq -r --arg kind "$manifest_kind" \ + '.[$kind] | to_entries[] | .value[]?' \ + benchmarks/scripts/provider-vars.json + else + jq -r --arg kind "$manifest_kind" --arg provider "${{ matrix.provider }}" \ + '.[$kind][$provider][]?' \ + benchmarks/scripts/provider-vars.json + fi | sort -u > /tmp/oss-vault-keys + printf '%s\n' COMPUTESDK_ADMIN_API_KEY BENCHMARKS_PLATFORM_API_KEY >> /tmp/oss-vault-keys + - name: Build E2B DAX template + if: matrix.kind == 'dax' && matrix.provider == 'e2b' + working-directory: oss + run: | + . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + npx tsx benchmarks/scripts/build-e2b-template.ts + - name: Build Superserve DAX template + if: matrix.kind == 'dax' && matrix.provider == 'superserve' + working-directory: oss + run: | + . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + npx tsx benchmarks/scripts/build-superserve-template.ts + - name: Resolve Northflank DAX plan + if: matrix.kind == 'dax' && matrix.provider == 'northflank' + working-directory: oss + run: | + . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + npx tsx benchmarks/scripts/find-northflank-plan.ts + - name: Run benchmark + working-directory: oss + env: + PROVIDER: ${{ matrix.provider }} + FILE_SIZE: ${{ matrix.fileSize }} + ITERATIONS: ${{ matrix.iterations }} + CONCURRENCY: '100' + DAX: ${{ matrix.kind == 'dax' && '1' || '' }} + THROUGHPUT_URLS: ${{ needs.throughput-pages.outputs.urls }} + DAILY_BENCH_SLUG: 'weekly' + DAILY_BENCH_NAME: 'Weekly' + run: | + . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + + RUNNER=(npx tsx packages/benchsdk-runner/dist/bin.js run) + # Matrix jobs for the same benchmark share one platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + case "${{ matrix.kind }}" in + sandbox) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + COMMON=(benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY") + "${RUNNER[@]}" "${COMMON[@]}" && + "${RUNNER[@]}" benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --run-key "$RUN_KEY" && + "${RUNNER[@]}" benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --concurrency "$CONCURRENCY" --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --run-key "$RUN_KEY" + ;; + dax) + export LIGHTNING_INSTANCE_TYPE=cpu-8 + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + "${RUNNER[@]}" benchmarks/sandbox/dax.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + storage) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FILE_SIZE}" + "${RUNNER[@]}" benchmarks/storage/storage.bench.ts --provider "$PROVIDER" --file-size "$FILE_SIZE" --concurrency 16 --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + snapshot-fork) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + "${RUNNER[@]}" benchmarks/storage/snapshot-fork.bench.ts --provider "$PROVIDER" --dataset small --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + browser) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + "${RUNNER[@]}" benchmarks/browser/browser.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + browser-throughput) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + "${RUNNER[@]}" benchmarks/browser/browser-throughput.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + ai-gateway) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + "${RUNNER[@]}" benchmarks/ai-gateway/ai-gateway.bench.ts --iterations "$ITERATIONS" --run-key "$RUN_KEY" + ;; + esac + - name: Upload raw results + if: always() + uses: actions/upload-artifact@v4 + with: + name: raw-${{ matrix.kind }}-${{ matrix.provider }}${{ matrix.fileSize && format('-{0}', matrix.fileSize) || '' }} + path: oss/results/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Merge, render, and publish + needs: [matrix, benchmark] + if: always() && needs.matrix.result == 'success' + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/checkout@v4 + with: + repository: ${{ env.OSS_REPOSITORY }} + ref: ${{ needs.matrix.outputs.oss_sha }} + path: ${{ env.OSS_PATH }} + - uses: pnpm/action-setup@v4 + with: + package_json_file: oss/package.json + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: oss/pnpm-lock.yaml + - name: Install OSS dependencies + working-directory: oss + run: pnpm install --frozen-lockfile + - name: Clear checked-in OSS results + working-directory: oss + run: rm -rf results artifacts + - name: Download raw results + uses: actions/download-artifact@v4 + with: + path: oss/artifacts + pattern: raw-* + - name: Partition raw results by category + working-directory: oss + run: | + set -e + mkdir -p artifacts/by-category/{sandbox,dax,storage,snapshot-fork,browser,browser-throughput,ai-gateway} + for artifact in artifacts/raw-*; do + [ -d "$artifact" ] || continue + case "$(basename "$artifact")" in + raw-sandbox-*) target=artifacts/by-category/sandbox ;; + raw-dax-*) target=artifacts/by-category/dax ;; + raw-storage-*) target=artifacts/by-category/storage ;; + raw-snapshot-fork-*) target=artifacts/by-category/snapshot-fork ;; + raw-browser-throughput-*) target=artifacts/by-category/browser-throughput ;; + raw-browser-*) target=artifacts/by-category/browser ;; + raw-ai-gateway-*) target=artifacts/by-category/ai-gateway ;; + *) echo "Unexpected artifact: $artifact" >&2; exit 1 ;; + esac + cp -a "$artifact" "$target/" + done + - name: Merge OSS results + working-directory: oss + run: | + set -e + run_merge() { + input="$1" + shift + if find "$input" -name latest.json -print -quit | grep -q .; then + npx tsx benchmarks/src/merge-results.ts --input "$input" "$@" + else + echo "No results found for $input; skipping" + fi + } + case "${{ github.event.inputs.benchmark || 'all' }}" in + all) + run_merge artifacts/by-category/sandbox + run_merge artifacts/by-category/dax + run_merge artifacts/by-category/storage --mode storage + run_merge artifacts/by-category/snapshot-fork --mode snapshot-fork + run_merge artifacts/by-category/browser --mode browser + run_merge artifacts/by-category/browser-throughput --mode browser-throughput + run_merge artifacts/by-category/ai-gateway --mode ai-gateway + ;; + sandbox|dax) + run_merge "artifacts/by-category/${{ github.event.inputs.benchmark || 'sandbox' }}" + ;; + storage) + run_merge artifacts/by-category/storage --mode storage + ;; + snapshot-fork) + run_merge artifacts/by-category/snapshot-fork --mode snapshot-fork + ;; + browser) + run_merge artifacts/by-category/browser --mode browser + ;; + browser-throughput) + run_merge artifacts/by-category/browser-throughput --mode browser-throughput + ;; + ai-gateway) + run_merge artifacts/by-category/ai-gateway --mode ai-gateway + ;; + esac + - name: Generate OSS charts + working-directory: oss + run: | + set -e + run_chart_if() { + input="$1" + shift + if [ -e "$input" ] || find "$input" -name latest.json -print -quit 2>/dev/null | grep -q .; then + "$@" + else + echo "No results found for $input; skipping chart" + fi + } + run_chart_if results/sequential_tti/latest.json pnpm run generate-svg + run_chart_if results/sequential_tti/latest.json pnpm run generate-pricing-svg + run_chart_if results/sandbox-dax/latest.json pnpm run generate-dax-svg + run_chart_if results/storage pnpm run generate-storage-svg + run_chart_if results/snapshot-fork pnpm run generate-snapshot-fork-svg + run_chart_if results/browser/latest.json pnpm run generate-browser-svg + run_chart_if results/browser-throughput/latest.json pnpm run generate-browser-throughput-svg + run_chart_if results/ai-gateway/latest.json pnpm run generate-ai-gateway-svg + - name: Copy generated results into repository + run: | + copy_tree_if() { + source="$1" + destination="$2" + if [ -d "$source" ]; then + mkdir -p "$destination" + cp -a "$source/." "$destination/" + else + echo "No directory found at $source; skipping" + fi + } + copy_file_if() { + source="$1" + dest="${2:-$(basename "$source")}" + if [ -f "$source" ]; then + cp "$source" "$dest" + else + echo "No file found at $source; skipping" + fi + } + copy_glob_if() { + directory="$1" + pattern="$2" + prefix="${3:-}" + find "$directory" -maxdepth 1 -type f -name "$pattern" -print0 | while IFS= read -r -d '' file; do + cp "$file" "./${prefix}$(basename "$file")" + done + } + case "${{ github.event.inputs.benchmark || 'all' }}" in + all) + copy_tree_if oss/results results-weekly + copy_glob_if oss '*.svg' 'weekly-' + ;; + sandbox) + copy_tree_if oss/results/sequential_tti results-weekly/sequential_tti + copy_tree_if oss/results/staggered_tti results-weekly/staggered_tti + copy_tree_if oss/results/burst_tti results-weekly/burst_tti + copy_file_if oss/results.svg weekly-results.svg + copy_glob_if oss '*_tti.svg' 'weekly-' + copy_file_if oss/pricing.svg weekly-pricing.svg + ;; + dax) + copy_tree_if oss/results/sandbox-dax results-weekly/sandbox-dax + copy_file_if oss/dax.svg weekly-dax.svg + ;; + storage) + copy_tree_if oss/results/storage results-weekly/storage + copy_glob_if oss 'storage_*.svg' 'weekly-' + ;; + snapshot-fork) + copy_tree_if oss/results/snapshot-fork results-weekly/snapshot-fork + copy_glob_if oss 'snapshot_fork_*.svg' 'weekly-' + ;; + browser) + copy_tree_if oss/results/browser results-weekly/browser + copy_file_if oss/browser.svg weekly-browser.svg + ;; + browser-throughput) + copy_tree_if oss/results/browser-throughput results-weekly/browser-throughput + copy_file_if oss/browser-throughput.svg weekly-browser-throughput.svg + ;; + ai-gateway) + copy_tree_if oss/results/ai-gateway results-weekly/ai-gateway + copy_file_if oss/ai-gateway.svg weekly-ai-gateway.svg + ;; + esac + printf '%s\n' "${{ needs.matrix.outputs.oss_sha }}" > .oss-benchmark-revision-weekly + - name: Ingest results + if: github.event.inputs.dry_run != 'true' + continue-on-error: true + run: | + . benchmarks/scripts/load-vault-secrets.sh '^(INGEST_URL|INGEST_SECRET)' + case "${{ github.event.inputs.benchmark || 'all' }}" in + all) + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type sandbox --dir results-weekly + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type storage --dir results-weekly + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type browser --dir results-weekly + ;; + sandbox|dax) + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type sandbox --dir results-weekly + ;; + storage) + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type storage --dir results-weekly + ;; + browser) + pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type browser --dir results-weekly + ;; + esac + - name: Commit and push weekly results + if: github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + add_if_exists() { + for f in "$@"; do + if [ -e "$f" ]; then + git add -- "$f" + fi + done + return 0 + } + add_if_exists .oss-benchmark-revision-weekly results-weekly/ weekly-*.svg + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update weekly OSS benchmark results [skip ci]" + for attempt in 1 2 3 4 5; do + git fetch origin master + git rebase origin/master + if git push origin HEAD:master; then + exit 0 + fi + done + exit 1 diff --git a/benchmarks/src/ingest.ts b/benchmarks/src/ingest.ts new file mode 100644 index 00000000..6d656fe9 --- /dev/null +++ b/benchmarks/src/ingest.ts @@ -0,0 +1,220 @@ +/** + * Transforms merged benchmark results into the platform ingest API format and POSTs them. + * + * Usage: tsx src/ingest.ts --type sandbox|storage|browser [--dir results] + * Env: INGEST_URL, INGEST_SECRET, GITHUB_SHA, GITHUB_REF, GITHUB_EVENT_NAME + */ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, '../..'); + +const INGEST_URL = process.env.INGEST_URL; +const INGEST_SECRET = process.env.INGEST_SECRET; + +if (!INGEST_URL || !INGEST_SECRET) { + console.error('INGEST_URL and INGEST_SECRET are required'); + process.exit(1); +} + +const args = process.argv.slice(2); +const typeArg = args[args.indexOf('--type') + 1] as 'sandbox' | 'storage' | 'browser' | undefined; +if (!typeArg || !['sandbox', 'storage', 'browser'].includes(typeArg)) { + console.error('Usage: tsx src/ingest.ts --type sandbox|storage|browser [--dir results]'); + process.exit(1); +} + +const dirFlagIndex = args.indexOf('--dir'); +const RESULTS_DIR = (dirFlagIndex >= 0 ? args[dirFlagIndex + 1] : undefined) || 'results'; + +const triggeredBy = + process.env.GITHUB_EVENT_NAME === 'schedule' ? 'scheduled' : + process.env.GITHUB_EVENT_NAME === 'pull_request' ? 'pr' : 'manual'; + +async function post(body: unknown, label: string) { + const res = await fetch(INGEST_URL!, { + method: 'POST', + headers: { + Authorization: `Bearer ${INGEST_SECRET}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Ingest failed for ${label}: ${res.status} ${text}`); + } + + const { runId } = await res.json() as { runId: string }; + console.log(`Ingested ${label} → runId=${runId}`); +} + +// --------------------------------------------------------------------------- +// Sandbox +// --------------------------------------------------------------------------- + +const SANDBOX_MODES = [ + { dir: 'sequential_tti', mode: 'sequential' }, + { dir: 'staggered_tti', mode: 'staggered' }, + { dir: 'burst_tti', mode: 'burst' }, +]; + +async function ingestSandbox() { + for (const { dir, mode } of SANDBOX_MODES) { + const latestPath = path.join(ROOT, RESULTS_DIR, dir, 'latest.json'); + if (!fs.existsSync(latestPath)) { + console.log(`Skipping sandbox/${mode}: ${latestPath} not found`); + continue; + } + + const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + + await post({ + run: { + benchmarkType: 'sandbox', + gitSha: process.env.GITHUB_SHA, + gitRef: process.env.GITHUB_REF, + triggeredBy, + environment: raw.environment, + }, + results: raw.results.map((r: any) => { + const dimensions: Record = { mode }; + if (r.concurrency != null) dimensions.concurrency = r.concurrency; + + const scalars = []; + if (r.wallClockMs != null) + scalars.push({ name: 'wall_clock_ms', value: r.wallClockMs, unit: 'ms' }); + if (r.timeToFirstReadyMs != null) + scalars.push({ name: 'time_to_first_ready_ms', value: r.timeToFirstReadyMs, unit: 'ms' }); + + return { + provider: r.provider, + dimensions, + iterations: r.iterations, + metrics: [{ + name: 'tti', unit: 'ms', + median: r.summary.ttiMs.median, + p95: r.summary.ttiMs.p95, + p99: r.summary.ttiMs.p99, + }], + scalars, + compositeScore: r.compositeScore, + scoringVersion: 'sandbox_v1', + successRate: r.successRate, + skipped: r.skipped ?? false, + skipReason: r.skipReason, + }; + }), + }, `sandbox/${mode}`); + } +} + +// --------------------------------------------------------------------------- +// Storage +// --------------------------------------------------------------------------- + +async function ingestStorage() { + const storageDir = path.join(ROOT, RESULTS_DIR, 'storage'); + if (!fs.existsSync(storageDir)) { + console.log(`Skipping storage: ${RESULTS_DIR}/storage/ not found`); + return; + } + + // Each subdirectory is a file size (e.g. 1mb, 4mb, 10mb, 16mb) + const sizeDirs = fs.readdirSync(storageDir, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name); + + for (const sizeDir of sizeDirs) { + const latestPath = path.join(storageDir, sizeDir, 'latest.json'); + if (!fs.existsSync(latestPath)) continue; + + const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + // Normalise dir name to match API convention: "10mb" → "10MB" + const fileSize = sizeDir.toUpperCase(); + + await post({ + run: { + benchmarkType: 'storage', + gitSha: process.env.GITHUB_SHA, + gitRef: process.env.GITHUB_REF, + triggeredBy, + environment: raw.environment, + }, + results: raw.results.map((r: any) => ({ + provider: r.provider, + dimensions: { file_size: fileSize }, + iterations: r.iterations, + metrics: [ + { name: 'upload', unit: 'ms', median: r.summary.uploadMs.median, p95: r.summary.uploadMs.p95, p99: r.summary.uploadMs.p99 }, + { name: 'download', unit: 'ms', median: r.summary.downloadMs.median, p95: r.summary.downloadMs.p95, p99: r.summary.downloadMs.p99 }, + { name: 'throughput', unit: 'mbps', median: r.summary.throughputMbps.median, p95: r.summary.throughputMbps.p95, p99: r.summary.throughputMbps.p99 }, + ], + scalars: [ + { name: 'file_size_bytes', value: r.fileSizeBytes, unit: 'bytes' }, + ], + compositeScore: r.compositeScore, + scoringVersion: 'storage_v1', + successRate: r.successRate, + skipped: r.skipped ?? false, + skipReason: r.skipReason, + })), + }, `storage/${fileSize}`); + } +} + +// --------------------------------------------------------------------------- +// Browser +// --------------------------------------------------------------------------- + +async function ingestBrowser() { + const latestPath = path.join(ROOT, RESULTS_DIR, 'browser', 'latest.json'); + if (!fs.existsSync(latestPath)) { + console.log('Skipping browser: results/browser/latest.json not found'); + return; + } + + const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + + await post({ + run: { + benchmarkType: 'browser', + gitSha: process.env.GITHUB_SHA, + gitRef: process.env.GITHUB_REF, + triggeredBy, + environment: raw.environment, + }, + results: raw.results.map((r: any) => ({ + provider: r.provider, + dimensions: {}, + iterations: r.iterations, + metrics: [ + { name: 'create', unit: 'ms', median: r.summary.createMs.median, p95: r.summary.createMs.p95, p99: r.summary.createMs.p99 }, + { name: 'connect', unit: 'ms', median: r.summary.connectMs.median, p95: r.summary.connectMs.p95, p99: r.summary.connectMs.p99 }, + { name: 'navigate', unit: 'ms', median: r.summary.navigateMs.median, p95: r.summary.navigateMs.p95, p99: r.summary.navigateMs.p99 }, + { name: 'release', unit: 'ms', median: r.summary.releaseMs.median, p95: r.summary.releaseMs.p95, p99: r.summary.releaseMs.p99 }, + { name: 'total', unit: 'ms', median: r.summary.totalMs.median, p95: r.summary.totalMs.p95, p99: r.summary.totalMs.p99 }, + ], + scalars: [], + compositeScore: r.compositeScore, + scoringVersion: 'browser_v1', + successRate: r.successRate, + skipped: r.skipped ?? false, + skipReason: r.skipReason, + })), + }, 'browser'); +} + +// --------------------------------------------------------------------------- +// Run +// --------------------------------------------------------------------------- + +const runners = { sandbox: ingestSandbox, storage: ingestStorage, browser: ingestBrowser }; + +runners[typeArg]().catch(err => { + console.error('Ingest failed:', err); + process.exit(1); +}); diff --git a/scripts/build-daily-matrix.mjs b/scripts/build-daily-matrix.mjs new file mode 100644 index 00000000..2b5f7f96 --- /dev/null +++ b/scripts/build-daily-matrix.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { resolveWorkflowDefaults } from './oss-workflow-defaults.mjs'; + +const providerFilter = process.env.PROVIDER_FILTER || ''; +const benchmarkFilter = process.env.BENCHMARK_FILTER || 'all'; +const runMode = process.env.RUN_MODE || 'manual'; +const iterationOverride = process.env.ITERATIONS_OVERRIDE || ''; +const categoryOverrides = { + sandbox: process.env.SANDBOX_ITERATIONS || '', + dax: process.env.DAX_ITERATIONS || '', + storage: process.env.STORAGE_ITERATIONS || '', + 'snapshot-fork': process.env.SNAPSHOT_FORK_ITERATIONS || '', + browser: process.env.BROWSER_ITERATIONS || '', + 'browser-throughput': process.env.BROWSER_THROUGHPUT_ITERATIONS || '', + 'ai-gateway': process.env.AI_GATEWAY_ITERATIONS || '', +}; + +function selected(kind) { + return benchmarkFilter === 'all' || benchmarkFilter === kind; +} + +function providerSelected(name) { + return !providerFilter || providerFilter === name; +} + +function iterationsFor(kind) { + if (runMode === 'manual') { + const override = categoryOverrides[kind] || iterationOverride; + if (override) return override === '0' ? null : override; + return resolveWorkflowDefaults(process.cwd(), kind).manual; + } + return resolveWorkflowDefaults(process.cwd(), kind)[runMode]; +} + +async function load(modulePath, exportName) { + const moduleNamespace = await import(pathToFileURL(path.resolve(process.cwd(), modulePath))); + const providers = moduleNamespace[exportName]; + if (!Array.isArray(providers)) { + throw new Error(`Expected ${exportName} to be an array in ${modulePath}`); + } + return providers; +} + +const include = []; +function add(kind, providers, extra = {}) { + if (!selected(kind)) return; + const iterations = iterationsFor(kind); + if (iterations === null) return; + for (const provider of providers) { + if (providerSelected(provider.name)) { + include.push({ kind, provider: provider.name, iterations, ...extra }); + } + } +} + +const sandboxProviders = await load('./benchmarks/sandbox/providers.ts', 'providers'); +add('sandbox', sandboxProviders); +add('dax', sandboxProviders); + +const storageProviders = await load('./benchmarks/storage/providers.ts', 'storageProviders'); +for (const size of ['1MB', '4MB', '10MB', '16MB']) { + add('storage', storageProviders, { fileSize: size }); +} +add('snapshot-fork', storageProviders.filter((provider) => provider.snapshotFork)); + +const browserProviders = await load('./benchmarks/browser/providers.ts', 'browserProviders'); +add('browser', browserProviders); + +const throughputProviders = await load( + './benchmarks/browser/throughput-providers.ts', + 'throughputProviders', +); +add('browser-throughput', throughputProviders); + +const aiGatewayProviders = await load('./benchmarks/ai-gateway/providers.ts', 'providers'); +if ( + selected('ai-gateway') && + aiGatewayProviders.some((provider) => providerSelected(provider.name)) +) { + const iterations = iterationsFor('ai-gateway'); + if (iterations !== null) { + include.push({ kind: 'ai-gateway', provider: 'all', iterations }); + } +} + +process.stdout.write(`${JSON.stringify({ include })}\n`); diff --git a/scripts/oss-workflow-defaults.mjs b/scripts/oss-workflow-defaults.mjs new file mode 100644 index 00000000..24e617c4 --- /dev/null +++ b/scripts/oss-workflow-defaults.mjs @@ -0,0 +1,75 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const WORKFLOWS = { + sandbox: 'sandbox-tti-benchmarks.yml', + dax: 'sandbox-dax-benchmarks.yml', + storage: 'storage-benchmarks.yml', + 'snapshot-fork': 'snapshot-fork-benchmarks.yml', + browser: 'browser-benchmarks.yml', + 'browser-throughput': 'browser-throughput-benchmarks.yml', + 'ai-gateway': 'ai-gateway-benchmarks.yml', +}; + +const FALLBACK_DEFAULTS = { + // ai-gateway uses a shell-computed ITERATIONS value and an empty manual + // default that resolves to 30 for non-schedule runs. + 'ai-gateway': { manual: '30', schedule: '10' }, +}; + +function workflowInputDefault(source) { + const workflowDispatch = source.match( + /^ workflow_dispatch:\n([\s\S]*?)(?=^ [A-Za-z0-9_-]+:|(?![\s\S]))/m, + )?.[1]; + if (!workflowDispatch) throw new Error('workflow_dispatch section not found'); + + const iterations = workflowDispatch.match( + /^\s{6}iterations:\n([\s\S]*?)(?=^\s{6}[A-Za-z0-9_-]+:|(?![\s\S]))/m, + )?.[1]; + const value = iterations?.match(/^\s{8}default:\s*['"]?([^'"\n]*)['"]?\s*$/m)?.[1]; + if (value === undefined) throw new Error('workflow_dispatch iterations default not found'); + return value.trim(); +} + +function scheduledDefault(source) { + const expressions = [...source.matchAll(/--iterations\s+\$\{\{([\s\S]*?)\}\}/g)]; + if (expressions.length === 0) throw new Error('benchmark iterations expression not found'); + + const expression = expressions[expressions.length - 1][1]; + const scheduled = expression.match( + /github\.event_name\s*==\s*['"]schedule['"]\s*&&\s*['"](\d+)['"]/, + )?.[1]; + if (scheduled) return scheduled; + + const fallback = [...expression.matchAll(/\|\|\s*['"](\d+)['"]/g)].at(-1)?.[1]; + if (!fallback) throw new Error('scheduled iterations fallback not found'); + return fallback; +} + +export function resolveWorkflowDefaults(ossRoot, kind) { + const workflow = WORKFLOWS[kind]; + if (!workflow) throw new Error(`Unknown benchmark kind: ${kind}`); + + const source = fs.readFileSync( + path.join(ossRoot, '.github', 'workflows', workflow), + 'utf8', + ); + try { + return { + manual: workflowInputDefault(source), + schedule: scheduledDefault(source), + }; + } catch (err) { + if (FALLBACK_DEFAULTS[kind]) return FALLBACK_DEFAULTS[kind]; + throw err; + } +} + +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + const root = process.argv[process.argv.indexOf('--root') + 1] || process.cwd(); + const kind = process.argv[process.argv.indexOf('--kind') + 1]; + const runMode = process.argv[process.argv.indexOf('--run-mode') + 1] || 'manual'; + const defaults = resolveWorkflowDefaults(root, kind); + process.stdout.write(`${defaults[runMode]}\n`); +} From 14d958c44e8b6357d131938e027a2b29830e98c4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:57:03 +0000 Subject: [PATCH 2/4] fix(weekly-workflow): scope permissions and avoid shell interpolation Co-Authored-By: Noah Kiser --- .github/workflows/weekly-oss-benchmarks.yml | 37 ++++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/weekly-oss-benchmarks.yml b/.github/workflows/weekly-oss-benchmarks.yml index d8bc784f..8156457c 100644 --- a/.github/workflows/weekly-oss-benchmarks.yml +++ b/.github/workflows/weekly-oss-benchmarks.yml @@ -65,8 +65,7 @@ concurrency: group: weekly-oss-benchmarks cancel-in-progress: false -permissions: - contents: write +permissions: {} env: OSS_REPOSITORY: computesdk/benchmarks @@ -77,6 +76,8 @@ jobs: matrix: name: Discover OSS providers runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + permissions: + contents: read outputs: include: ${{ steps.matrix.outputs.include }} oss_sha: ${{ steps.oss_sha.outputs.sha }} @@ -128,6 +129,7 @@ jobs: name: Resolve throughput pages needs: matrix runs-on: ubuntu-latest + permissions: {} outputs: urls: ${{ steps.pages.outputs.urls }} steps: @@ -156,7 +158,13 @@ jobs: needs: [matrix, throughput-pages] if: needs.matrix.result == 'success' runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + permissions: + contents: read timeout-minutes: 90 + env: + BENCH_KIND: ${{ matrix.kind }} + BENCH_PROVIDER: ${{ matrix.provider }} + BENCH_FILE_SIZE: ${{ matrix.fileSize }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.matrix.outputs.include) }} @@ -186,18 +194,18 @@ jobs: - name: Build vault key list from OSS provider manifest working-directory: oss run: | - case "${{ matrix.kind }}" in + case "$BENCH_KIND" in dax) manifest_kind=sandbox ;; browser-throughput) manifest_kind=throughput ;; - *) manifest_kind="${{ matrix.kind }}" ;; + *) manifest_kind="$BENCH_KIND" ;; esac - if [ "${{ matrix.provider }}" = "all" ]; then + if [ "$BENCH_PROVIDER" = "all" ]; then jq -r --arg kind "$manifest_kind" \ '.[$kind] | to_entries[] | .value[]?' \ benchmarks/scripts/provider-vars.json else - jq -r --arg kind "$manifest_kind" --arg provider "${{ matrix.provider }}" \ + jq -r --arg kind "$manifest_kind" --arg provider "$BENCH_PROVIDER" \ '.[$kind][$provider][]?' \ benchmarks/scripts/provider-vars.json fi | sort -u > /tmp/oss-vault-keys @@ -237,7 +245,7 @@ jobs: RUNNER=(npx tsx packages/benchsdk-runner/dist/bin.js run) # Matrix jobs for the same benchmark share one platform run. The # attempt number is included so re-runs do not rejoin completed runs. - case "${{ matrix.kind }}" in + case "$BENCH_KIND" in sandbox) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" COMMON=(benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY") @@ -285,6 +293,11 @@ jobs: needs: [matrix, benchmark] if: always() && needs.matrix.result == 'success' runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + permissions: + contents: write + env: + BENCHMARK_FILTER: ${{ github.event.inputs.benchmark || 'all' }} + OSS_SHA: ${{ needs.matrix.outputs.oss_sha }} steps: - uses: actions/checkout@v4 with: @@ -345,7 +358,7 @@ jobs: echo "No results found for $input; skipping" fi } - case "${{ github.event.inputs.benchmark || 'all' }}" in + case "$BENCHMARK_FILTER" in all) run_merge artifacts/by-category/sandbox run_merge artifacts/by-category/dax @@ -356,7 +369,7 @@ jobs: run_merge artifacts/by-category/ai-gateway --mode ai-gateway ;; sandbox|dax) - run_merge "artifacts/by-category/${{ github.event.inputs.benchmark || 'sandbox' }}" + run_merge "artifacts/by-category/$BENCHMARK_FILTER" ;; storage) run_merge artifacts/by-category/storage --mode storage @@ -424,7 +437,7 @@ jobs: cp "$file" "./${prefix}$(basename "$file")" done } - case "${{ github.event.inputs.benchmark || 'all' }}" in + case "$BENCHMARK_FILTER" in all) copy_tree_if oss/results results-weekly copy_glob_if oss '*.svg' 'weekly-' @@ -462,13 +475,13 @@ jobs: copy_file_if oss/ai-gateway.svg weekly-ai-gateway.svg ;; esac - printf '%s\n' "${{ needs.matrix.outputs.oss_sha }}" > .oss-benchmark-revision-weekly + printf '%s\n' "$OSS_SHA" > .oss-benchmark-revision-weekly - name: Ingest results if: github.event.inputs.dry_run != 'true' continue-on-error: true run: | . benchmarks/scripts/load-vault-secrets.sh '^(INGEST_URL|INGEST_SECRET)' - case "${{ github.event.inputs.benchmark || 'all' }}" in + case "$BENCHMARK_FILTER" in all) pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type sandbox --dir results-weekly pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type storage --dir results-weekly From f0beaab8268c6a75c3d12fb1c2210707a83fe8b8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:17:30 +0000 Subject: [PATCH 3/4] fix weekly workflow: remove legacy ingest, burst-only sandbox, correct AI gateway/tts paths, vault regex, nscloud setup, flag parsing Co-Authored-By: Noah Kiser --- .github/workflows/weekly-oss-benchmarks.yml | 68 +++--- benchmarks/sandbox/generate-pricing-svg.ts | 6 +- benchmarks/src/ingest.ts | 220 -------------------- scripts/oss-workflow-defaults.mjs | 10 +- 4 files changed, 34 insertions(+), 270 deletions(-) delete mode 100644 benchmarks/src/ingest.ts diff --git a/.github/workflows/weekly-oss-benchmarks.yml b/.github/workflows/weekly-oss-benchmarks.yml index 8156457c..3f5e0607 100644 --- a/.github/workflows/weekly-oss-benchmarks.yml +++ b/.github/workflows/weekly-oss-benchmarks.yml @@ -56,7 +56,7 @@ on: required: false default: '' dry_run: - description: 'Run without ingesting or committing results' + description: 'Run without committing results' required: false default: false type: boolean @@ -160,6 +160,7 @@ jobs: runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe permissions: contents: read + id-token: write timeout-minutes: 90 env: BENCH_KIND: ${{ matrix.kind }} @@ -191,6 +192,7 @@ jobs: - name: Clear stale OSS results working-directory: oss run: rm -rf results/ + - uses: namespacelabs/nscloud-setup@v0 - name: Build vault key list from OSS provider manifest working-directory: oss run: | @@ -208,25 +210,28 @@ jobs: jq -r --arg kind "$manifest_kind" --arg provider "$BENCH_PROVIDER" \ '.[$kind][$provider][]?' \ benchmarks/scripts/provider-vars.json - fi | sort -u > /tmp/oss-vault-keys + fi | sort -u | grep -v '^$' > /tmp/oss-vault-keys printf '%s\n' COMPUTESDK_ADMIN_API_KEY BENCHMARKS_PLATFORM_API_KEY >> /tmp/oss-vault-keys + # Build a grep-compatible regex for load-vault-secrets.sh + regex="^($(paste -sd'|' /tmp/oss-vault-keys))" + echo "VAULT_KEYS_REGEX=$regex" >> "$GITHUB_ENV" - name: Build E2B DAX template if: matrix.kind == 'dax' && matrix.provider == 'e2b' working-directory: oss run: | - . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + . ../daily/benchmarks/scripts/load-vault-secrets.sh "$VAULT_KEYS_REGEX" npx tsx benchmarks/scripts/build-e2b-template.ts - name: Build Superserve DAX template if: matrix.kind == 'dax' && matrix.provider == 'superserve' working-directory: oss run: | - . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + . ../daily/benchmarks/scripts/load-vault-secrets.sh "$VAULT_KEYS_REGEX" npx tsx benchmarks/scripts/build-superserve-template.ts - name: Resolve Northflank DAX plan if: matrix.kind == 'dax' && matrix.provider == 'northflank' working-directory: oss run: | - . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + . ../daily/benchmarks/scripts/load-vault-secrets.sh "$VAULT_KEYS_REGEX" npx tsx benchmarks/scripts/find-northflank-plan.ts - name: Run benchmark working-directory: oss @@ -239,8 +244,9 @@ jobs: THROUGHPUT_URLS: ${{ needs.throughput-pages.outputs.urls }} DAILY_BENCH_SLUG: 'weekly' DAILY_BENCH_NAME: 'Weekly' + NO_INGEST: ${{ (github.event_name == 'push' || github.event.inputs.dry_run == 'true') && '--no-ingest' || '' }} run: | - . ../daily/benchmarks/scripts/load-vault-secrets.sh --keys-file /tmp/oss-vault-keys + . ../daily/benchmarks/scripts/load-vault-secrets.sh "$VAULT_KEYS_REGEX" RUNNER=(npx tsx packages/benchsdk-runner/dist/bin.js run) # Matrix jobs for the same benchmark share one platform run. The @@ -248,35 +254,32 @@ jobs: case "$BENCH_KIND" in sandbox) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - COMMON=(benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY") - "${RUNNER[@]}" "${COMMON[@]}" && - "${RUNNER[@]}" benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --run-key "$RUN_KEY" && - "${RUNNER[@]}" benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --concurrency "$CONCURRENCY" --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/sandbox/tti.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --concurrency "$CONCURRENCY" --run-key "$RUN_KEY" $NO_INGEST ;; dax) export LIGHTNING_INSTANCE_TYPE=cpu-8 RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - "${RUNNER[@]}" benchmarks/sandbox/dax.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/sandbox/dax.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; storage) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FILE_SIZE}" - "${RUNNER[@]}" benchmarks/storage/storage.bench.ts --provider "$PROVIDER" --file-size "$FILE_SIZE" --concurrency 16 --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/storage/storage.bench.ts --provider "$PROVIDER" --file-size "$FILE_SIZE" --concurrency 16 --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; snapshot-fork) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - "${RUNNER[@]}" benchmarks/storage/snapshot-fork.bench.ts --provider "$PROVIDER" --dataset small --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/storage/snapshot-fork.bench.ts --provider "$PROVIDER" --dataset small --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; browser) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - "${RUNNER[@]}" benchmarks/browser/browser.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/browser/browser.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; browser-throughput) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - "${RUNNER[@]}" benchmarks/browser/browser-throughput.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/browser/browser-throughput.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; ai-gateway) RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - "${RUNNER[@]}" benchmarks/ai-gateway/ai-gateway.bench.ts --iterations "$ITERATIONS" --run-key "$RUN_KEY" + "${RUNNER[@]}" benchmarks/ai-gateway/ai-gateway.bench.ts --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; esac - name: Upload raw results @@ -400,14 +403,14 @@ jobs: echo "No results found for $input; skipping chart" fi } - run_chart_if results/sequential_tti/latest.json pnpm run generate-svg - run_chart_if results/sequential_tti/latest.json pnpm run generate-pricing-svg + run_chart_if results/burst_tti/latest.json pnpm run generate-svg + run_chart_if results/burst_tti/latest.json pnpm run generate-pricing-svg run_chart_if results/sandbox-dax/latest.json pnpm run generate-dax-svg run_chart_if results/storage pnpm run generate-storage-svg run_chart_if results/snapshot-fork pnpm run generate-snapshot-fork-svg run_chart_if results/browser/latest.json pnpm run generate-browser-svg run_chart_if results/browser-throughput/latest.json pnpm run generate-browser-throughput-svg - run_chart_if results/ai-gateway/latest.json pnpm run generate-ai-gateway-svg + run_chart_if results/ai-gateway-latency/anthropic/latest.json pnpm run generate-ai-gateway-svg - name: Copy generated results into repository run: | copy_tree_if() { @@ -443,11 +446,9 @@ jobs: copy_glob_if oss '*.svg' 'weekly-' ;; sandbox) - copy_tree_if oss/results/sequential_tti results-weekly/sequential_tti - copy_tree_if oss/results/staggered_tti results-weekly/staggered_tti copy_tree_if oss/results/burst_tti results-weekly/burst_tti copy_file_if oss/results.svg weekly-results.svg - copy_glob_if oss '*_tti.svg' 'weekly-' + copy_file_if oss/burst_tti.svg weekly-burst_tti.svg copy_file_if oss/pricing.svg weekly-pricing.svg ;; dax) @@ -471,32 +472,11 @@ jobs: copy_file_if oss/browser-throughput.svg weekly-browser-throughput.svg ;; ai-gateway) - copy_tree_if oss/results/ai-gateway results-weekly/ai-gateway + copy_tree_if oss/results/ai-gateway-latency/anthropic results-weekly/ai-gateway-latency/anthropic copy_file_if oss/ai-gateway.svg weekly-ai-gateway.svg ;; esac printf '%s\n' "$OSS_SHA" > .oss-benchmark-revision-weekly - - name: Ingest results - if: github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(INGEST_URL|INGEST_SECRET)' - case "$BENCHMARK_FILTER" in - all) - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type sandbox --dir results-weekly - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type storage --dir results-weekly - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type browser --dir results-weekly - ;; - sandbox|dax) - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type sandbox --dir results-weekly - ;; - storage) - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type storage --dir results-weekly - ;; - browser) - pnpm --dir oss exec tsx ../benchmarks/src/ingest.ts --type browser --dir results-weekly - ;; - esac - name: Commit and push weekly results if: github.event.inputs.dry_run != 'true' run: | diff --git a/benchmarks/sandbox/generate-pricing-svg.ts b/benchmarks/sandbox/generate-pricing-svg.ts index 695ea0a6..f3f9c41b 100644 --- a/benchmarks/sandbox/generate-pricing-svg.ts +++ b/benchmarks/sandbox/generate-pricing-svg.ts @@ -144,11 +144,11 @@ function valueColorClass(score: number | null): string { } /** - * Load live benchmark scores from sequential results. + * Load live benchmark scores from burst results. * Returns a map of provider id -> { score, success_rate } or null if no results found. */ function loadLiveBenchmarkScores(): Map | null { - const latestPath = path.join(RESULTS_DIR, 'sequential_tti', 'latest.json'); + const latestPath = path.join(RESULTS_DIR, 'burst_tti', 'latest.json'); if (!fs.existsSync(latestPath)) return null; try { @@ -171,7 +171,7 @@ function loadLiveBenchmarkScores(): Map= 0 ? args[dirFlagIndex + 1] : undefined) || 'results'; - -const triggeredBy = - process.env.GITHUB_EVENT_NAME === 'schedule' ? 'scheduled' : - process.env.GITHUB_EVENT_NAME === 'pull_request' ? 'pr' : 'manual'; - -async function post(body: unknown, label: string) { - const res = await fetch(INGEST_URL!, { - method: 'POST', - headers: { - Authorization: `Bearer ${INGEST_SECRET}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }); - - if (!res.ok) { - const text = await res.text(); - throw new Error(`Ingest failed for ${label}: ${res.status} ${text}`); - } - - const { runId } = await res.json() as { runId: string }; - console.log(`Ingested ${label} → runId=${runId}`); -} - -// --------------------------------------------------------------------------- -// Sandbox -// --------------------------------------------------------------------------- - -const SANDBOX_MODES = [ - { dir: 'sequential_tti', mode: 'sequential' }, - { dir: 'staggered_tti', mode: 'staggered' }, - { dir: 'burst_tti', mode: 'burst' }, -]; - -async function ingestSandbox() { - for (const { dir, mode } of SANDBOX_MODES) { - const latestPath = path.join(ROOT, RESULTS_DIR, dir, 'latest.json'); - if (!fs.existsSync(latestPath)) { - console.log(`Skipping sandbox/${mode}: ${latestPath} not found`); - continue; - } - - const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); - - await post({ - run: { - benchmarkType: 'sandbox', - gitSha: process.env.GITHUB_SHA, - gitRef: process.env.GITHUB_REF, - triggeredBy, - environment: raw.environment, - }, - results: raw.results.map((r: any) => { - const dimensions: Record = { mode }; - if (r.concurrency != null) dimensions.concurrency = r.concurrency; - - const scalars = []; - if (r.wallClockMs != null) - scalars.push({ name: 'wall_clock_ms', value: r.wallClockMs, unit: 'ms' }); - if (r.timeToFirstReadyMs != null) - scalars.push({ name: 'time_to_first_ready_ms', value: r.timeToFirstReadyMs, unit: 'ms' }); - - return { - provider: r.provider, - dimensions, - iterations: r.iterations, - metrics: [{ - name: 'tti', unit: 'ms', - median: r.summary.ttiMs.median, - p95: r.summary.ttiMs.p95, - p99: r.summary.ttiMs.p99, - }], - scalars, - compositeScore: r.compositeScore, - scoringVersion: 'sandbox_v1', - successRate: r.successRate, - skipped: r.skipped ?? false, - skipReason: r.skipReason, - }; - }), - }, `sandbox/${mode}`); - } -} - -// --------------------------------------------------------------------------- -// Storage -// --------------------------------------------------------------------------- - -async function ingestStorage() { - const storageDir = path.join(ROOT, RESULTS_DIR, 'storage'); - if (!fs.existsSync(storageDir)) { - console.log(`Skipping storage: ${RESULTS_DIR}/storage/ not found`); - return; - } - - // Each subdirectory is a file size (e.g. 1mb, 4mb, 10mb, 16mb) - const sizeDirs = fs.readdirSync(storageDir, { withFileTypes: true }) - .filter(e => e.isDirectory()) - .map(e => e.name); - - for (const sizeDir of sizeDirs) { - const latestPath = path.join(storageDir, sizeDir, 'latest.json'); - if (!fs.existsSync(latestPath)) continue; - - const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); - // Normalise dir name to match API convention: "10mb" → "10MB" - const fileSize = sizeDir.toUpperCase(); - - await post({ - run: { - benchmarkType: 'storage', - gitSha: process.env.GITHUB_SHA, - gitRef: process.env.GITHUB_REF, - triggeredBy, - environment: raw.environment, - }, - results: raw.results.map((r: any) => ({ - provider: r.provider, - dimensions: { file_size: fileSize }, - iterations: r.iterations, - metrics: [ - { name: 'upload', unit: 'ms', median: r.summary.uploadMs.median, p95: r.summary.uploadMs.p95, p99: r.summary.uploadMs.p99 }, - { name: 'download', unit: 'ms', median: r.summary.downloadMs.median, p95: r.summary.downloadMs.p95, p99: r.summary.downloadMs.p99 }, - { name: 'throughput', unit: 'mbps', median: r.summary.throughputMbps.median, p95: r.summary.throughputMbps.p95, p99: r.summary.throughputMbps.p99 }, - ], - scalars: [ - { name: 'file_size_bytes', value: r.fileSizeBytes, unit: 'bytes' }, - ], - compositeScore: r.compositeScore, - scoringVersion: 'storage_v1', - successRate: r.successRate, - skipped: r.skipped ?? false, - skipReason: r.skipReason, - })), - }, `storage/${fileSize}`); - } -} - -// --------------------------------------------------------------------------- -// Browser -// --------------------------------------------------------------------------- - -async function ingestBrowser() { - const latestPath = path.join(ROOT, RESULTS_DIR, 'browser', 'latest.json'); - if (!fs.existsSync(latestPath)) { - console.log('Skipping browser: results/browser/latest.json not found'); - return; - } - - const raw = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); - - await post({ - run: { - benchmarkType: 'browser', - gitSha: process.env.GITHUB_SHA, - gitRef: process.env.GITHUB_REF, - triggeredBy, - environment: raw.environment, - }, - results: raw.results.map((r: any) => ({ - provider: r.provider, - dimensions: {}, - iterations: r.iterations, - metrics: [ - { name: 'create', unit: 'ms', median: r.summary.createMs.median, p95: r.summary.createMs.p95, p99: r.summary.createMs.p99 }, - { name: 'connect', unit: 'ms', median: r.summary.connectMs.median, p95: r.summary.connectMs.p95, p99: r.summary.connectMs.p99 }, - { name: 'navigate', unit: 'ms', median: r.summary.navigateMs.median, p95: r.summary.navigateMs.p95, p99: r.summary.navigateMs.p99 }, - { name: 'release', unit: 'ms', median: r.summary.releaseMs.median, p95: r.summary.releaseMs.p95, p99: r.summary.releaseMs.p99 }, - { name: 'total', unit: 'ms', median: r.summary.totalMs.median, p95: r.summary.totalMs.p95, p99: r.summary.totalMs.p99 }, - ], - scalars: [], - compositeScore: r.compositeScore, - scoringVersion: 'browser_v1', - successRate: r.successRate, - skipped: r.skipped ?? false, - skipReason: r.skipReason, - })), - }, 'browser'); -} - -// --------------------------------------------------------------------------- -// Run -// --------------------------------------------------------------------------- - -const runners = { sandbox: ingestSandbox, storage: ingestStorage, browser: ingestBrowser }; - -runners[typeArg]().catch(err => { - console.error('Ingest failed:', err); - process.exit(1); -}); diff --git a/scripts/oss-workflow-defaults.mjs b/scripts/oss-workflow-defaults.mjs index 24e617c4..12574146 100644 --- a/scripts/oss-workflow-defaults.mjs +++ b/scripts/oss-workflow-defaults.mjs @@ -67,9 +67,13 @@ export function resolveWorkflowDefaults(ossRoot, kind) { } if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { - const root = process.argv[process.argv.indexOf('--root') + 1] || process.cwd(); - const kind = process.argv[process.argv.indexOf('--kind') + 1]; - const runMode = process.argv[process.argv.indexOf('--run-mode') + 1] || 'manual'; + const flag = (name) => { + const i = process.argv.indexOf(name); + return i >= 0 ? process.argv[i + 1] : undefined; + }; + const root = flag('--root') || process.cwd(); + const kind = flag('--kind'); + const runMode = flag('--run-mode') || 'manual'; const defaults = resolveWorkflowDefaults(root, kind); process.stdout.write(`${defaults[runMode]}\n`); } From 6ef4e8a2d096cb0cb1ade89793683242c45b94be Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:26:53 +0000 Subject: [PATCH 4/4] fix weekly workflow: throughput env, DAX env remap, stale SVG cleanup Co-Authored-By: Noah Kiser --- .github/workflows/weekly-oss-benchmarks.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-oss-benchmarks.yml b/.github/workflows/weekly-oss-benchmarks.yml index 3f5e0607..a2a6e612 100644 --- a/.github/workflows/weekly-oss-benchmarks.yml +++ b/.github/workflows/weekly-oss-benchmarks.yml @@ -132,10 +132,12 @@ jobs: permissions: {} outputs: urls: ${{ steps.pages.outputs.urls }} + env: + THROUGHPUT_ITERATIONS: ${{ needs.matrix.outputs.throughput_iterations || '0' }} steps: - id: pages run: | - count=${{ needs.matrix.outputs.throughput_iterations || '0' }} + count="$THROUGHPUT_ITERATIONS" if [ "$count" -eq 0 ]; then echo 'urls<> "$GITHUB_OUTPUT" echo 'URLS_EOF' >> "$GITHUB_OUTPUT" @@ -211,6 +213,9 @@ jobs: '.[$kind][$provider][]?' \ benchmarks/scripts/provider-vars.json fi | sort -u | grep -v '^$' > /tmp/oss-vault-keys + if [ "$BENCH_KIND" = "dax" ]; then + printf '%s\n' BL_DAX_API_KEY BL_DAX_WORKSPACE >> /tmp/oss-vault-keys + fi printf '%s\n' COMPUTESDK_ADMIN_API_KEY BENCHMARKS_PLATFORM_API_KEY >> /tmp/oss-vault-keys # Build a grep-compatible regex for load-vault-secrets.sh regex="^($(paste -sd'|' /tmp/oss-vault-keys))" @@ -258,6 +263,12 @@ jobs: ;; dax) export LIGHTNING_INSTANCE_TYPE=cpu-8 + export ARKER_PLATFORMS=graviton4 + if [ -n "${BL_DAX_API_KEY:-}" ] && [ -n "${BL_DAX_WORKSPACE:-}" ]; then + export BL_API_KEY="$BL_DAX_API_KEY" + export BL_WORKSPACE="$BL_DAX_WORKSPACE" + unset BL_DAX_API_KEY BL_DAX_WORKSPACE + fi RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" "${RUNNER[@]}" benchmarks/sandbox/dax.bench.ts --provider "$PROVIDER" --iterations "$ITERATIONS" --run-key "$RUN_KEY" $NO_INGEST ;; @@ -323,7 +334,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Clear checked-in OSS results working-directory: oss - run: rm -rf results artifacts + run: rm -rf results artifacts *.svg - name: Download raw results uses: actions/download-artifact@v4 with: