diff --git a/.github/workflows/sandbox-cpu-node.yml b/.github/workflows/sandbox-cpu-node.yml index 473495f4..c7bf3183 100644 --- a/.github/workflows/sandbox-cpu-node.yml +++ b/.github/workflows/sandbox-cpu-node.yml @@ -1,21 +1,23 @@ -name: Cpu-node Benchmark +name: Sandbox Cpu-node Benchmark on: push: branches: [master] paths: - - 'benchmarks/sandbox/cpu-node/**' + - 'benchmarks/sandbox/cpu-node.ts' + - 'benchmarks/sandbox/cpu-node.bench.ts' + - 'benchmarks/sandbox/cpu-node-legacy-results.ts' + - 'benchmarks/sandbox/generate-cpu-node-svg.ts' + - 'benchmarks/scripts/cpu-node-workload.js' + - 'benchmarks/scripts/cpu-node-stdout.js' - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - 'benchmarks/scripts/load-vault-secrets.sh' - 'package.json' - '.github/workflows/sandbox-cpu-node.yml' workflow_dispatch: inputs: - replicas: - description: 'Replicates per provider' + iterations: + description: 'Iterations per provider' required: false default: '3' provider: @@ -93,11 +95,6 @@ jobs: else pnpm install --frozen-lockfile fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - name: Clear stale results if: | github.event_name != 'workflow_dispatch' || @@ -107,13 +104,60 @@ jobs: if: | github.event_name != 'workflow_dispatch' || (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) + env: + PROVIDER: ${{ matrix.provider }} + ITERATIONS: ${{ github.event.inputs.iterations || '3' }} run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' + COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY' + case "$PROVIDER" in + archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) PROVIDER_KEYS='ARKER_API_KEY' ;; + beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) PROVIDER_KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) PROVIDER_KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) PROVIDER_KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) PROVIDER_KEYS='CSB_API_KEY' ;; + createos) PROVIDER_KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) PROVIDER_KEYS='DAYTONA_API_KEY' ;; + declaw) PROVIDER_KEYS='DECLAW_API_KEY' ;; + e2b) PROVIDER_KEYS='E2B_API_KEY' ;; + hopx) PROVIDER_KEYS='HOPX_API_KEY' ;; + isorun) PROVIDER_KEYS='ISORUN_API_KEY' ;; + lightning) PROVIDER_KEYS='LIGHTNING_API_KEY' ;; + modal) PROVIDER_KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + microsandbox) PROVIDER_KEYS='MSB_API_KEY' ;; + miosa) PROVIDER_KEYS='MIOSA_API_KEY' ;; + mosaic) PROVIDER_KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) PROVIDER_KEYS='NSC_TOKEN' ;; + northflank) PROVIDER_KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) PROVIDER_KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + run-cloud) PROVIDER_KEYS='RUN_CLOUD_API_KEY' ;; + runloop) PROVIDER_KEYS='RUNLOOP_API_KEY' ;; + sail) PROVIDER_KEYS='SAIL_API_KEY' ;; + sandbox0) PROVIDER_KEYS='SANDBOX0_TOKEN' ;; + sprites) PROVIDER_KEYS='SPRITES_TOKEN' ;; + superserve) PROVIDER_KEYS='SUPERSERVE_API_KEY' ;; + tenki) PROVIDER_KEYS='TENKI_API_KEY' ;; + tensorlake) PROVIDER_KEYS='TENSORLAKE_API_KEY' ;; + upstash) PROVIDER_KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) PROVIDER_KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) PROVIDER_KEYS='' ;; + esac + if [ -n "$PROVIDER_KEYS" ]; then + KEYS="^(${COMMON_KEYS}|${PROVIDER_KEYS})=" + else + KEYS="^(${COMMON_KEYS})=" + fi + . benchmarks/scripts/load-vault-secrets.sh "$KEYS" - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode cpu-node \ - --replicas ${{ github.event.inputs.replicas || '3' }} + # All matrix jobs for this workflow run share a single platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/cpu-node.bench.ts \ + --provider "$PROVIDER" \ + --iterations "$ITERATIONS" \ + --run-key "$RUN_KEY" \ + ${{ (github.event_name == 'push' || github.event.inputs.dry_run == 'true') && '--no-ingest' || '' }} - name: Upload results if: | always() && ( diff --git a/.github/workflows/sandbox-disk.yml b/.github/workflows/sandbox-disk.yml deleted file mode 100644 index 599a7d36..00000000 --- a/.github/workflows/sandbox-disk.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Disk Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/disk/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-disk.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: disk-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run disk benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode disk \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-disk-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: disk-benchmark-svg - path: 'disk.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type disk - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ disk.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update disk benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-dns.yml b/.github/workflows/sandbox-dns.yml deleted file mode 100644 index e7ca4a3f..00000000 --- a/.github/workflows/sandbox-dns.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: Dns Benchmark - -on: - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: dns-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run dns benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode dns \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-dns-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: dns-benchmark-svg - path: 'dns.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type dns - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ dns.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update dns benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-download.yml b/.github/workflows/sandbox-download.yml deleted file mode 100644 index 390c6de5..00000000 --- a/.github/workflows/sandbox-download.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: Download Benchmark - -on: - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: download-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run download benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode download \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-download-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: download-benchmark-svg - path: 'download.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type download - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ download.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update download benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-latency.yml b/.github/workflows/sandbox-latency.yml deleted file mode 100644 index 3e5173a9..00000000 --- a/.github/workflows/sandbox-latency.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: Latency Benchmark - -on: - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: latency-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run latency benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode latency \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-latency-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: latency-benchmark-svg - path: 'latency.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type latency - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ latency.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update latency benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-memory.yml b/.github/workflows/sandbox-memory.yml deleted file mode 100644 index e46daae2..00000000 --- a/.github/workflows/sandbox-memory.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Memory Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/memory/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-memory.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: memory-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run memory benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode memory \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-memory-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: memory-benchmark-svg - path: 'memory.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type memory - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ memory.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update memory benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-network-localhost.yml b/.github/workflows/sandbox-network-localhost.yml deleted file mode 100644 index 231e505b..00000000 --- a/.github/workflows/sandbox-network-localhost.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Network-localhost Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/network-localhost/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-network-localhost.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: network-localhost-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run network-localhost benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode network-localhost \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-network-localhost-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: network-localhost-benchmark-svg - path: 'network_localhost.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type network-localhost - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ network_localhost.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update network-localhost benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-network-wan.yml b/.github/workflows/sandbox-network-wan.yml deleted file mode 100644 index 8d630b77..00000000 --- a/.github/workflows/sandbox-network-wan.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Network-wan Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/network-wan/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-network-wan.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: network-wan-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run network-wan benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode network-wan \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-network-wan-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: network-wan-benchmark-svg - path: 'network_wan.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type network-wan - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ network_wan.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update network-wan benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-pgbench.yml b/.github/workflows/sandbox-pgbench.yml deleted file mode 100644 index f8a1411e..00000000 --- a/.github/workflows/sandbox-pgbench.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Pgbench Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/pgbench/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-pgbench.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: pgbench-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run pgbench benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode pgbench \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-pgbench-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: pgbench-benchmark-svg - path: 'pgbench.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type pgbench - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ pgbench.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update pgbench benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-realworld.yml b/.github/workflows/sandbox-realworld.yml deleted file mode 100644 index e02847ee..00000000 --- a/.github/workflows/sandbox-realworld.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: Realworld Benchmark - -on: - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: realworld-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run realworld benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode realworld \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-realworld-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: realworld-benchmark-svg - path: 'realworld.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type realworld - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ realworld.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update realworld benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/.github/workflows/sandbox-system.yml b/.github/workflows/sandbox-system.yml deleted file mode 100644 index 49849c6e..00000000 --- a/.github/workflows/sandbox-system.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: System Benchmark - -on: - push: - branches: [master] - paths: - - 'benchmarks/sandbox/system/**' - - 'benchmarks/scripts/smoke.ts' - - 'benchmarks/scripts/build-bundles.ts' - - 'benchmarks/scripts/ensure-bundle.ts' - - 'benchmarks/src/run.ts' - - 'benchmarks/scripts/load-vault-secrets.sh' - - 'package.json' - - '.github/workflows/sandbox-system.yml' - workflow_dispatch: - inputs: - replicas: - description: 'Replicates per provider' - required: false - default: '3' - provider: - description: 'Comma-separated provider filter. Leave empty for all 24 providers.' - required: false - default: '' - dry_run: - description: 'Run without ingesting or committing results' - required: false - default: false - type: boolean - -concurrency: - group: system-benchmark - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - id-token: write - -jobs: - bench: - name: ${{ matrix.provider }} - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - lightning - - modal - - microsandbox - - miosa - - mosaic - - northflank - - opencomputer - - run-cloud - - runloop - - sail - - sandbox0 - - superserve - - tenki - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Build bundles - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: pnpm run build:bundles - - name: Clear stale results - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: rm -rf results/ - - name: Run system benchmark - if: | - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' - - pnpm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode system \ - --replicas ${{ github.event.inputs.replicas || '3' }} - - name: Upload results - if: | - always() && ( - github.event_name != 'workflow_dispatch' || - (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) - ) - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }} - path: results/ - if-no-files-found: ignore - retention-days: 14 - - collect: - name: Collect Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - # runs-on: self-hosted - needs: bench - if: always() - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts/ - pattern: results-* - - name: Merge per-provider results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark - - name: Generate SVG chart - run: pnpm run generate-system-svg - - name: Upload SVG as artifact - if: github.event_name == 'push' - uses: actions/upload-artifact@v4 - with: - name: system-benchmark-svg - path: 'system.svg' - if-no-files-found: ignore - retention-days: 7 - - name: Ingest results to platform - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - continue-on-error: true - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' - npx tsx benchmarks/src/ingest.ts --type system - - name: Commit and push - if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/ system.svg - git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update system benchmark results [skip ci]" - - branch="${GITHUB_REF#refs/heads/}" - for attempt in 1 2 3 4 5; do - git fetch origin "${branch}" - git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } - if git push origin "HEAD:${branch}"; then - echo "Pushed on attempt ${attempt}" - exit 0 - fi - echo "Push rejected (attempt ${attempt}); will rebase and retry" - done - echo "Failed to push after multiple attempts" >&2 diff --git a/benchmarks/sandbox/__tests__/cpu-node.test.ts b/benchmarks/sandbox/__tests__/cpu-node.test.ts new file mode 100644 index 00000000..7323beda --- /dev/null +++ b/benchmarks/sandbox/__tests__/cpu-node.test.ts @@ -0,0 +1,43 @@ +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { computeStats, scoreMetric, SUITE_CONFIG } from '../cpu-node.js'; +import type { CpuNodeWorkloadResult } from '../cpu-node.js'; + +describe('scoring: cpu-node', () => { + test('scoreMetric clamps to [0, 100]', () => { + const suite = SUITE_CONFIG; + assert.equal(scoreMetric(0, suite), 0); // guard: value <= 0 returns 0 + assert.equal(scoreMetric(suite.ceiling, suite), 0); + assert.equal(scoreMetric(suite.ceiling * 2, suite), 0); + }); + + test('computeStats returns zeros for empty results', () => { + const stats = computeStats([], SUITE_CONFIG); + assert.equal(stats.n, 0); + assert.equal(stats.compositeScore, 0); + }); + + test('computeStats handles all-failure iterations', () => { + const results: CpuNodeWorkloadResult[] = [ + { ok: false, suite: 'cpu-node', reason: 'error', error: 'test', meta: {} }, + { ok: false, suite: 'cpu-node', reason: 'timeout', error: 'test', meta: {} }, + ]; + const stats = computeStats(results, SUITE_CONFIG); + assert.equal(stats.n, 0); + assert.equal(stats.successRate, 0); + }); + + test('computeStats produces median and score for successful runs', () => { + const suite = SUITE_CONFIG; + const val = suite.ceiling * 0.5; + const results: CpuNodeWorkloadResult[] = [ + { ok: true, suite: 'cpu-node', metric: { value: val, unit: suite.unit, higherIsBetter: suite.higherIsBetter }, meta: {} }, + { ok: true, suite: 'cpu-node', metric: { value: val, unit: suite.unit, higherIsBetter: suite.higherIsBetter }, meta: {} }, + { ok: true, suite: 'cpu-node', metric: { value: val, unit: suite.unit, higherIsBetter: suite.higherIsBetter }, meta: {} }, + ]; + const stats = computeStats(results, suite); + assert.equal(stats.n, 3); + assert.equal(stats.median, val); + assert.equal(stats.successRate, 1); + }); +}); diff --git a/benchmarks/sandbox/cpu-node-legacy-results.ts b/benchmarks/sandbox/cpu-node-legacy-results.ts new file mode 100644 index 00000000..fb4be4e7 --- /dev/null +++ b/benchmarks/sandbox/cpu-node-legacy-results.ts @@ -0,0 +1,102 @@ +import { mkdirSync, copyFileSync } from 'node:fs'; +import path from 'node:path'; +import type { ParticipantRecords } from '@benchsdk/runner'; +import type { TaskResultRecord } from '@benchsdk/client'; +import { byTaskIndex } from '../src/util/records.js'; +import { computeStats, writeCpuNodeResultsJson, SUITE_CONFIG } from './cpu-node.js'; +import type { CpuNodeBenchmarkResult, CpuNodeWorkloadResult } from './cpu-node.js'; + +/** + * Unwrap one cpu-node task record into the single CpuNodeWorkloadResult it ran. + * The bench file may attach either a full CpuNodeBenchmarkResult (with one + * iteration) or the raw CpuNodeWorkloadResult. + */ +function recordToWorkloadResult(record: TaskResultRecord): CpuNodeWorkloadResult | undefined { + const data = record.data; + if (!data || typeof data !== 'object') return undefined; + const anyData = data as any; + + if (Array.isArray(anyData.iterations) && anyData.iterations.length > 0) { + return anyData.iterations[0] as CpuNodeWorkloadResult; + } + + if (typeof anyData.ok === 'boolean' && anyData.suite === 'cpu-node') { + return anyData as CpuNodeWorkloadResult; + } + + return undefined; +} + +/** Map per-participant task records to legacy cpu-node CpuNodeBenchmarkResult[]. */ +export function recordsToCpuNodeResults(participants: ParticipantRecords[]): CpuNodeBenchmarkResult[] { + return participants.map((participant) => { + const records = byTaskIndex(participant.records); + const iterations: CpuNodeWorkloadResult[] = []; + const replicateMs: number[] = []; + let wallClockMs = 0; + let skipped = false; + let skipReason: string | undefined; + + for (const record of records) { + const data = record.data as any; + if (data && typeof data === 'object') { + if (data.skipped) { + skipped = true; + skipReason = data.skipReason; + continue; + } + if (Array.isArray(data.replicateMs)) { + replicateMs.push(...(data.replicateMs as number[])); + } + if (typeof data.wallClockMs === 'number') { + wallClockMs += data.wallClockMs; + } + } + + const workload = recordToWorkloadResult(record); + if (workload) iterations.push(workload); + } + + const summary = computeStats(iterations, SUITE_CONFIG); + const result: CpuNodeBenchmarkResult = { + provider: participant.participant, + suite: SUITE_CONFIG.id, + mode: 'cpu-node', + iterations, + summary, + wallClockMs, + replicateMs, + compositeScore: summary.compositeScore, + }; + + if (skipped && iterations.length === 0) { + result.skipped = true; + if (skipReason) result.skipReason = skipReason; + } + + return result; + }); +} + +/** + * Map records -> CpuNodeBenchmarkResult[] and write both `.json` + * and `latest.json` into resultsDir, reusing the legacy serializer so the + * `results/cpu_node/` shape is preserved verbatim. + * TEMPORARY BRIDGE until the platform read API exposes per-iteration data. + */ +export async function writeCpuNodeLegacyResults( + participants: ParticipantRecords[], + opts: { resultsDir: string }, +): Promise { + const results = recordsToCpuNodeResults(participants); + + mkdirSync(opts.resultsDir, { recursive: true }); + + const timestamp = new Date().toISOString().slice(0, 10); + const outPath = path.join(opts.resultsDir, `${timestamp}.json`); + await writeCpuNodeResultsJson(results, outPath); + + const latestPath = path.join(opts.resultsDir, 'latest.json'); + copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} diff --git a/benchmarks/sandbox/cpu-node.bench.ts b/benchmarks/sandbox/cpu-node.bench.ts new file mode 100644 index 00000000..1bc36f6e --- /dev/null +++ b/benchmarks/sandbox/cpu-node.bench.ts @@ -0,0 +1,64 @@ +/** + * CPU-node sandbox benchmark: runs the node-web-tooling build inside a fresh + * sandbox once per iteration and measures the build duration. + * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. + * + * Run: + * bench run benchmarks/sandbox/cpu-node.bench.ts + * bench run benchmarks/sandbox/cpu-node.bench.ts --provider e2b,modal --iterations 3 + */ +import '../src/env.js'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { JsonObject } from '@benchsdk/client'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import { providers } from './providers.js'; +import type { ProviderConfig } from './types.js'; +import { runCpuNodeBenchmark, SUITE_CONFIG } from './cpu-node.js'; +import { writeCpuNodeLegacyResults } from './cpu-node-legacy-results.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +export const config = defineBenchmarkConfig({ + benchmarkSlug: `sandbox-cpu-node${process.env.DAILY_BENCH_SLUG ? `-${process.env.DAILY_BENCH_SLUG}` : ''}`, + benchmarkName: `Sandbox CPU node${process.env.DAILY_BENCH_NAME ? ` - ${process.env.DAILY_BENCH_NAME}` : ''}`, + iterations: 3, + concurrency: 1, + groupBy: 'round', + participants: providers, + onScore: (lowerIsBetter) => ({ + metrics: [ + lowerIsBetter('buildMs', { + unit: 'ms', + ceiling: SUITE_CONFIG.ceiling, + value: (record) => record.latencyMs, + weights: { median: 1, p95: 0, p99: 0 }, + }), + ], + }), + onComplete: (outcome) => + writeCpuNodeLegacyResults(outcome.participants, { + resultsDir: path.resolve(__dirname, '../../results/cpu_node'), + }), +}); + +export const task = defineTask(async (ctx) => { + const { participant } = ctx; + const result = await runCpuNodeBenchmark({ ...participant, replicas: 1 }); + const data = result as unknown as JsonObject; + + if (result.skipped) { + return { data }; + } + + const workload = result.iterations[0]; + if (!workload || workload.ok === false) { + const message = workload?.error ?? 'cpu-node workload failed'; + throw new TaskError(message, { + code: 'cpu_node_workload_failed', + data: { ...result, errorMessage: message } as unknown as JsonObject, + }); + } + + return { data, latencyMs: workload.metric.value }; +}); diff --git a/benchmarks/sandbox/cpu-node.ts b/benchmarks/sandbox/cpu-node.ts new file mode 100644 index 00000000..803b8663 --- /dev/null +++ b/benchmarks/sandbox/cpu-node.ts @@ -0,0 +1,451 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { ProviderConfig } from './types.js'; +import { percentile } from '../src/util/stats.js'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; + +// --------------------------------------------------------------------------- +// Suite configuration +// --------------------------------------------------------------------------- + +const SUITE_CONFIG = { + id: 'cpu-node' as const, + label: 'CPU — node-web-tooling build', + unit: 'ms' as const, + higherIsBetter: false, + ceiling: 45000, + defaultReplicas: 3, + workloadPath: 'cpu-node-workload.js', + timeoutMs: 300_000, +}; + +const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../scripts/cpu-node-workload.js'); +const BENCH_STDOUT_PATH = path.resolve(import.meta.dirname, '../scripts/cpu-node-stdout.js'); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface CpuNodeWorkloadMeta { + cpuCount?: number; + memoryMb?: number; + workloadMs?: number; + iterationsReported?: number; + providerNotes?: string; +} + +export type CpuNodeWorkloadResult = + | { + ok: true; + suite: 'cpu-node'; + metric: { value: number; unit: 'ms'; higherIsBetter: boolean }; + meta: CpuNodeWorkloadMeta; + notes?: string; + } + | { + ok: false; + suite: 'cpu-node'; + reason: 'timeout' | 'error' | 'gap' | 'no_tool' | 'unexpected'; + error?: string; + meta: CpuNodeWorkloadMeta; + }; + +export interface CpuNodeSuiteStats { + median: number; + p95: number; + p99: number; + min: number; + max: number; + successRate: number; + n: number; + scoreBeforeReliability: number; + compositeScore: number; + meta: CpuNodeWorkloadMeta; +} + +export interface CpuNodeBenchmarkResult { + provider: string; + suite: 'cpu-node'; + mode: 'cpu-node'; + iterations: CpuNodeWorkloadResult[]; + summary: CpuNodeSuiteStats; + wallClockMs: number; + replicateMs: number[]; + compositeScore: number; + skipped?: boolean; + skipReason?: string; +} + +// Schema-compliant shape written to results/cpu_node/*.json. +export interface MetricSummary { + median: number; + p95?: number; + p99?: number; + min?: number; + max?: number; + avg?: number; +} + +export interface SerializedCpuNodeBenchmarkResult { + provider: string; + suite: 'cpu-node'; + mode: 'cpu-node'; + iterations: CpuNodeWorkloadResult[]; + summary: { totalMs: MetricSummary }; + wallClockMs: number; + replicateMs: number[]; + compositeScore: number; + successRate: number; + n: number; + scoreBeforeReliability: number; + meta: CpuNodeWorkloadMeta; + skipped?: boolean; + skipReason?: string; +} + +// --------------------------------------------------------------------------- +// Scoring +// --------------------------------------------------------------------------- + +/** + * Score a single value against the suite's calibration ceiling. + * Lower is better: 100 * (1 - value/ceiling). Clamped to [0, 100]. + */ +export function scoreMetric(value: number, suite: typeof SUITE_CONFIG): number { + if (!Number.isFinite(value) || value <= 0) return 0; + const ratio = suite.higherIsBetter ? value / suite.ceiling : 1 - value / suite.ceiling; + return clamp(ratio * 100, 0, 100); +} + +/** + * Compute median/min/max/p95/p99/score for a single provider cell. + * 2-sigma outlier trim. If trim leaves < 1 sample, keep the raw median. + */ +export function computeStats(results: CpuNodeWorkloadResult[], suite: typeof SUITE_CONFIG): CpuNodeSuiteStats { + const successful = results.filter(r => r.ok === true); + const ok = successful.length; + const total = results.length; + + if (ok === 0) { + return { + median: 0, p95: 0, p99: 0, min: 0, max: 0, + successRate: 0, n: 0, scoreBeforeReliability: 0, + compositeScore: 0, meta: {}, + }; + } + + const values = successful.map(r => r.metric.value); + const sorted = [...values].sort((a, b) => a - b); + const median = sorted.length % 2 === 0 + ? (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2 + : sorted[Math.floor(sorted.length / 2)]; + + const trimmed = stripOutliersBySigma(sorted, 2); + const statsValues = trimmed.length >= 1 ? trimmed : sorted; + + const p95 = percentile(statsValues, 95); + const p99 = percentile(statsValues, 99); + + return { + median, p95, p99, + min: sorted[0], max: sorted[sorted.length - 1], + successRate: ok / total, n: ok, + scoreBeforeReliability: scoreMetric(median, suite), + compositeScore: round1(scoreMetric(median, suite) * (ok / total)), + meta: lastMeta(successful) ?? {}, + }; +} + +// --------------------------------------------------------------------------- +// Stdout parsing +// --------------------------------------------------------------------------- + +/** + * Read the last JSON line of stdout. Returns a gap WorkloadResult if + * no line parses — never throws. + */ +export function parseWorkloadResult(stdout: string): CpuNodeWorkloadResult { + const lines = stdout.split(/\r?\n/); + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i].trim(); + if (!line || !line.startsWith('{')) continue; + try { + const parsed = JSON.parse(line) as CpuNodeWorkloadResult; + if (!parsed || typeof parsed !== 'object') continue; + if (typeof parsed.suite !== 'string' || parsed.suite !== 'cpu-node') continue; + return parsed; + } catch { + continue; + } + } + + return { + ok: false, + suite: 'cpu-node', + reason: 'unexpected', + error: 'no parseable WorkloadResult JSON line found in stdout', + meta: {}, + }; +} + +// --------------------------------------------------------------------------- +// Benchmark runner +// --------------------------------------------------------------------------- + +/** + * Run the cpu-node benchmark for one provider at the given replicate count. + * + * Each replicate: + * - creates a fresh sandbox via ComputeSDK + * - uploads the workload script + stdout helper via heredoc and executes + * node in a SINGLE runCommand call + * - parses the last WorkloadResult JSON line from stdout + * - destroys the sandbox unconditionally + */ +export async function runCpuNodeBenchmark( + config: ProviderConfig & { replicas?: number }, +): Promise { + const { name, replicas = SUITE_CONFIG.defaultReplicas, timeout = 120_000, requiredEnvVars, sandboxOptions, destroyTimeoutMs = 15_000 } = config; + const suite = SUITE_CONFIG; + + const t0 = performance.now(); + + const missingVars = requiredEnvVars.filter(v => !process.env[v]); + if (missingVars.length > 0) { + return { + provider: name, suite: suite.id, mode: suite.id, + iterations: [], summary: emptyStats(), wallClockMs: 0, + replicateMs: [], compositeScore: 0, + skipped: true, skipReason: 'Missing: ' + missingVars.join(', '), + }; + } + + const payload = buildPayload(); + if (!payload.ok) { + return { + provider: name, suite: suite.id, mode: suite.id, + iterations: [{ ok: false, suite: suite.id, reason: 'error', error: payload.error, meta: {} }], + summary: emptyStats(), wallClockMs: 0, replicateMs: [], compositeScore: 0, + }; + } + + const iterations: CpuNodeWorkloadResult[] = []; + const replicateMs: number[] = []; + + for (let r = 0; r < replicas; r++) { + console.log(' [' + name + '/' + suite.id + '] replicate ' + (r + 1) + '/' + replicas + '...'); + const rStart = performance.now(); + + let sandbox: any = null; + try { + const compute = config.createCompute(); + sandbox = await withTimeout( + compute.sandbox.create(sandboxOptions), + timeout, + 'Sandbox creation timed out', + ); + + const shellCmd = buildSingleCommand(suite, payload); + const result = await withTimeout( + sandbox.runCommand(shellCmd, { timeout: suite.timeoutMs }), + suite.timeoutMs, + suite.id + ' workload timed out', + ) as { exitCode: number; stdout?: string; stderr?: string }; + + if (result.exitCode !== 0) { + const errTail = (result.stderr || '').slice(-500); + iterations.push({ ok: false, suite: suite.id, reason: 'error', error: 'non-zero exit: ' + (errTail || 'no stderr'), meta: {} }); + } else { + const parsed = parseWorkloadResult(result.stdout || ''); + if (!parsed.ok && parsed.reason === 'unexpected') { + const stderrTail = (result.stderr || '').slice(-200); + parsed.error = stderrTail + ? parsed.error + '; stderr: ' + stderrTail + : parsed.error + '; stdout was empty (exitCode=0) — provider may not wait for command completion'; + } + iterations.push(parsed); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + iterations.push({ ok: false, suite: suite.id, reason: classifyError(message), error: message, meta: {} }); + } finally { + replicateMs.push(performance.now() - rStart); + if (sandbox) { + try { + await withTimeout(sandbox.destroy(), destroyTimeoutMs, 'destroy timeout'); + } catch (err) { + console.warn(' [cleanup] destroy failed: ' + formatError(err)); + } + } + } + } + + const summary = computeStats(iterations, suite); + const wallClockMs = performance.now() - t0; + + console.log(' [' + name + '/' + suite.id + '] ' + summary.n + '/' + replicas + ' OK · median ' + summary.median.toFixed(1) + 'ms · score ' + summary.compositeScore); + + return { + provider: name, suite: suite.id, mode: suite.id, + iterations, summary, wallClockMs, replicateMs, + compositeScore: summary.compositeScore, + }; +} + +// --------------------------------------------------------------------------- +// Result writer +// --------------------------------------------------------------------------- + +export async function writeCpuNodeResultsJson(results: CpuNodeBenchmarkResult[], outPath: string): Promise { + const cleanResults: SerializedCpuNodeBenchmarkResult[] = results.map(r => ({ + provider: r.provider, + suite: r.suite, + mode: r.mode, + iterations: r.iterations.map(i => { + if (i.ok) { + return { + ok: true, + suite: i.suite, + metric: { value: round(i.metric.value), unit: i.metric.unit, higherIsBetter: i.metric.higherIsBetter }, + meta: i.meta, + ...(i.notes ? { notes: i.notes } : {}), + }; + } + return { + ok: false, + suite: i.suite, + reason: i.reason, + ...(i.error ? { error: i.error } : {}), + meta: i.meta, + }; + }), + summary: { + totalMs: { + median: round(r.summary.median), + p95: round(r.summary.p95), + p99: round(r.summary.p99), + min: round(r.summary.min), + max: round(r.summary.max), + }, + }, + wallClockMs: round(r.wallClockMs), + replicateMs: r.replicateMs.map(round), + compositeScore: round(r.compositeScore), + successRate: round(r.summary.successRate), + n: r.summary.n, + scoreBeforeReliability: round(r.summary.scoreBeforeReliability), + meta: r.summary.meta, + ...(r.skipped ? { skipped: r.skipped, skipReason: r.skipReason } : {}), + })); + + const output = { + version: '1.0', + timestamp: new Date().toISOString(), + environment: { node: process.version, platform: os.platform(), arch: os.arch() }, + config: { mode: 'cpu-node', ceiling: SUITE_CONFIG.ceiling, unit: SUITE_CONFIG.unit, timeoutMs: SUITE_CONFIG.timeoutMs }, + results: cleanResults, + }; + + fs.writeFileSync(outPath, JSON.stringify(output, null, 2)); + console.log('Results written to ' + outPath); +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +function emptyStats(): CpuNodeSuiteStats { + return { median: 0, p95: 0, p99: 0, min: 0, max: 0, successRate: 0, n: 0, scoreBeforeReliability: 0, compositeScore: 0, meta: {} }; +} + +type FailureReason = Extract['reason']; + +function classifyError(message: string): FailureReason { + if (/timed out|timeout/i.test(message)) return 'timeout'; + if (/gap|skip|not available/i.test(message)) return 'gap'; + return 'error'; +} + +interface Payload { + ok: true; scriptName: string; scriptContent: string; stdoutContent: string; +} +interface PayloadErr { ok: false; error: string } + +function buildPayload(): Payload | PayloadErr { + if (!fs.existsSync(BENCH_SCRIPT_PATH)) { + return { ok: false, error: 'Workload script missing on disk: ' + BENCH_SCRIPT_PATH }; + } + if (!fs.existsSync(BENCH_STDOUT_PATH)) { + return { ok: false, error: 'Stdout helper missing on disk: ' + BENCH_STDOUT_PATH }; + } + const scriptContent = fs.readFileSync(BENCH_SCRIPT_PATH, 'utf8'); + const stdoutContent = fs.readFileSync(BENCH_STDOUT_PATH, 'utf8'); + + return { + ok: true, + scriptName: SUITE_CONFIG.workloadPath, + scriptContent, stdoutContent, + }; +} + +function randomMarker(prefix: string): string { + return '__BENCH_' + prefix + '_' + Math.random().toString(36).slice(2, 10) + '__'; +} + +function buildSingleCommand(suite: typeof SUITE_CONFIG, payload: Payload): string { + const lines: string[] = []; + lines.push('set -e'); + lines.push('mkdir -p /tmp/bench'); + + if (payload.stdoutContent) { + const m1 = randomMarker('STDOUT'); + lines.push('cat > /tmp/bench/stdout.js <<\'' + m1 + '\''); + lines.push(payload.stdoutContent); + lines.push(m1); + } + + const m2 = randomMarker('WORKLOAD'); + lines.push('cat > /tmp/bench/' + payload.scriptName + ' <<\'' + m2 + '\''); + lines.push(payload.scriptContent); + lines.push(m2); + + lines.push('BENCH_SUITE=' + suite.id + ' node /tmp/bench/' + payload.scriptName); + + return lines.join('\n'); +} + +function lastMeta(results: CpuNodeWorkloadResult[]): CpuNodeWorkloadMeta | undefined { + for (let i = results.length - 1; i >= 0; i--) { + if (results[i].meta) return results[i].meta; + } + return undefined; +} + +function stripOutliersBySigma(sorted: number[], sigma: number): number[] { + if (sorted.length < 4) return sorted; + const mean = sorted.reduce((s, v) => s + v, 0) / sorted.length; + const variance = sorted.reduce((s, v) => s + (v - mean) ** 2, 0) / sorted.length; + const sd = Math.sqrt(variance); + if (sd === 0) return sorted; + const lo = mean - sigma * sd; + const hi = mean + sigma * sd; + const trimmed = sorted.filter(v => v >= lo && v <= hi); + return trimmed.length >= 1 ? trimmed : sorted; +} + +function clamp(v: number, lo: number, hi: number): number { + return Math.min(hi, Math.max(lo, v)); +} + +function round1(v: number): number { + return Math.round(v * 10) / 10; +} + +function round(n: number): number { + return Math.round(n * 100) / 100; +} + +// Export SUITE_CONFIG for smoke test and SVG generator +export { SUITE_CONFIG }; diff --git a/benchmarks/sandbox/generate-cpu-node-svg.ts b/benchmarks/sandbox/generate-cpu-node-svg.ts new file mode 100644 index 00000000..633353e8 --- /dev/null +++ b/benchmarks/sandbox/generate-cpu-node-svg.ts @@ -0,0 +1,93 @@ +/** + * Render cpu-node SVG bar chart from results/cpu_node/latest.json. + * Pure SVG (no external libs); renders identically on GitHub. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { SUITE_CONFIG, type SerializedCpuNodeBenchmarkResult } from './cpu-node.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, '..', '..'); +const RESULTS_DIR = path.join(ROOT, 'results'); +const OUT_DIR = ROOT; + +function readLatest(): SerializedCpuNodeBenchmarkResult[] | null { + const dir = path.join(RESULTS_DIR, 'cpu_node'); + const f = path.join(dir, 'latest.json'); + if (!fs.existsSync(f)) return null; + const data = JSON.parse(fs.readFileSync(f, 'utf8')); + if (Array.isArray(data?.results)) return data.results; + return null; +} + +function color(score: number): string { + if (score >= 90) return '#16a34a'; + if (score >= 75) return '#22c55e'; + if (score >= 50) return '#eab308'; + if (score > 0) return '#f97316'; + return '#9ca3af'; +} + +function escapeXml(s: string): string { + return s.replace(/[&<>'"]/g, function (c) { + const map: Record = { + '&': '&', '<': '<', '>': '>', "'": ''', '"': '"', + }; + return map[c] || c; + }); +} + +function generateSvg(): string { + const suite = SUITE_CONFIG; + const results = (readLatest() ?? []).filter(function (r) { return !r.skipped; }); + const visible = results.slice().sort(function (a, b) { return b.compositeScore - a.compositeScore; }); + + const W = 760; + const ROW = 30; + const HEADER_H = 64; + const FOOTER_H = 32; + const H = HEADER_H + Math.max(1, visible.length) * ROW + FOOTER_H; + + const parts: string[] = []; + parts.push(''); + parts.push(''); + parts.push('' + escapeXml(suite.label) + ''); + parts.push('unit: ' + suite.unit + ' ceiling: ' + suite.ceiling + ' method: median, 2-sigma trim'); + + if (visible.length === 0) { + parts.push('no results yet'); + parts.push(''); + return parts.join('\n'); + } + + const labelX = 20; + const barX = 220; + const barW = 440; + for (let i = 0; i < visible.length; i++) { + const y = HEADER_H + 8 + i * ROW; + const r = visible[i]; + const w = Math.max(2, (r.compositeScore / 100) * barW); + const sublabel = 'n=' + r.n + ' ' + suite.unit + ' ceiling=' + suite.ceiling; + parts.push('' + escapeXml(r.provider) + ''); + parts.push(''); + parts.push(''); + parts.push('' + r.compositeScore.toFixed(1) + ''); + parts.push('' + escapeXml(sublabel) + ''); + } + parts.push(''); + return parts.join('\n'); +} + +function main(): void { + const svg = generateSvg(); + const name = 'cpu_node.svg'; + const target = path.join(OUT_DIR, name); + fs.writeFileSync(target, svg); + console.log('Wrote ' + name + ' (' + svg.length + ' bytes)'); +} + +if (process.argv[1] && process.argv[1].endsWith('generate-cpu-node-svg.ts')) { + main(); +} diff --git a/benchmarks/scripts/cpu-node-stdout.js b/benchmarks/scripts/cpu-node-stdout.js new file mode 100644 index 00000000..90ffa488 --- /dev/null +++ b/benchmarks/scripts/cpu-node-stdout.js @@ -0,0 +1,51 @@ +/** + * Workloads are uploaded to a fresh sandbox and executed with `node .js`. + * Every workload must `console.log` a single WorkloadResult JSON line as + * its LAST line of stdout — the runner parses only the last line. + * + * This helper guarantees the line stays last even if intermediate logging + * happens (e.g. status updates). + * + * Pure CommonJS — no transpile step runs inside ComputeSDK sandboxes, so we + * intentionally avoid `import`/`export` and TypeScript syntax here. The TS + * counterpart of this file is `stdout.ts` in the same directory, which the + * build pipeline does NOT upload; only this `.js` is shipped with each suite. + */ + +function emitWorkloadResult(result) { + process.stdout.write(JSON.stringify(result) + '\n'); +} + +// Global crash handler: if a workload throws during module loading (e.g. +// require() of a native module fails, or an unhandled rejection fires +// before the workload's own catch handler is installed), emit a failure +// result so the orchestrator gets a parseable JSON line instead of empty +// stdout. Without this, providers like beam that crash before emit show +// up as "no parseable WorkloadResult" with no diagnostic info. +let _crashed = false; +process.on('uncaughtException', (err) => { + if (_crashed) return; + _crashed = true; + emitWorkloadResult({ + ok: false, + suite: process.env.BENCH_SUITE || 'unknown', + reason: 'error', + error: err && err.message ? err.message : String(err), + meta: { stack: err && err.stack ? err.stack.split('\n').slice(0, 3).join(' | ') : undefined }, + }); + process.exit(0); +}); +process.on('unhandledRejection', (reason) => { + if (_crashed) return; + _crashed = true; + emitWorkloadResult({ + ok: false, + suite: process.env.BENCH_SUITE || 'unknown', + reason: 'error', + error: reason instanceof Error ? reason.message : String(reason), + meta: {}, + }); + process.exit(0); +}); + +module.exports = { emitWorkloadResult }; diff --git a/benchmarks/scripts/cpu-node-workload.js b/benchmarks/scripts/cpu-node-workload.js new file mode 100644 index 00000000..44648a38 --- /dev/null +++ b/benchmarks/scripts/cpu-node-workload.js @@ -0,0 +1,150 @@ +'use strict'; + +/** + * In-sandbox workload for the cpu-node benchmark. + * + * Runs four deterministic compute phases (JSON round-trip, SHA-256 + * hashing, regex text walk, sum-of-primes) and emits the total wall-clock + * as the suite metric via a WorkloadResult JSON line. + * + * Pure stdlib — no node_modules, no native bindings, runs identically on + * any Node 18+ sandbox regardless of provider. + */ + +const crypto = require('node:crypto'); +const { emitWorkloadResult } = require('./stdout.js'); + +const ITERATIONS = 64; + +// ---- Phase 1: JSON round-trip cost --------------------------- +function makeAst(depth) { + if (depth === 0) return { kind: 'leaf', value: 'x', pos: 0, parent: null }; + return { + kind: 'branch', + children: Array.from({ length: 5 }, (_, i) => { + const c = makeAst(depth - 1); + c.parent = { idx: i, depth }; + c.pos = i; + return c; + }), + pos: depth, + }; +} + +function phase1(iters, astDepth) { + let total = 0; + for (let i = 0; i < iters; i++) { + const ast = makeAst(astDepth); + const ser = JSON.stringify(ast); + const parsed = JSON.parse(ser); + total += parsed.children.length; + } + return total; +} + +// ---- Phase 2: crypto throughput ----------------------------- +function phase2(iters, bytesPerBlock, blocks) { + let totalBytes = 0; + const buf = crypto.randomFillSync(Buffer.alloc(bytesPerBlock)); + for (let i = 0; i < iters; i++) { + for (let b = 0; b < blocks; b++) { + const h = crypto.createHash('sha256'); + h.update(buf); + h.digest(); + totalBytes += bytesPerBlock; + } + } + return totalBytes; +} + +// ---- Phase 3: regex + text walk ----------------------------- +function makeCorpus(bytes) { + const rng = (() => { + let s = 0x12345678; + return () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 0x100000000; + }; + })(); + const words = ['module', 'export', 'class', 'function', 'const', 'return', 'await', + 'async', 'throw', 'import', 'from', 'with', 'yield', 'new', 'extends']; + const out = []; + let acc = ''; + let totalLen = 0; + while (totalLen < bytes) { + const w = words[Math.floor(rng() * words.length)]; + acc += w + ' '; + totalLen += w.length + 1; + if (rng() < 0.05) { + out.push(acc); + acc = ''; + } + } + if (acc.length) out.push(acc); + return out.join('\n'); +} + +function phase3(iters, bytes) { + const corpus = makeCorpus(Math.max(64 * 1024, bytes)); + const patterns = [ + /function\s+\w+/g, + /class\s+\w+\s+extends\s+\w+/g, + /\bawait\s+\w+/g, + /\bimport\s+.*?from\s+/g, + ]; + let totalMatches = 0; + for (let i = 0; i < iters; i++) { + for (const pat of patterns) { + pat.lastIndex = 0; + const m = corpus.match(pat); + if (m) totalMatches += m.length; + } + } + return totalMatches; +} + +// ---- Phase 4: numeric loop ---------------------------------- +function sumOfPrimes(limit) { + let total = 0; + for (let n = 2; n <= limit; n++) { + let prime = true; + for (let d = 2; d * d <= n; d++) { + if (n % d === 0) { prime = false; break; } + } + if (prime) total += n; + } + return total; +} + +function phase4(iters) { + let checksum = 0; + for (let i = 0; i < iters; i++) checksum += sumOfPrimes(5_000_000); + return checksum; +} + +// ---- Driver -------------------------------------------------- + +const t0 = process.hrtime.bigint(); + +const phase1_iters = Math.max(1, Math.floor(ITERATIONS / 4)); +const phase2_iters = Math.max(1, Math.floor(ITERATIONS / 8)); +const phase3_iters = Math.max(1, Math.floor(ITERATIONS / 4)); +const phase4_iters = Math.max(1, Math.floor(ITERATIONS / 4)); + +phase1(phase1_iters, 7); +phase2(phase2_iters, 1024 * 1024, 64); +phase3(phase3_iters, 256 * 1024); +phase4(phase4_iters); + +const totalMs = Number((process.hrtime.bigint() - t0) / 1_000_000n); + +emitWorkloadResult({ + ok: true, + suite: 'cpu-node', + metric: { value: totalMs, unit: 'ms', higherIsBetter: false }, + meta: { + iterationsReported: ITERATIONS, + workloadMs: totalMs, + }, +}); +process.exit(0); diff --git a/benchmarks/scripts/load-vault-secrets.sh b/benchmarks/scripts/load-vault-secrets.sh index 828ded85..0b3c4737 100755 --- a/benchmarks/scripts/load-vault-secrets.sh +++ b/benchmarks/scripts/load-vault-secrets.sh @@ -60,6 +60,9 @@ eval "$(nsc vault export --envdef /tmp/vault.envdef --shell=bash)" # masked here. Multi-line values like PEM keys register as one `::add-mask::` # call; GH Actions splits the value internally so each line gets masked. awk -F= '{print $1}' /tmp/vault.envdef | while IFS= read -r key; do + [ -n "$key" ] || continue v="${!key-}" - [ -n "$v" ] && echo "::add-mask::$v" + if [ -n "$v" ]; then + echo "::add-mask::$v" + fi done diff --git a/benchmarks/scripts/smoke.ts b/benchmarks/scripts/smoke.ts new file mode 100644 index 00000000..c5c9d639 --- /dev/null +++ b/benchmarks/scripts/smoke.ts @@ -0,0 +1,78 @@ +/** + * Smoke harness for the cpu-node benchmark. Runs the workload locally + * (no ComputeSDK, no cloud sandbox) and asserts it emits a parseable + * WorkloadResult JSON line. + * + * Usage: tsx benchmarks/scripts/smoke.ts --suite=cpu-node + */ + +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import fs from 'node:fs'; +import { SUITE_CONFIG, scoreMetric, parseWorkloadResult } from '../sandbox/cpu-node.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, '..', '..'); + +function getArg(flag: string): string | undefined { + const args = process.argv.slice(2).filter(a => a !== '--'); + for (const arg of args) { + if (arg.startsWith(flag + '=')) return arg.slice(flag.length + 1); + } + const idx = args.indexOf(flag); + return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined; +} + +async function main(): Promise { + const suite = SUITE_CONFIG; + const scriptsDir = path.join(ROOT, 'benchmarks', 'scripts'); + const workloadPath = path.join(scriptsDir, suite.workloadPath); + const stdoutPath = path.join(scriptsDir, 'cpu-node-stdout.js'); + + console.log(`\n[smoke] ▶ cpu-node (${suite.label})`); + console.log(` ceiling: ${suite.ceiling} ${suite.unit} (${suite.higherIsBetter ? '↑ better' : '↓ better'})`); + + // Write the workload + stdout helper to a temp dir and run with node + const workdir = `/tmp/bench-smoke-${process.pid}-${Date.now()}`; + fs.mkdirSync(path.join(workdir, 'bench'), { recursive: true }); + + fs.copyFileSync(workloadPath, path.join(workdir, 'bench', suite.workloadPath)); + if (fs.existsSync(stdoutPath)) { + fs.copyFileSync(stdoutPath, path.join(workdir, 'bench', 'stdout.js')); + } + + const startedAt = Date.now(); + const result = spawnSync('node', [path.join(workdir, 'bench', suite.workloadPath)], { + cwd: path.join(workdir, 'bench'), + encoding: 'utf8', + timeout: suite.timeoutMs, + env: { ...process.env, BENCH_SUITE: 'cpu-node' }, + }); + spawnSync('rm', ['-rf', workdir]); + const elapsedMs = Date.now() - startedAt; + + if (result.error) { + console.error(`[smoke] ✗ spawn failed: ${result.error.message}`); + process.exit(1); + } + if (result.status !== 0) { + console.error(`[smoke] ✗ exit ${result.status}`); + console.error(result.stderr || '(no stderr)'); + process.exit(1); + } + + const parsed = parseWorkloadResult(result.stdout); + if (!parsed.ok) { + console.error(`[smoke] ✗ parse failed (reason: ${parsed.reason ?? 'unknown'})`); + if ('error' in parsed && parsed.error) console.error(` error: ${parsed.error}`); + process.exit(1); + } + + const score = scoreMetric(parsed.metric.value, suite); + console.log(` ✓ ${parsed.metric.value.toLocaleString()} ${parsed.metric.unit} in ${elapsedMs} ms (score ${score.toFixed(1)}/100)`); + console.log(`\n[smoke] 1 passed · 0 failed · 1 total`); + process.exit(0); +} + +main(); diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index 5a0b4688..d432b6ab 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -43,6 +43,24 @@ import type { SnapshotForkBenchmarkResult } from '../storage/snapshot-fork-types import type { BrowserBenchmarkResult } from '../browser/types.js'; import type { ThroughputBenchmarkResult } from '../browser/throughput-types.js'; import type { AIGatewayBenchmarkResult } from '../ai-gateway/types.js'; +// Generic result type for benchmark merge (all benchmarks share the same shape) +type GenericBenchmarkResult = { + provider: string; + suite: string; + compositeScore: number; + skipped?: boolean; + skipReason?: string; + iterations: any[]; + successRate?: number; + n?: number; + summary: Record>; +}; + +// Benchmark suite IDs for the standalone benchmarks +const BENCHMARK_SUITE_IDS = [ + 'cpu-node', +]; +const BENCHMARK_DIR_NAMES = new Set(BENCHMARK_SUITE_IDS.map(id => id.replace(/-/g, '_'))); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '../..'); @@ -737,6 +755,164 @@ async function mainAIGateway() { console.log(`Copied latest: ${latestPath}`); } +/** + * Print a benchmark results table to stdout. + */ +function printHpcResultsTable(results: GenericBenchmarkResult[], suiteDir: string): void { + const suiteId = suiteDir.replace(/_/g, '-'); + const sorted = [...results].sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); + + const suiteLabel = suiteId; + const unit = ''; + const ceiling = 0; + + console.log(`\n${'='.repeat(95)}`); + console.log(` BENCHMARK RESULTS - ${suiteLabel}`); + console.log('='.repeat(95)); + console.log(` Unit: ${unit} Ceiling: ${ceiling} Providers: ${sorted.length}`); + console.log( + ['#', 'Provider', 'Score', 'Median', 'Unit', 'n', 'Status'] + .map((h, i) => h.padEnd([4, 14, 8, 12, 12, 4, 10][i])) + .join(' | ') + ); + console.log( + [4, 14, 8, 12, 12, 4, 10].map(w => '-'.repeat(w)).join('-+-') + ); + + for (let i = 0; i < sorted.length; i++) { + const r = sorted[i]; + if (r.skipped) { + console.log([String(i + 1).padEnd(4), r.provider.padEnd(14), '--'.padEnd(8), '--'.padEnd(12), unit.padEnd(12), '--'.padEnd(4), 'SKIPPED'.padEnd(10)].join(' | ')); + continue; + } + const ok = (r.iterations || []).filter(it => !(it as any).error && (it as any).ok).length; + const total = (r.iterations || []).length; + const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; + const totalMs = (r.summary as any)?.totalMs; + const median = typeof totalMs?.median === 'number' ? totalMs.median.toFixed(1) : '--'; + console.log([String(i + 1).padEnd(4), r.provider.padEnd(14), score.padEnd(8), median.padEnd(12), unit.padEnd(12), String(r.n ?? 0).padEnd(4), `${ok}/${total} OK`.padEnd(10)].join(' | ')); + } + console.log('='.repeat(95)); +} + +/** + * Merge HPC benchmark results, grouped by suite. + * + * Each matrix cell produces a results/hpc_/latest.json with a single + * provider's result. This function walks the downloaded artifacts, groups by + * suite directory, deduplicates by provider (preferring single-provider files), + * and writes combined results/hpc_/latest.json with all providers. + * + * Composite scores are already computed per-provider in the matrix cells, so + * merging is just dedupe + write — no cross-provider recomputation needed. + */ +async function mainHpc() { + const jsonFiles: string[] = []; + function walk(dir: string) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === 'latest.json') jsonFiles.push(full); + } + } + walk(inputDir!); + + if (jsonFiles.length === 0) { + console.error(`No latest.json files found in ${inputDir}`); + process.exit(1); + } + + console.log(`Found ${jsonFiles.length} result files`); + + // Group by suite directory (e.g. hpc_cpu_node, hpc_memory, ...). + // Only pick up files whose parent dir starts with hpc_. + const bySuite: Record< + string, + { + version?: string; + timestamp?: string; + environment?: Record; + config?: Record; + results: { result: GenericBenchmarkResult; fromSingleProvider: boolean }[]; + } + > = {}; + + for (const file of jsonFiles) { + const dirName = path.basename(path.dirname(file)); + if (!dirName.startsWith('hpc_') && !BENCHMARK_DIR_NAMES.has(dirName)) continue; + + const raw = JSON.parse(fs.readFileSync(file, 'utf-8')) as { + version?: string; + timestamp?: string; + environment?: Record; + config?: Record; + results?: GenericBenchmarkResult[]; + }; + if (!raw.results || raw.results.length === 0) continue; + + const fromSingleProvider = raw.results.length === 1; + if (!bySuite[dirName]) bySuite[dirName] = { results: [] }; + const suite = bySuite[dirName]; + if (!suite.version && raw.version) suite.version = raw.version; + if (!suite.timestamp && raw.timestamp) suite.timestamp = raw.timestamp; + if (!suite.environment && raw.environment) suite.environment = raw.environment; + if (!suite.config && raw.config) suite.config = raw.config; + for (const result of raw.results) { + suite.results.push({ result, fromSingleProvider }); + } + } + + const suiteDirs = Object.keys(bySuite).sort(); + if (suiteDirs.length === 0) { + console.log('No HPC result files found in artifacts'); + process.exit(0); + } + + for (const suiteDir of suiteDirs) { + const suite = bySuite[suiteDir]; + const { results } = suite; + + // Deduplicate by provider, preferring fresh single-provider files. + const seen = new Map(); + for (const entry of results) { + const existing = seen.get(entry.result.provider); + if (!existing || (entry.fromSingleProvider && !existing.fromSingleProvider)) { + seen.set(entry.result.provider, entry); + } + } + const deduped = Array.from(seen.values()).map(e => e.result); + + if (deduped.length !== results.length) { + console.log(`\nMerging ${deduped.length} provider results for ${suiteDir} (deduplicated from ${results.length})`); + } else { + console.log(`\nMerging ${deduped.length} provider results for ${suiteDir}`); + } + + printHpcResultsTable(deduped, suiteDir); + + const timestamp = new Date().toISOString().slice(0, 10); + const resultsDir = path.resolve(ROOT, `results/${suiteDir}`); + fs.mkdirSync(resultsDir, { recursive: true }); + + const output = { + version: suite.version ?? '1.0', + timestamp: new Date().toISOString(), + environment: suite.environment ?? {}, + config: suite.config ?? {}, + results: deduped, + }; + + const outPath = path.join(resultsDir, `${timestamp}.json`); + fs.writeFileSync(outPath, JSON.stringify(output, null, 2)); + console.log(`Results written to ${outPath}`); + + const latestPath = path.join(resultsDir, 'latest.json'); + fs.copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); + } +} + const runner = mergeMode === 'storage' ? mainStorage : mergeMode === 'snapshot-fork' @@ -747,6 +923,8 @@ const runner = mergeMode === 'storage' ? mainBrowserThroughput : mergeMode === 'ai-gateway' ? mainAIGateway + : (mergeMode === 'hpc' || mergeMode === 'benchmark') + ? mainHpc : main; runner().catch(err => { console.error('Merge failed:', err); diff --git a/package.json b/package.json index f0b301e0..a51f123a 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,11 @@ "bench:ai-gateway-kimi:novita": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-kimi.bench.ts --provider novita", "bench:ai-gateway-kimi:ramp": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-kimi.bench.ts --provider ramp", "bench:ai-gateway-kimi:neon": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-kimi.bench.ts --provider neon", - "generate-ai-gateway-kimi-svg": "tsx benchmarks/ai-gateway/generate-svg.ts --dir ai-gateway-latency/kimi --title \"AI Gateway Benchmarks — Kimi\" --out ai-gateway-kimi.svg" + "generate-ai-gateway-kimi-svg": "tsx benchmarks/ai-gateway/generate-svg.ts --dir ai-gateway-latency/kimi --title \"AI Gateway Benchmarks — Kimi\" --out ai-gateway-kimi.svg", + "bench:cpu-node": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/cpu-node.bench.ts", + "generate-cpu-node-svg": "tsx benchmarks/sandbox/generate-cpu-node-svg.ts", + "smoke:cpu-node": "tsx benchmarks/scripts/smoke.ts --suite=cpu-node", + "test:benchmarks": "node --import tsx --test 'benchmarks/sandbox/__tests__/*.test.ts'" }, "dependencies": { "@aws-sdk/client-s3": "^3.1110.0", diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index c542ada9..49efd7df 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -331,7 +331,9 @@ function defaultOnResult(record: TaskResultRecord, meta: { iterations: number; p const data = record.data && Object.keys(record.data).length > 0 ? ` ${JSON.stringify(record.data)}` : ''; console.log(` [${meta.participant}] Task ${n}/${meta.iterations}: success${data}`); } else { - console.log(` [${meta.participant}] Task ${n}/${meta.iterations}: FAILED — ${record.errorCode ?? 'unknown error'}`); + const errorMessage = record.data?.['errorMessage']; + const suffix = typeof errorMessage === 'string' ? `: ${errorMessage}` : ''; + console.log(` [${meta.participant}] Task ${n}/${meta.iterations}: FAILED — ${record.errorCode ?? 'unknown error'}${suffix}`); } }