weval-org · nojibe · Oct 2, 2026 · Oct 2, 2026
diff --git a/.github/workflows/delete-runs.yml b/.github/workflows/delete-runs.yml
@@ -0,0 +1,47 @@
+name: Delete Runs
+on:
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: 'Comma-separated runs to delete, each as configId/fileName (e.g. yka-set/370f988e819882dc_2026-10-01T16-25-01-568Z_comparison.json)'
+        required: true
+      dry_run:
+        description: 'Only list what would be deleted (untick to delete; each run is copied to archive/deleted-runs/ first)'
+        type: boolean
+        default: true
+      rebuild_summaries:
+        description: 'After deleting, rebuild homepage, leaderboards and model summaries so the runs drop out of them'
+        type: boolean
+        default: true
+
+jobs:
+  start:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send the request to the server
+        env:
+          RUNS: ${{ github.event.inputs.runs }}
+          DRY_RUN: ${{ github.event.inputs.dry_run }}
+          REBUILD: ${{ github.event.inputs.rebuild_summaries }}
+        run: |
+          BODY=$(python3 -c 'import json, os; runs = [x.strip() for x in os.environ["RUNS"].split(",") if x.strip()]; print(json.dumps({"runs": [{"configId": r.split("/", 1)[0], "fileName": r.split("/", 1)[-1]} for r in runs], "dryRun": os.environ["DRY_RUN"] != "false", "rebuildSummaries": os.environ["REBUILD"] == "true"}))')
+          # RAILWAY_APP_URL may redirect (http -> https, www -> apex); follow it as a POST.
+          STATUS=$(curl -sS -L --post301 --post302 --post303 --max-time 300 -o response.txt -w '%{http_code}' -X POST \
+            -H "Content-Type: application/json" \
+            -H "X-Background-Function-Auth-Token: ${{ secrets.BACKGROUND_FUNCTION_AUTH_TOKEN }}" \
+            -d "$BODY" \
+            "${{ secrets.RAILWAY_APP_URL }}/api/internal/delete-runs")
+          python3 -m json.tool response.txt || cat response.txt
+          if [ "$DRY_RUN" != "false" ]; then
+            if [ "$STATUS" != "200" ]; then
+              echo "::error::Expected 200 from the server, got $STATUS"
+              exit 1
+            fi
+            echo "::notice::Dry run only; nothing was deleted."
+          else
+            if [ "$STATUS" != "202" ]; then
+              echo "::error::Expected 202 from the server, got $STATUS"
+              exit 1
+            fi
+            echo "::notice::Started. Progress is in the Railway logs under 'delete-runs:' / '[DeleteRuns]'."
+          fi
diff --git a/src/app/api/internal/delete-runs/__tests__/route.test.ts b/src/app/api/internal/delete-runs/__tests__/route.test.ts
@@ -0,0 +1,110 @@
+/**
+ * @vitest-environment node
+ */
+import { vi } from 'vitest';
+import { NextRequest, NextResponse } from 'next/server';
+
+vi.mock('@/lib/storageService', () => ({ archiveAndDeleteRun: vi.fn(), listRunsForConfig: vi.fn() }));
+vi.mock('@/cli/commands/backfill-summary', () => ({ actionBackfillSummary: vi.fn() }));
+vi.mock('@/lib/background-function-auth', () => ({ checkBackgroundAuth: vi.fn(() => null) }));
+vi.mock('@/utils/logger', () => ({
+  getLogger: vi.fn(async () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })),
+}));
+
+import { POST } from '../route';
+import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService';
+import { actionBackfillSummary } from '@/cli/commands/backfill-summary';
+import { checkBackgroundAuth } from '@/lib/background-function-auth';
+
+const LATEST = 'abc_2026-10-01T20-11-28-654Z_comparison.json';
+const OLD_1 = 'abc_2026-10-01T19-47-15-945Z_comparison.json';
+const OLD_2 = 'abc_2026-10-01T05-36-17-308Z_comparison.json';
+const request = (body: unknown) =>
+  new NextRequest('http://localhost:3172/api/internal/delete-runs', { method: 'POST', body: JSON.stringify(body) });
+const flush = () => new Promise(r => setTimeout(r, 0));
+const run = (fileName: string, configId = 'yka-set') => ({ configId, fileName });
+
+describe('POST /api/internal/delete-runs', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(listRunsForConfig).mockResolvedValue(
+      [LATEST, OLD_1, OLD_2].map(fileName => ({ runLabel: 'abc', timestamp: null, fileName })),
+    );
+    vi.mocked(archiveAndDeleteRun).mockImplementation(async (configId, fileName, archiveName, dryRun) => ({
+      keys: [`live/blueprints/${configId}/${fileName}`, `live/blueprints/${configId}/x/core.json`],
+      archivePrefix: dryRun ? null : `archive/deleted-runs/${archiveName}`,
+    }));
+  });
+
+  it('defaults to a dry run that lists the runs and deletes nothing', async () => {
+    const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)] }));
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    expect(body.dryRun).toBe(true);
+    expect(body.runs).toEqual([
+      { configId: 'yka-set', fileName: OLD_1, objects: 2 },
+      { configId: 'yka-set', fileName: OLD_2, objects: 2 },
+    ]);
+    expect(vi.mocked(archiveAndDeleteRun).mock.calls.every(c => c[3] === true)).toBe(true);
+    expect(actionBackfillSummary).not.toHaveBeenCalled();
+  });
+
+  it('deletes each run in the background when dryRun is false, then rebuilds when asked', async () => {
+    const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false, rebuildSummaries: true }));
+    expect(res.status).toBe(202);
+    await flush();
+
+    expect(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => [c[1], c[3]])).toEqual([[OLD_1, false], [OLD_2, false]]);
+    const archiveNames = new Set(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => c[2]));
+    expect(archiveNames.size).toBe(1);
+    expect(actionBackfillSummary).toHaveBeenCalledTimes(1);
+  });
+
+  it('keeps going after a run fails, and does not rebuild unless asked', async () => {
+    vi.mocked(archiveAndDeleteRun).mockRejectedValueOnce(new Error('copy failed'));
+
+    const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false }));
+    expect(res.status).toBe(202);
+    await flush();
+
+    expect(archiveAndDeleteRun).toHaveBeenCalledTimes(2);
+    expect(actionBackfillSummary).not.toHaveBeenCalled();
+  });
+
+  it("refuses to delete a blueprint's latest run, and touches nothing", async () => {
+    const res = await POST(request({ runs: [run(OLD_1), run(LATEST)], dryRun: false }));
+    expect(res.status).toBe(400);
+    const body = await res.json();
+    expect(body.problems).toEqual([`yka-set/${LATEST}: is the latest run of yka-set; refusing to delete it`]);
+    expect(archiveAndDeleteRun).not.toHaveBeenCalled();
+  });
+
+  it('refuses runs that do not exist or are listed twice', async () => {
+    const missing = 'abc_2020-01-01T00-00-00Z_comparison.json';
+    const res = await POST(request({ runs: [run(missing), run(OLD_1), run(OLD_1)], dryRun: false }));
+    expect(res.status).toBe(400);
+    const body = await res.json();
+    expect(body.problems).toEqual([`yka-set/${OLD_1}: listed twice`, `yka-set/${missing}: no such run`]);
+    expect(archiveAndDeleteRun).not.toHaveBeenCalled();
+  });
+
+  it('rejects unauthenticated requests', async () => {
+    vi.mocked(checkBackgroundAuth).mockReturnValueOnce(NextResponse.json({ error: 'Unauthorized' }, { status: 401 }));
+    const res = await POST(request({ runs: [run(OLD_1)] }));
+    expect(res.status).toBe(401);
+    expect(listRunsForConfig).not.toHaveBeenCalled();
+    expect(archiveAndDeleteRun).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    ['no runs', { runs: [] }],
+    ['runs not a list', { runs: 'yka-set' }],
+    ['a path in the file name', { runs: [run(`../${OLD_1}`)] }],
+    ['not a run file', { runs: [run('summary.json')] }],
+    ['a bad config ID', { runs: [run(OLD_1, '../live')] }],
+  ])('rejects %s', async (_label, body) => {
+    const res = await POST(request(body));
+    expect(res.status).toBe(400);
+    expect(archiveAndDeleteRun).not.toHaveBeenCalled();
+  });
+});
diff --git a/src/app/api/internal/delete-runs/route.ts b/src/app/api/internal/delete-runs/route.ts
@@ -0,0 +1,127 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { checkBackgroundAuth } from '@/lib/background-function-auth';
+import { configure } from '@/cli/config';
+import { getLogger } from '@/utils/logger';
+import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService';
+import { actionBackfillSummary } from '@/cli/commands/backfill-summary';
+import { toSafeTimestamp } from '@/lib/timestampUtils';
+
+const CONFIG_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,250}$/;
+const FILE_NAME_RE = /^[A-Za-z0-9._-]+_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:-\d{3})?Z_comparison\.json$/;
+const MAX_RUNS = 50;
+
+interface RunRef { configId: string; fileName: string }
+
+/**
+ * Removes specific published runs (e.g. ones whose judging failed), copying
+ * each to archive/deleted-runs/<timestamp>/ first so it can be put back (see
+ * archiveAndDeleteRun). Runs on the server, where the storage credentials
+ * live; triggered by the "Delete Runs" workflow.
+ *
+ * Every run is checked before anything is touched: it must exist, and it must
+ * not be its blueprint's latest run, so a blueprint always keeps the run its
+ * pages and leaderboards are built from. dryRun defaults to true and only
+ * reports what would be removed (200). A real run answers 202 and works in the
+ * background, logging under "delete-runs:"; with rebuildSummaries it then
+ * rebuilds the homepage, leaderboards and model summaries so the removed runs
+ * drop out of them.
+ */
+export async function POST(req: NextRequest) {
+  const authError = checkBackgroundAuth(req);
+  if (authError) return authError;
+
+  let body: any;
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON in request body.' }, { status: 400 });
+  }
+  const runs: unknown = body?.runs;
+  const dryRun = body?.dryRun !== false;
+  const rebuildSummaries = body?.rebuildSummaries === true;
+
+  if (
+    !Array.isArray(runs) || runs.length === 0 || runs.length > MAX_RUNS ||
+    !runs.every(r => typeof r?.configId === 'string' && CONFIG_ID_RE.test(r.configId) &&
+      typeof r?.fileName === 'string' && FILE_NAME_RE.test(r.fileName))
+  ) {
+    return NextResponse.json(
+      { error: `'runs' must be a list of 1-${MAX_RUNS} { configId, fileName } entries naming *_comparison.json run files.` },
+      { status: 400 },
+    );
+  }
+  const refs = runs as RunRef[];
+
+  const problems: string[] = [];
+  const byConfig = new Map<string, string[]>();
+  for (const { configId, fileName } of refs) {
+    const listed = byConfig.get(configId) || [];
+    if (listed.includes(fileName)) problems.push(`${configId}/${fileName}: listed twice`);
+    byConfig.set(configId, [...listed, fileName]);
+  }
+  for (const [configId, fileNames] of byConfig) {
+    const existing = await listRunsForConfig(configId);
+    const latest = existing[0]?.fileName;
+    for (const fileName of fileNames) {
+      if (!existing.some(r => r.fileName === fileName)) problems.push(`${configId}/${fileName}: no such run`);
+      else if (fileName === latest) problems.push(`${configId}/${fileName}: is the latest run of ${configId}; refusing to delete it`);
+    }
+  }
+  if (problems.length > 0) {
+    return NextResponse.json({ error: 'Nothing was deleted.', problems }, { status: 400 });
+  }
+
+  const archiveName = toSafeTimestamp(new Date().toISOString());
+
+  if (dryRun) {
+    const plan = [];
+    for (const { configId, fileName } of refs) {
+      const { keys } = await archiveAndDeleteRun(configId, fileName, archiveName, true);
+      plan.push({ configId, fileName, objects: keys.length });
+    }
+    return NextResponse.json({ dryRun: true, message: 'Nothing was deleted.', runs: plan });
+  }
+
+  const logger = await getLogger('delete-runs');
+  configure({
+    errorHandler: (error: Error) => logger.error(`CLI Error: ${error.message}`, error),
+    logger: {
+      info: (msg: string) => logger.info(msg),
+      warn: (msg: string) => logger.warn(msg),
+      error: (msg: string) => logger.error(msg),
+      success: (msg: string) => logger.info(msg),
+    },
+  });
+
+  void runJob(refs, archiveName, rebuildSummaries, logger);
+
+  return NextResponse.json(
+    { message: 'Accepted. Progress is logged under "delete-runs:".', runs: refs.length, archiveName, rebuildSummaries },
+    { status: 202 },
+  );
+}
+
+async function runJob(refs: RunRef[], archiveName: string, rebuildSummaries: boolean, logger: any): Promise<void> {
+  let deleted = 0;
+  for (const [i, { configId, fileName }] of refs.entries()) {
+    const label = `(${i + 1}/${refs.length}) ${configId}/${fileName}`;
+    try {
+      const { keys, archivePrefix } = await archiveAndDeleteRun(configId, fileName, archiveName, false);
+      deleted++;
+      logger.info(`[DeleteRuns] ${label}: deleted ${keys.length} objects (copies in ${archivePrefix}).`);
+    } catch (error: any) {
+      logger.error(`[DeleteRuns] ${label}: failed, run left in place: ${error?.message || error}`);
+    }
+  }
+  logger.info(`[DeleteRuns] Done: ${deleted} of ${refs.length} runs deleted.`);
+
+  if (rebuildSummaries) {
+    try {
+      logger.info('[DeleteRuns] Rebuilding homepage, leaderboards and model summaries...');
+      await actionBackfillSummary({ verbose: false, dryRun: false });
+      logger.info('[DeleteRuns] Summaries rebuilt.');
+    } catch (error: any) {
+      logger.error(`[DeleteRuns] Summary rebuild failed: ${error?.message || error}`);
+    }
+  }
+}
diff --git a/src/lib/__tests__/archive-delete-run.test.ts b/src/lib/__tests__/archive-delete-run.test.ts
@@ -0,0 +1,74 @@
+process.env.STORAGE_PROVIDER = 'local';
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import path from 'path';
+import { saveResult, archiveAndDeleteRun, listRunsForConfig, DELETED_RUNS_ARCHIVE_DIR } from '@/lib/storageService';
+import { RESULTS_DIR, LIVE_DIR } from '@/cli/constants';
+
+describe('archiveAndDeleteRun (local storage)', () => {
+  const configId = 'test_config_archive_delete';
+  const archiveName = 'test-archive';
+  const keep = 'keep_2024-01-02T00-00-00Z_comparison.json';
+  const remove = 'remove_2024-01-01T00-00-00Z_comparison.json';
+  const configDir = path.join(RESULTS_DIR, LIVE_DIR, 'blueprints', configId);
+  const archiveDir = path.join(RESULTS_DIR, DELETED_RUNS_ARCHIVE_DIR, archiveName);
+
+  const sample = (runLabel: string, timestamp: string): any => ({
+    configId,
+    configTitle: 'Sample',
+    runLabel,
+    timestamp,
+    config: { id: configId, title: 'Sample', models: ['modelA'], prompts: [] },
+    effectiveModels: ['modelA'],
+    promptIds: ['p1'],
+    allFinalAssistantResponses: { p1: { modelA: 'Hello' } },
+    evaluationResults: {
+      llmCoverageScores: { p1: { modelA: { avgCoverageExtent: 0.5, pointAssessments: [] } } },
+      similarityMatrix: {},
+    },
+  });
+
+  beforeEach(async () => {
+    await fs.rm(configDir, { recursive: true, force: true });
+    await fs.rm(archiveDir, { recursive: true, force: true });
+    await saveResult(configId, keep, sample('keep', '2024-01-02T00-00-00Z'));
+    await saveResult(configId, remove, sample('remove', '2024-01-01T00-00-00Z'));
+  });
+
+  afterAll(async () => {
+    await fs.rm(configDir, { recursive: true, force: true });
+    await fs.rm(archiveDir, { recursive: true, force: true });
+  });
+
+  it('lists the comparison file and its artefacts on a dry run, and leaves them in place', async () => {
+    const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, true);
+    const runDir = path.join(LIVE_DIR, 'blueprints', configId, 'remove_2024-01-01T00-00-00Z');
+
+    expect(archivePrefix).toBeNull();
+    expect(keys[0]).toBe(path.join(LIVE_DIR, 'blueprints', configId, remove));
+    expect(keys).toContain(path.join(runDir, 'core.json'));
+    expect(keys.slice(1).every(k => k.startsWith(runDir + path.sep))).toBe(true);
+    expect(fsSync.existsSync(path.join(RESULTS_DIR, keys[0]))).toBe(true);
+    expect(fsSync.existsSync(archiveDir)).toBe(false);
+  });
+
+  it('copies the run to the archive, then deletes it and only it', async () => {
+    const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, false);
+
+    expect(archivePrefix).toBe(path.join(DELETED_RUNS_ARCHIVE_DIR, archiveName));
+    for (const key of keys) {
+      expect(fsSync.existsSync(path.join(RESULTS_DIR, key))).toBe(false);
+      expect(fsSync.existsSync(path.join(RESULTS_DIR, archivePrefix!, key))).toBe(true);
+    }
+    expect(fsSync.existsSync(path.join(configDir, 'remove_2024-01-01T00-00-00Z'))).toBe(false);
+    expect((await listRunsForConfig(configId)).map(r => r.fileName)).toEqual([keep]);
+    expect(fsSync.existsSync(path.join(configDir, 'keep_2024-01-02T00-00-00Z', 'core.json'))).toBe(true);
+  });
+
+  it('throws for a run that does not exist, and for names that are not run files', async () => {
+    await expect(archiveAndDeleteRun(configId, 'missing_2024-01-03T00-00-00Z_comparison.json', archiveName, false))
+      .rejects.toThrow('Run not found');
+    await expect(archiveAndDeleteRun(configId, `../${keep}`, archiveName, false)).rejects.toThrow('Not a run file name');
+    expect((await listRunsForConfig(configId)).length).toBe(2);
+  });
+});