From caf2e90ee4c8a68e7666e7ba075d8cb6169d0569 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 00:54:31 +0000 Subject: [PATCH] Delete specific runs, archiving them first Adds a way to remove individual published runs (for example, runs whose judging failed) so they stop counting towards model pages, which average every stored run. There was no working way to do this: the only delete command removes a whole blueprint, and deleteResultByFileName still points at the old multi/ layout. - storageService.archiveAndDeleteRun copies a run's comparison file and its artefact folder to archive/deleted-runs// under their original keys, and deletes them only if every copy succeeded. The archive is outside backups/ so restore-data can never mistake it for a full backup. - POST /api/internal/delete-runs checks every run first (it must exist and must not be its blueprint's latest run), dry-runs by default, and on a real run works in the background and can rebuild summaries afterwards. - The "Delete Runs" workflow calls it; dry run is ticked by default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015roAwvpcszBvjX1Ce5NFuj --- .github/workflows/delete-runs.yml | 47 +++++++ .../delete-runs/__tests__/route.test.ts | 110 +++++++++++++++ src/app/api/internal/delete-runs/route.ts | 127 ++++++++++++++++++ src/lib/__tests__/archive-delete-run.test.ts | 74 ++++++++++ src/lib/__tests__/storageService.test.ts | 65 +++++++++ src/lib/storageService.ts | 121 +++++++++++++++++ 6 files changed, 544 insertions(+) create mode 100644 .github/workflows/delete-runs.yml create mode 100644 src/app/api/internal/delete-runs/__tests__/route.test.ts create mode 100644 src/app/api/internal/delete-runs/route.ts create mode 100644 src/lib/__tests__/archive-delete-run.test.ts diff --git a/.github/workflows/delete-runs.yml b/.github/workflows/delete-runs.yml new file mode 100644 index 00000000..dc0cb793 --- /dev/null +++ b/.github/workflows/delete-runs.yml @@ -0,0 +1,47 @@ +name: Delete Runs +on: + workflow_dispatch: + inputs: + runs: + description: 'Comma-separated runs to delete, each as configId/fileName (e.g. yka-set/370f988e819882dc_2026-10-01T16-25-01-568Z_comparison.json)' + required: true + dry_run: + description: 'Only list what would be deleted (untick to delete; each run is copied to archive/deleted-runs/ first)' + type: boolean + default: true + rebuild_summaries: + description: 'After deleting, rebuild homepage, leaderboards and model summaries so the runs drop out of them' + type: boolean + default: true + +jobs: + start: + runs-on: ubuntu-latest + steps: + - name: Send the request to the server + env: + RUNS: ${{ github.event.inputs.runs }} + DRY_RUN: ${{ github.event.inputs.dry_run }} + REBUILD: ${{ github.event.inputs.rebuild_summaries }} + run: | + BODY=$(python3 -c 'import json, os; runs = [x.strip() for x in os.environ["RUNS"].split(",") if x.strip()]; print(json.dumps({"runs": [{"configId": r.split("/", 1)[0], "fileName": r.split("/", 1)[-1]} for r in runs], "dryRun": os.environ["DRY_RUN"] != "false", "rebuildSummaries": os.environ["REBUILD"] == "true"}))') + # RAILWAY_APP_URL may redirect (http -> https, www -> apex); follow it as a POST. + STATUS=$(curl -sS -L --post301 --post302 --post303 --max-time 300 -o response.txt -w '%{http_code}' -X POST \ + -H "Content-Type: application/json" \ + -H "X-Background-Function-Auth-Token: ${{ secrets.BACKGROUND_FUNCTION_AUTH_TOKEN }}" \ + -d "$BODY" \ + "${{ secrets.RAILWAY_APP_URL }}/api/internal/delete-runs") + python3 -m json.tool response.txt || cat response.txt + if [ "$DRY_RUN" != "false" ]; then + if [ "$STATUS" != "200" ]; then + echo "::error::Expected 200 from the server, got $STATUS" + exit 1 + fi + echo "::notice::Dry run only; nothing was deleted." + else + if [ "$STATUS" != "202" ]; then + echo "::error::Expected 202 from the server, got $STATUS" + exit 1 + fi + echo "::notice::Started. Progress is in the Railway logs under 'delete-runs:' / '[DeleteRuns]'." + fi diff --git a/src/app/api/internal/delete-runs/__tests__/route.test.ts b/src/app/api/internal/delete-runs/__tests__/route.test.ts new file mode 100644 index 00000000..5df8adf2 --- /dev/null +++ b/src/app/api/internal/delete-runs/__tests__/route.test.ts @@ -0,0 +1,110 @@ +/** + * @vitest-environment node + */ +import { vi } from 'vitest'; +import { NextRequest, NextResponse } from 'next/server'; + +vi.mock('@/lib/storageService', () => ({ archiveAndDeleteRun: vi.fn(), listRunsForConfig: vi.fn() })); +vi.mock('@/cli/commands/backfill-summary', () => ({ actionBackfillSummary: vi.fn() })); +vi.mock('@/lib/background-function-auth', () => ({ checkBackgroundAuth: vi.fn(() => null) })); +vi.mock('@/utils/logger', () => ({ + getLogger: vi.fn(async () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })), +})); + +import { POST } from '../route'; +import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService'; +import { actionBackfillSummary } from '@/cli/commands/backfill-summary'; +import { checkBackgroundAuth } from '@/lib/background-function-auth'; + +const LATEST = 'abc_2026-10-01T20-11-28-654Z_comparison.json'; +const OLD_1 = 'abc_2026-10-01T19-47-15-945Z_comparison.json'; +const OLD_2 = 'abc_2026-10-01T05-36-17-308Z_comparison.json'; +const request = (body: unknown) => + new NextRequest('http://localhost:3172/api/internal/delete-runs', { method: 'POST', body: JSON.stringify(body) }); +const flush = () => new Promise(r => setTimeout(r, 0)); +const run = (fileName: string, configId = 'yka-set') => ({ configId, fileName }); + +describe('POST /api/internal/delete-runs', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listRunsForConfig).mockResolvedValue( + [LATEST, OLD_1, OLD_2].map(fileName => ({ runLabel: 'abc', timestamp: null, fileName })), + ); + vi.mocked(archiveAndDeleteRun).mockImplementation(async (configId, fileName, archiveName, dryRun) => ({ + keys: [`live/blueprints/${configId}/${fileName}`, `live/blueprints/${configId}/x/core.json`], + archivePrefix: dryRun ? null : `archive/deleted-runs/${archiveName}`, + })); + }); + + it('defaults to a dry run that lists the runs and deletes nothing', async () => { + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)] })); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.dryRun).toBe(true); + expect(body.runs).toEqual([ + { configId: 'yka-set', fileName: OLD_1, objects: 2 }, + { configId: 'yka-set', fileName: OLD_2, objects: 2 }, + ]); + expect(vi.mocked(archiveAndDeleteRun).mock.calls.every(c => c[3] === true)).toBe(true); + expect(actionBackfillSummary).not.toHaveBeenCalled(); + }); + + it('deletes each run in the background when dryRun is false, then rebuilds when asked', async () => { + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false, rebuildSummaries: true })); + expect(res.status).toBe(202); + await flush(); + + expect(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => [c[1], c[3]])).toEqual([[OLD_1, false], [OLD_2, false]]); + const archiveNames = new Set(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => c[2])); + expect(archiveNames.size).toBe(1); + expect(actionBackfillSummary).toHaveBeenCalledTimes(1); + }); + + it('keeps going after a run fails, and does not rebuild unless asked', async () => { + vi.mocked(archiveAndDeleteRun).mockRejectedValueOnce(new Error('copy failed')); + + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false })); + expect(res.status).toBe(202); + await flush(); + + expect(archiveAndDeleteRun).toHaveBeenCalledTimes(2); + expect(actionBackfillSummary).not.toHaveBeenCalled(); + }); + + it("refuses to delete a blueprint's latest run, and touches nothing", async () => { + const res = await POST(request({ runs: [run(OLD_1), run(LATEST)], dryRun: false })); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.problems).toEqual([`yka-set/${LATEST}: is the latest run of yka-set; refusing to delete it`]); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it('refuses runs that do not exist or are listed twice', async () => { + const missing = 'abc_2020-01-01T00-00-00Z_comparison.json'; + const res = await POST(request({ runs: [run(missing), run(OLD_1), run(OLD_1)], dryRun: false })); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.problems).toEqual([`yka-set/${OLD_1}: listed twice`, `yka-set/${missing}: no such run`]); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it('rejects unauthenticated requests', async () => { + vi.mocked(checkBackgroundAuth).mockReturnValueOnce(NextResponse.json({ error: 'Unauthorized' }, { status: 401 })); + const res = await POST(request({ runs: [run(OLD_1)] })); + expect(res.status).toBe(401); + expect(listRunsForConfig).not.toHaveBeenCalled(); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it.each([ + ['no runs', { runs: [] }], + ['runs not a list', { runs: 'yka-set' }], + ['a path in the file name', { runs: [run(`../${OLD_1}`)] }], + ['not a run file', { runs: [run('summary.json')] }], + ['a bad config ID', { runs: [run(OLD_1, '../live')] }], + ])('rejects %s', async (_label, body) => { + const res = await POST(request(body)); + expect(res.status).toBe(400); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/internal/delete-runs/route.ts b/src/app/api/internal/delete-runs/route.ts new file mode 100644 index 00000000..08d856dd --- /dev/null +++ b/src/app/api/internal/delete-runs/route.ts @@ -0,0 +1,127 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { checkBackgroundAuth } from '@/lib/background-function-auth'; +import { configure } from '@/cli/config'; +import { getLogger } from '@/utils/logger'; +import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService'; +import { actionBackfillSummary } from '@/cli/commands/backfill-summary'; +import { toSafeTimestamp } from '@/lib/timestampUtils'; + +const CONFIG_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,250}$/; +const FILE_NAME_RE = /^[A-Za-z0-9._-]+_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:-\d{3})?Z_comparison\.json$/; +const MAX_RUNS = 50; + +interface RunRef { configId: string; fileName: string } + +/** + * Removes specific published runs (e.g. ones whose judging failed), copying + * each to archive/deleted-runs// first so it can be put back (see + * archiveAndDeleteRun). Runs on the server, where the storage credentials + * live; triggered by the "Delete Runs" workflow. + * + * Every run is checked before anything is touched: it must exist, and it must + * not be its blueprint's latest run, so a blueprint always keeps the run its + * pages and leaderboards are built from. dryRun defaults to true and only + * reports what would be removed (200). A real run answers 202 and works in the + * background, logging under "delete-runs:"; with rebuildSummaries it then + * rebuilds the homepage, leaderboards and model summaries so the removed runs + * drop out of them. + */ +export async function POST(req: NextRequest) { + const authError = checkBackgroundAuth(req); + if (authError) return authError; + + let body: any; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON in request body.' }, { status: 400 }); + } + const runs: unknown = body?.runs; + const dryRun = body?.dryRun !== false; + const rebuildSummaries = body?.rebuildSummaries === true; + + if ( + !Array.isArray(runs) || runs.length === 0 || runs.length > MAX_RUNS || + !runs.every(r => typeof r?.configId === 'string' && CONFIG_ID_RE.test(r.configId) && + typeof r?.fileName === 'string' && FILE_NAME_RE.test(r.fileName)) + ) { + return NextResponse.json( + { error: `'runs' must be a list of 1-${MAX_RUNS} { configId, fileName } entries naming *_comparison.json run files.` }, + { status: 400 }, + ); + } + const refs = runs as RunRef[]; + + const problems: string[] = []; + const byConfig = new Map(); + for (const { configId, fileName } of refs) { + const listed = byConfig.get(configId) || []; + if (listed.includes(fileName)) problems.push(`${configId}/${fileName}: listed twice`); + byConfig.set(configId, [...listed, fileName]); + } + for (const [configId, fileNames] of byConfig) { + const existing = await listRunsForConfig(configId); + const latest = existing[0]?.fileName; + for (const fileName of fileNames) { + if (!existing.some(r => r.fileName === fileName)) problems.push(`${configId}/${fileName}: no such run`); + else if (fileName === latest) problems.push(`${configId}/${fileName}: is the latest run of ${configId}; refusing to delete it`); + } + } + if (problems.length > 0) { + return NextResponse.json({ error: 'Nothing was deleted.', problems }, { status: 400 }); + } + + const archiveName = toSafeTimestamp(new Date().toISOString()); + + if (dryRun) { + const plan = []; + for (const { configId, fileName } of refs) { + const { keys } = await archiveAndDeleteRun(configId, fileName, archiveName, true); + plan.push({ configId, fileName, objects: keys.length }); + } + return NextResponse.json({ dryRun: true, message: 'Nothing was deleted.', runs: plan }); + } + + const logger = await getLogger('delete-runs'); + configure({ + errorHandler: (error: Error) => logger.error(`CLI Error: ${error.message}`, error), + logger: { + info: (msg: string) => logger.info(msg), + warn: (msg: string) => logger.warn(msg), + error: (msg: string) => logger.error(msg), + success: (msg: string) => logger.info(msg), + }, + }); + + void runJob(refs, archiveName, rebuildSummaries, logger); + + return NextResponse.json( + { message: 'Accepted. Progress is logged under "delete-runs:".', runs: refs.length, archiveName, rebuildSummaries }, + { status: 202 }, + ); +} + +async function runJob(refs: RunRef[], archiveName: string, rebuildSummaries: boolean, logger: any): Promise { + let deleted = 0; + for (const [i, { configId, fileName }] of refs.entries()) { + const label = `(${i + 1}/${refs.length}) ${configId}/${fileName}`; + try { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, fileName, archiveName, false); + deleted++; + logger.info(`[DeleteRuns] ${label}: deleted ${keys.length} objects (copies in ${archivePrefix}).`); + } catch (error: any) { + logger.error(`[DeleteRuns] ${label}: failed, run left in place: ${error?.message || error}`); + } + } + logger.info(`[DeleteRuns] Done: ${deleted} of ${refs.length} runs deleted.`); + + if (rebuildSummaries) { + try { + logger.info('[DeleteRuns] Rebuilding homepage, leaderboards and model summaries...'); + await actionBackfillSummary({ verbose: false, dryRun: false }); + logger.info('[DeleteRuns] Summaries rebuilt.'); + } catch (error: any) { + logger.error(`[DeleteRuns] Summary rebuild failed: ${error?.message || error}`); + } + } +} diff --git a/src/lib/__tests__/archive-delete-run.test.ts b/src/lib/__tests__/archive-delete-run.test.ts new file mode 100644 index 00000000..6718aa8d --- /dev/null +++ b/src/lib/__tests__/archive-delete-run.test.ts @@ -0,0 +1,74 @@ +process.env.STORAGE_PROVIDER = 'local'; +import fs from 'fs/promises'; +import fsSync from 'fs'; +import path from 'path'; +import { saveResult, archiveAndDeleteRun, listRunsForConfig, DELETED_RUNS_ARCHIVE_DIR } from '@/lib/storageService'; +import { RESULTS_DIR, LIVE_DIR } from '@/cli/constants'; + +describe('archiveAndDeleteRun (local storage)', () => { + const configId = 'test_config_archive_delete'; + const archiveName = 'test-archive'; + const keep = 'keep_2024-01-02T00-00-00Z_comparison.json'; + const remove = 'remove_2024-01-01T00-00-00Z_comparison.json'; + const configDir = path.join(RESULTS_DIR, LIVE_DIR, 'blueprints', configId); + const archiveDir = path.join(RESULTS_DIR, DELETED_RUNS_ARCHIVE_DIR, archiveName); + + const sample = (runLabel: string, timestamp: string): any => ({ + configId, + configTitle: 'Sample', + runLabel, + timestamp, + config: { id: configId, title: 'Sample', models: ['modelA'], prompts: [] }, + effectiveModels: ['modelA'], + promptIds: ['p1'], + allFinalAssistantResponses: { p1: { modelA: 'Hello' } }, + evaluationResults: { + llmCoverageScores: { p1: { modelA: { avgCoverageExtent: 0.5, pointAssessments: [] } } }, + similarityMatrix: {}, + }, + }); + + beforeEach(async () => { + await fs.rm(configDir, { recursive: true, force: true }); + await fs.rm(archiveDir, { recursive: true, force: true }); + await saveResult(configId, keep, sample('keep', '2024-01-02T00-00-00Z')); + await saveResult(configId, remove, sample('remove', '2024-01-01T00-00-00Z')); + }); + + afterAll(async () => { + await fs.rm(configDir, { recursive: true, force: true }); + await fs.rm(archiveDir, { recursive: true, force: true }); + }); + + it('lists the comparison file and its artefacts on a dry run, and leaves them in place', async () => { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, true); + const runDir = path.join(LIVE_DIR, 'blueprints', configId, 'remove_2024-01-01T00-00-00Z'); + + expect(archivePrefix).toBeNull(); + expect(keys[0]).toBe(path.join(LIVE_DIR, 'blueprints', configId, remove)); + expect(keys).toContain(path.join(runDir, 'core.json')); + expect(keys.slice(1).every(k => k.startsWith(runDir + path.sep))).toBe(true); + expect(fsSync.existsSync(path.join(RESULTS_DIR, keys[0]))).toBe(true); + expect(fsSync.existsSync(archiveDir)).toBe(false); + }); + + it('copies the run to the archive, then deletes it and only it', async () => { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, false); + + expect(archivePrefix).toBe(path.join(DELETED_RUNS_ARCHIVE_DIR, archiveName)); + for (const key of keys) { + expect(fsSync.existsSync(path.join(RESULTS_DIR, key))).toBe(false); + expect(fsSync.existsSync(path.join(RESULTS_DIR, archivePrefix!, key))).toBe(true); + } + expect(fsSync.existsSync(path.join(configDir, 'remove_2024-01-01T00-00-00Z'))).toBe(false); + expect((await listRunsForConfig(configId)).map(r => r.fileName)).toEqual([keep]); + expect(fsSync.existsSync(path.join(configDir, 'keep_2024-01-02T00-00-00Z', 'core.json'))).toBe(true); + }); + + it('throws for a run that does not exist, and for names that are not run files', async () => { + await expect(archiveAndDeleteRun(configId, 'missing_2024-01-03T00-00-00Z_comparison.json', archiveName, false)) + .rejects.toThrow('Run not found'); + await expect(archiveAndDeleteRun(configId, `../${keep}`, archiveName, false)).rejects.toThrow('Not a run file name'); + expect((await listRunsForConfig(configId)).length).toBe(2); + }); +}); diff --git a/src/lib/__tests__/storageService.test.ts b/src/lib/__tests__/storageService.test.ts index 28bc40f3..32ce695b 100644 --- a/src/lib/__tests__/storageService.test.ts +++ b/src/lib/__tests__/storageService.test.ts @@ -622,4 +622,69 @@ describe('storageService', () => { }); }); }); + + describe('archiveAndDeleteRun (S3)', () => { + const fileName = 'run-old_2024-01-15T10-00-00-000Z_comparison.json'; + const base = 'live/blueprints/test-config'; + const runDir = `${base}/run-old_2024-01-15T10-00-00-000Z`; + const keys = [`${base}/${fileName}`, `${runDir}/core.json`, `${runDir}/coverage/p 1/m[temp:0].json`]; + const sentOf = (name: string) => mockSend.mock.calls.map(c => c[0]).filter((cmd: any) => cmd.constructor.name === name); + + const setUp = async (overrides: Record any> = {}) => { + process.env.STORAGE_PROVIDER = 's3'; + process.env.APP_S3_BUCKET_NAME = 'test-bucket'; + process.env.APP_S3_REGION = 'us-east-1'; + mockedFs.rm.mockResolvedValue(undefined); + let listCalls = 0; + mockSend.mockImplementation(async (cmd: any) => { + const name = cmd.constructor.name; + if (overrides[name]) return overrides[name](); + if (name === 'ListObjectsV2Command') { + // Two pages, to check the listing follows continuation tokens. + return listCalls++ === 0 + ? { Contents: [{ Key: keys[1] }], NextContinuationToken: 'next' } + : { Contents: [{ Key: keys[2] }] }; + } + return {}; + }); + return (await import('../storageService')).archiveAndDeleteRun; + }; + + it('lists the comparison file and every artefact page on a dry run, without copying or deleting', async () => { + const archiveAndDeleteRun = await setUp(); + const result = await archiveAndDeleteRun('test-config', fileName, 'a1', true); + + expect(result).toEqual({ keys, archivePrefix: null }); + expect((sentOf('ListObjectsV2Command')[0] as any).input.Prefix).toBe(`${runDir}/`); + expect(sentOf('CopyObjectCommand')).toHaveLength(0); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + + it('copies every object to the archive with an encoded source, then deletes them', async () => { + const archiveAndDeleteRun = await setUp(); + const result = await archiveAndDeleteRun('test-config', fileName, 'a1', false); + + expect(result.archivePrefix).toBe('archive/deleted-runs/a1'); + const copies = sentOf('CopyObjectCommand').map((cmd: any) => cmd.input); + expect(copies.map(c => c.Key)).toEqual(keys.map(k => `archive/deleted-runs/a1/${k}`)); + expect(copies[2].CopySource).toBe(`test-bucket/${runDir}/coverage/p%201/m%5Btemp%3A0%5D.json`); + const deletes = sentOf('DeleteObjectsCommand').map((cmd: any) => cmd.input.Delete.Objects.map((o: any) => o.Key)); + expect(deletes).toEqual([keys]); + }); + + it('deletes nothing if any copy fails', async () => { + const archiveAndDeleteRun = await setUp({ CopyObjectCommand: () => { throw new Error('AccessDenied'); } }); + await expect(archiveAndDeleteRun('test-config', fileName, 'a1', false)).rejects.toThrow('AccessDenied'); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + + it('refuses a run whose comparison file is missing', async () => { + const archiveAndDeleteRun = await setUp({ + HeadObjectCommand: () => { throw Object.assign(new Error('Not Found'), { name: 'NotFound' }); }, + }); + await expect(archiveAndDeleteRun('test-config', fileName, 'a1', false)).rejects.toThrow('Run not found'); + expect(sentOf('ListObjectsV2Command')).toHaveLength(0); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + }); }); \ No newline at end of file diff --git a/src/lib/storageService.ts b/src/lib/storageService.ts index 0de7e3dc..2c7467d5 100644 --- a/src/lib/storageService.ts +++ b/src/lib/storageService.ts @@ -1765,6 +1765,127 @@ export async function deleteResultByFileName(configId: string, fileName: string) } } +/** Where archiveAndDeleteRun keeps copies of the runs it removes. */ +export const DELETED_RUNS_ARCHIVE_DIR = path.join('archive', 'deleted-runs'); + +export interface ArchiveAndDeleteRunResult { + /** Storage keys that make up the run: its comparison file, then its artefacts. */ + keys: string[]; + /** Where the run was copied before deletion; null on a dry run. */ + archivePrefix: string | null; +} + +async function listFilesUnder(dir: string): Promise { + const files: string[] = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) files.push(...await listFilesUnder(entryPath)); + else files.push(entryPath); + } + return files; +} + +/** + * Removes one run of a blueprint: its comparison file and the folder of + * artefacts beside it (core.json, responses, coverage, histories). + * + * Every object is first copied to archive/deleted-runs// under its + * original key, so the run can be put back by copying those objects back, and + * nothing is deleted unless every copy succeeded. The archive sits outside + * backups/ on purpose: restore-data replaces all live data with what a backup + * holds, and a one-run archive must never be mistaken for a full backup. + * + * With dryRun, only lists the keys. Throws if the comparison file is missing. + * Summaries still list the run until they are rebuilt (actionBackfillSummary). + */ +export async function archiveAndDeleteRun( + configId: string, + fileName: string, + archiveName: string, + dryRun: boolean, +): Promise { + if (!/^[^/\\]+_comparison\.json$/.test(fileName)) { + throw new Error(`Not a run file name: ${fileName}`); + } + const basePath = getConfigBasePath(configId); + const runBase = fileName.replace(/_comparison\.json$/, ''); + const comparisonKey = path.join(basePath, fileName); + const artefactPrefix = path.join(basePath, runBase); + const archivePrefix = path.join(DELETED_RUNS_ARCHIVE_DIR, archiveName); + let keys: string[]; + + if (storageProvider === 's3' && s3Client && s3BucketName) { + const client = s3Client; + const bucket = s3BucketName; + try { + await client.send(new HeadObjectCommand({ Bucket: bucket, Key: comparisonKey })); + } catch (error: any) { + if (error?.name === 'NotFound' || error?.$metadata?.httpStatusCode === 404) { + throw new Error(`Run not found: ${comparisonKey}`); + } + throw error; + } + + const artefactKeys: string[] = []; + let continuationToken: string | undefined = undefined; + do { + const response: ListObjectsV2CommandOutput = await client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `${artefactPrefix}/`, + ContinuationToken: continuationToken, + })); + for (const obj of response.Contents || []) if (obj.Key) artefactKeys.push(obj.Key); + continuationToken = response.NextContinuationToken; + } while (continuationToken); + keys = [comparisonKey, ...artefactKeys]; + if (dryRun) return { keys, archivePrefix: null }; + + // CopySource must be URL-encoded; prompt and model IDs in artefact keys can hold any character. + const encodeKey = (key: string) => key.split('/').map(encodeURIComponent).join('/'); + const limit = pLimit(20); + await Promise.all(keys.map(key => limit(() => client.send(new CopyObjectCommand({ + Bucket: bucket, + CopySource: `${bucket}/${encodeKey(key)}`, + Key: path.join(archivePrefix, key), + }))))); + + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + const response = await client.send(new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: batch.map(Key => ({ Key })), Quiet: true }, + })); + if (response.Errors && response.Errors.length > 0) { + const first = response.Errors[0]; + throw new Error(`Failed to delete ${response.Errors.length} object(s) of ${comparisonKey}, e.g. ${first.Key}: ${first.Message}`); + } + } + } else if (storageProvider === 'local') { + const comparisonPath = path.join(RESULTS_DIR, comparisonKey); + if (!fsSync.existsSync(comparisonPath)) throw new Error(`Run not found: ${comparisonKey}`); + const artefactDir = path.join(RESULTS_DIR, artefactPrefix); + const artefactKeys = fsSync.existsSync(artefactDir) + ? (await listFilesUnder(artefactDir)).map(file => path.join(artefactPrefix, path.relative(artefactDir, file))) + : []; + keys = [comparisonKey, ...artefactKeys]; + if (dryRun) return { keys, archivePrefix: null }; + + for (const key of keys) { + const dest = path.join(RESULTS_DIR, archivePrefix, key); + await fs.mkdir(path.dirname(dest), { recursive: true }); + await fs.copyFile(path.join(RESULTS_DIR, key), dest); + } + await fs.unlink(comparisonPath); + await fs.rm(artefactDir, { recursive: true, force: true }); + } else { + throw new Error('No valid storage provider configured; nothing deleted.'); + } + + // Drop this server's cached copies of the run's artefacts. + await fs.rm(path.join(CACHE_DIR, configId, runBase), { recursive: true, force: true }).catch(() => {}); + return { keys, archivePrefix }; +} + export async function getLatestRunsSummary(): Promise { let fileContent: string | null = null; const s3Key = path.join(LIVE_DIR, 'aggregates', 'latest_runs_summary.json');