diff --git a/.github/workflows/delete-runs.yml b/.github/workflows/delete-runs.yml new file mode 100644 index 0000000..dc0cb79 --- /dev/null +++ b/.github/workflows/delete-runs.yml @@ -0,0 +1,47 @@ +name: Delete Runs +on: + workflow_dispatch: + inputs: + runs: + description: 'Comma-separated runs to delete, each as configId/fileName (e.g. yka-set/370f988e819882dc_2026-10-01T16-25-01-568Z_comparison.json)' + required: true + dry_run: + description: 'Only list what would be deleted (untick to delete; each run is copied to archive/deleted-runs/ first)' + type: boolean + default: true + rebuild_summaries: + description: 'After deleting, rebuild homepage, leaderboards and model summaries so the runs drop out of them' + type: boolean + default: true + +jobs: + start: + runs-on: ubuntu-latest + steps: + - name: Send the request to the server + env: + RUNS: ${{ github.event.inputs.runs }} + DRY_RUN: ${{ github.event.inputs.dry_run }} + REBUILD: ${{ github.event.inputs.rebuild_summaries }} + run: | + BODY=$(python3 -c 'import json, os; runs = [x.strip() for x in os.environ["RUNS"].split(",") if x.strip()]; print(json.dumps({"runs": [{"configId": r.split("/", 1)[0], "fileName": r.split("/", 1)[-1]} for r in runs], "dryRun": os.environ["DRY_RUN"] != "false", "rebuildSummaries": os.environ["REBUILD"] == "true"}))') + # RAILWAY_APP_URL may redirect (http -> https, www -> apex); follow it as a POST. + STATUS=$(curl -sS -L --post301 --post302 --post303 --max-time 300 -o response.txt -w '%{http_code}' -X POST \ + -H "Content-Type: application/json" \ + -H "X-Background-Function-Auth-Token: ${{ secrets.BACKGROUND_FUNCTION_AUTH_TOKEN }}" \ + -d "$BODY" \ + "${{ secrets.RAILWAY_APP_URL }}/api/internal/delete-runs") + python3 -m json.tool response.txt || cat response.txt + if [ "$DRY_RUN" != "false" ]; then + if [ "$STATUS" != "200" ]; then + echo "::error::Expected 200 from the server, got $STATUS" + exit 1 + fi + echo "::notice::Dry run only; nothing was deleted." + else + if [ "$STATUS" != "202" ]; then + echo "::error::Expected 202 from the server, got $STATUS" + exit 1 + fi + echo "::notice::Started. Progress is in the Railway logs under 'delete-runs:' / '[DeleteRuns]'." + fi diff --git a/src/app/api/internal/delete-runs/__tests__/route.test.ts b/src/app/api/internal/delete-runs/__tests__/route.test.ts new file mode 100644 index 0000000..5df8adf --- /dev/null +++ b/src/app/api/internal/delete-runs/__tests__/route.test.ts @@ -0,0 +1,110 @@ +/** + * @vitest-environment node + */ +import { vi } from 'vitest'; +import { NextRequest, NextResponse } from 'next/server'; + +vi.mock('@/lib/storageService', () => ({ archiveAndDeleteRun: vi.fn(), listRunsForConfig: vi.fn() })); +vi.mock('@/cli/commands/backfill-summary', () => ({ actionBackfillSummary: vi.fn() })); +vi.mock('@/lib/background-function-auth', () => ({ checkBackgroundAuth: vi.fn(() => null) })); +vi.mock('@/utils/logger', () => ({ + getLogger: vi.fn(async () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })), +})); + +import { POST } from '../route'; +import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService'; +import { actionBackfillSummary } from '@/cli/commands/backfill-summary'; +import { checkBackgroundAuth } from '@/lib/background-function-auth'; + +const LATEST = 'abc_2026-10-01T20-11-28-654Z_comparison.json'; +const OLD_1 = 'abc_2026-10-01T19-47-15-945Z_comparison.json'; +const OLD_2 = 'abc_2026-10-01T05-36-17-308Z_comparison.json'; +const request = (body: unknown) => + new NextRequest('http://localhost:3172/api/internal/delete-runs', { method: 'POST', body: JSON.stringify(body) }); +const flush = () => new Promise(r => setTimeout(r, 0)); +const run = (fileName: string, configId = 'yka-set') => ({ configId, fileName }); + +describe('POST /api/internal/delete-runs', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listRunsForConfig).mockResolvedValue( + [LATEST, OLD_1, OLD_2].map(fileName => ({ runLabel: 'abc', timestamp: null, fileName })), + ); + vi.mocked(archiveAndDeleteRun).mockImplementation(async (configId, fileName, archiveName, dryRun) => ({ + keys: [`live/blueprints/${configId}/${fileName}`, `live/blueprints/${configId}/x/core.json`], + archivePrefix: dryRun ? null : `archive/deleted-runs/${archiveName}`, + })); + }); + + it('defaults to a dry run that lists the runs and deletes nothing', async () => { + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)] })); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.dryRun).toBe(true); + expect(body.runs).toEqual([ + { configId: 'yka-set', fileName: OLD_1, objects: 2 }, + { configId: 'yka-set', fileName: OLD_2, objects: 2 }, + ]); + expect(vi.mocked(archiveAndDeleteRun).mock.calls.every(c => c[3] === true)).toBe(true); + expect(actionBackfillSummary).not.toHaveBeenCalled(); + }); + + it('deletes each run in the background when dryRun is false, then rebuilds when asked', async () => { + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false, rebuildSummaries: true })); + expect(res.status).toBe(202); + await flush(); + + expect(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => [c[1], c[3]])).toEqual([[OLD_1, false], [OLD_2, false]]); + const archiveNames = new Set(vi.mocked(archiveAndDeleteRun).mock.calls.map(c => c[2])); + expect(archiveNames.size).toBe(1); + expect(actionBackfillSummary).toHaveBeenCalledTimes(1); + }); + + it('keeps going after a run fails, and does not rebuild unless asked', async () => { + vi.mocked(archiveAndDeleteRun).mockRejectedValueOnce(new Error('copy failed')); + + const res = await POST(request({ runs: [run(OLD_1), run(OLD_2)], dryRun: false })); + expect(res.status).toBe(202); + await flush(); + + expect(archiveAndDeleteRun).toHaveBeenCalledTimes(2); + expect(actionBackfillSummary).not.toHaveBeenCalled(); + }); + + it("refuses to delete a blueprint's latest run, and touches nothing", async () => { + const res = await POST(request({ runs: [run(OLD_1), run(LATEST)], dryRun: false })); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.problems).toEqual([`yka-set/${LATEST}: is the latest run of yka-set; refusing to delete it`]); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it('refuses runs that do not exist or are listed twice', async () => { + const missing = 'abc_2020-01-01T00-00-00Z_comparison.json'; + const res = await POST(request({ runs: [run(missing), run(OLD_1), run(OLD_1)], dryRun: false })); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.problems).toEqual([`yka-set/${OLD_1}: listed twice`, `yka-set/${missing}: no such run`]); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it('rejects unauthenticated requests', async () => { + vi.mocked(checkBackgroundAuth).mockReturnValueOnce(NextResponse.json({ error: 'Unauthorized' }, { status: 401 })); + const res = await POST(request({ runs: [run(OLD_1)] })); + expect(res.status).toBe(401); + expect(listRunsForConfig).not.toHaveBeenCalled(); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); + + it.each([ + ['no runs', { runs: [] }], + ['runs not a list', { runs: 'yka-set' }], + ['a path in the file name', { runs: [run(`../${OLD_1}`)] }], + ['not a run file', { runs: [run('summary.json')] }], + ['a bad config ID', { runs: [run(OLD_1, '../live')] }], + ])('rejects %s', async (_label, body) => { + const res = await POST(request(body)); + expect(res.status).toBe(400); + expect(archiveAndDeleteRun).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/internal/delete-runs/route.ts b/src/app/api/internal/delete-runs/route.ts new file mode 100644 index 0000000..08d856d --- /dev/null +++ b/src/app/api/internal/delete-runs/route.ts @@ -0,0 +1,127 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { checkBackgroundAuth } from '@/lib/background-function-auth'; +import { configure } from '@/cli/config'; +import { getLogger } from '@/utils/logger'; +import { archiveAndDeleteRun, listRunsForConfig } from '@/lib/storageService'; +import { actionBackfillSummary } from '@/cli/commands/backfill-summary'; +import { toSafeTimestamp } from '@/lib/timestampUtils'; + +const CONFIG_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,250}$/; +const FILE_NAME_RE = /^[A-Za-z0-9._-]+_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:-\d{3})?Z_comparison\.json$/; +const MAX_RUNS = 50; + +interface RunRef { configId: string; fileName: string } + +/** + * Removes specific published runs (e.g. ones whose judging failed), copying + * each to archive/deleted-runs// first so it can be put back (see + * archiveAndDeleteRun). Runs on the server, where the storage credentials + * live; triggered by the "Delete Runs" workflow. + * + * Every run is checked before anything is touched: it must exist, and it must + * not be its blueprint's latest run, so a blueprint always keeps the run its + * pages and leaderboards are built from. dryRun defaults to true and only + * reports what would be removed (200). A real run answers 202 and works in the + * background, logging under "delete-runs:"; with rebuildSummaries it then + * rebuilds the homepage, leaderboards and model summaries so the removed runs + * drop out of them. + */ +export async function POST(req: NextRequest) { + const authError = checkBackgroundAuth(req); + if (authError) return authError; + + let body: any; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON in request body.' }, { status: 400 }); + } + const runs: unknown = body?.runs; + const dryRun = body?.dryRun !== false; + const rebuildSummaries = body?.rebuildSummaries === true; + + if ( + !Array.isArray(runs) || runs.length === 0 || runs.length > MAX_RUNS || + !runs.every(r => typeof r?.configId === 'string' && CONFIG_ID_RE.test(r.configId) && + typeof r?.fileName === 'string' && FILE_NAME_RE.test(r.fileName)) + ) { + return NextResponse.json( + { error: `'runs' must be a list of 1-${MAX_RUNS} { configId, fileName } entries naming *_comparison.json run files.` }, + { status: 400 }, + ); + } + const refs = runs as RunRef[]; + + const problems: string[] = []; + const byConfig = new Map(); + for (const { configId, fileName } of refs) { + const listed = byConfig.get(configId) || []; + if (listed.includes(fileName)) problems.push(`${configId}/${fileName}: listed twice`); + byConfig.set(configId, [...listed, fileName]); + } + for (const [configId, fileNames] of byConfig) { + const existing = await listRunsForConfig(configId); + const latest = existing[0]?.fileName; + for (const fileName of fileNames) { + if (!existing.some(r => r.fileName === fileName)) problems.push(`${configId}/${fileName}: no such run`); + else if (fileName === latest) problems.push(`${configId}/${fileName}: is the latest run of ${configId}; refusing to delete it`); + } + } + if (problems.length > 0) { + return NextResponse.json({ error: 'Nothing was deleted.', problems }, { status: 400 }); + } + + const archiveName = toSafeTimestamp(new Date().toISOString()); + + if (dryRun) { + const plan = []; + for (const { configId, fileName } of refs) { + const { keys } = await archiveAndDeleteRun(configId, fileName, archiveName, true); + plan.push({ configId, fileName, objects: keys.length }); + } + return NextResponse.json({ dryRun: true, message: 'Nothing was deleted.', runs: plan }); + } + + const logger = await getLogger('delete-runs'); + configure({ + errorHandler: (error: Error) => logger.error(`CLI Error: ${error.message}`, error), + logger: { + info: (msg: string) => logger.info(msg), + warn: (msg: string) => logger.warn(msg), + error: (msg: string) => logger.error(msg), + success: (msg: string) => logger.info(msg), + }, + }); + + void runJob(refs, archiveName, rebuildSummaries, logger); + + return NextResponse.json( + { message: 'Accepted. Progress is logged under "delete-runs:".', runs: refs.length, archiveName, rebuildSummaries }, + { status: 202 }, + ); +} + +async function runJob(refs: RunRef[], archiveName: string, rebuildSummaries: boolean, logger: any): Promise { + let deleted = 0; + for (const [i, { configId, fileName }] of refs.entries()) { + const label = `(${i + 1}/${refs.length}) ${configId}/${fileName}`; + try { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, fileName, archiveName, false); + deleted++; + logger.info(`[DeleteRuns] ${label}: deleted ${keys.length} objects (copies in ${archivePrefix}).`); + } catch (error: any) { + logger.error(`[DeleteRuns] ${label}: failed, run left in place: ${error?.message || error}`); + } + } + logger.info(`[DeleteRuns] Done: ${deleted} of ${refs.length} runs deleted.`); + + if (rebuildSummaries) { + try { + logger.info('[DeleteRuns] Rebuilding homepage, leaderboards and model summaries...'); + await actionBackfillSummary({ verbose: false, dryRun: false }); + logger.info('[DeleteRuns] Summaries rebuilt.'); + } catch (error: any) { + logger.error(`[DeleteRuns] Summary rebuild failed: ${error?.message || error}`); + } + } +} diff --git a/src/lib/__tests__/archive-delete-run.test.ts b/src/lib/__tests__/archive-delete-run.test.ts new file mode 100644 index 0000000..6718aa8 --- /dev/null +++ b/src/lib/__tests__/archive-delete-run.test.ts @@ -0,0 +1,74 @@ +process.env.STORAGE_PROVIDER = 'local'; +import fs from 'fs/promises'; +import fsSync from 'fs'; +import path from 'path'; +import { saveResult, archiveAndDeleteRun, listRunsForConfig, DELETED_RUNS_ARCHIVE_DIR } from '@/lib/storageService'; +import { RESULTS_DIR, LIVE_DIR } from '@/cli/constants'; + +describe('archiveAndDeleteRun (local storage)', () => { + const configId = 'test_config_archive_delete'; + const archiveName = 'test-archive'; + const keep = 'keep_2024-01-02T00-00-00Z_comparison.json'; + const remove = 'remove_2024-01-01T00-00-00Z_comparison.json'; + const configDir = path.join(RESULTS_DIR, LIVE_DIR, 'blueprints', configId); + const archiveDir = path.join(RESULTS_DIR, DELETED_RUNS_ARCHIVE_DIR, archiveName); + + const sample = (runLabel: string, timestamp: string): any => ({ + configId, + configTitle: 'Sample', + runLabel, + timestamp, + config: { id: configId, title: 'Sample', models: ['modelA'], prompts: [] }, + effectiveModels: ['modelA'], + promptIds: ['p1'], + allFinalAssistantResponses: { p1: { modelA: 'Hello' } }, + evaluationResults: { + llmCoverageScores: { p1: { modelA: { avgCoverageExtent: 0.5, pointAssessments: [] } } }, + similarityMatrix: {}, + }, + }); + + beforeEach(async () => { + await fs.rm(configDir, { recursive: true, force: true }); + await fs.rm(archiveDir, { recursive: true, force: true }); + await saveResult(configId, keep, sample('keep', '2024-01-02T00-00-00Z')); + await saveResult(configId, remove, sample('remove', '2024-01-01T00-00-00Z')); + }); + + afterAll(async () => { + await fs.rm(configDir, { recursive: true, force: true }); + await fs.rm(archiveDir, { recursive: true, force: true }); + }); + + it('lists the comparison file and its artefacts on a dry run, and leaves them in place', async () => { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, true); + const runDir = path.join(LIVE_DIR, 'blueprints', configId, 'remove_2024-01-01T00-00-00Z'); + + expect(archivePrefix).toBeNull(); + expect(keys[0]).toBe(path.join(LIVE_DIR, 'blueprints', configId, remove)); + expect(keys).toContain(path.join(runDir, 'core.json')); + expect(keys.slice(1).every(k => k.startsWith(runDir + path.sep))).toBe(true); + expect(fsSync.existsSync(path.join(RESULTS_DIR, keys[0]))).toBe(true); + expect(fsSync.existsSync(archiveDir)).toBe(false); + }); + + it('copies the run to the archive, then deletes it and only it', async () => { + const { keys, archivePrefix } = await archiveAndDeleteRun(configId, remove, archiveName, false); + + expect(archivePrefix).toBe(path.join(DELETED_RUNS_ARCHIVE_DIR, archiveName)); + for (const key of keys) { + expect(fsSync.existsSync(path.join(RESULTS_DIR, key))).toBe(false); + expect(fsSync.existsSync(path.join(RESULTS_DIR, archivePrefix!, key))).toBe(true); + } + expect(fsSync.existsSync(path.join(configDir, 'remove_2024-01-01T00-00-00Z'))).toBe(false); + expect((await listRunsForConfig(configId)).map(r => r.fileName)).toEqual([keep]); + expect(fsSync.existsSync(path.join(configDir, 'keep_2024-01-02T00-00-00Z', 'core.json'))).toBe(true); + }); + + it('throws for a run that does not exist, and for names that are not run files', async () => { + await expect(archiveAndDeleteRun(configId, 'missing_2024-01-03T00-00-00Z_comparison.json', archiveName, false)) + .rejects.toThrow('Run not found'); + await expect(archiveAndDeleteRun(configId, `../${keep}`, archiveName, false)).rejects.toThrow('Not a run file name'); + expect((await listRunsForConfig(configId)).length).toBe(2); + }); +}); diff --git a/src/lib/__tests__/storageService.test.ts b/src/lib/__tests__/storageService.test.ts index 28bc40f..32ce695 100644 --- a/src/lib/__tests__/storageService.test.ts +++ b/src/lib/__tests__/storageService.test.ts @@ -622,4 +622,69 @@ describe('storageService', () => { }); }); }); + + describe('archiveAndDeleteRun (S3)', () => { + const fileName = 'run-old_2024-01-15T10-00-00-000Z_comparison.json'; + const base = 'live/blueprints/test-config'; + const runDir = `${base}/run-old_2024-01-15T10-00-00-000Z`; + const keys = [`${base}/${fileName}`, `${runDir}/core.json`, `${runDir}/coverage/p 1/m[temp:0].json`]; + const sentOf = (name: string) => mockSend.mock.calls.map(c => c[0]).filter((cmd: any) => cmd.constructor.name === name); + + const setUp = async (overrides: Record any> = {}) => { + process.env.STORAGE_PROVIDER = 's3'; + process.env.APP_S3_BUCKET_NAME = 'test-bucket'; + process.env.APP_S3_REGION = 'us-east-1'; + mockedFs.rm.mockResolvedValue(undefined); + let listCalls = 0; + mockSend.mockImplementation(async (cmd: any) => { + const name = cmd.constructor.name; + if (overrides[name]) return overrides[name](); + if (name === 'ListObjectsV2Command') { + // Two pages, to check the listing follows continuation tokens. + return listCalls++ === 0 + ? { Contents: [{ Key: keys[1] }], NextContinuationToken: 'next' } + : { Contents: [{ Key: keys[2] }] }; + } + return {}; + }); + return (await import('../storageService')).archiveAndDeleteRun; + }; + + it('lists the comparison file and every artefact page on a dry run, without copying or deleting', async () => { + const archiveAndDeleteRun = await setUp(); + const result = await archiveAndDeleteRun('test-config', fileName, 'a1', true); + + expect(result).toEqual({ keys, archivePrefix: null }); + expect((sentOf('ListObjectsV2Command')[0] as any).input.Prefix).toBe(`${runDir}/`); + expect(sentOf('CopyObjectCommand')).toHaveLength(0); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + + it('copies every object to the archive with an encoded source, then deletes them', async () => { + const archiveAndDeleteRun = await setUp(); + const result = await archiveAndDeleteRun('test-config', fileName, 'a1', false); + + expect(result.archivePrefix).toBe('archive/deleted-runs/a1'); + const copies = sentOf('CopyObjectCommand').map((cmd: any) => cmd.input); + expect(copies.map(c => c.Key)).toEqual(keys.map(k => `archive/deleted-runs/a1/${k}`)); + expect(copies[2].CopySource).toBe(`test-bucket/${runDir}/coverage/p%201/m%5Btemp%3A0%5D.json`); + const deletes = sentOf('DeleteObjectsCommand').map((cmd: any) => cmd.input.Delete.Objects.map((o: any) => o.Key)); + expect(deletes).toEqual([keys]); + }); + + it('deletes nothing if any copy fails', async () => { + const archiveAndDeleteRun = await setUp({ CopyObjectCommand: () => { throw new Error('AccessDenied'); } }); + await expect(archiveAndDeleteRun('test-config', fileName, 'a1', false)).rejects.toThrow('AccessDenied'); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + + it('refuses a run whose comparison file is missing', async () => { + const archiveAndDeleteRun = await setUp({ + HeadObjectCommand: () => { throw Object.assign(new Error('Not Found'), { name: 'NotFound' }); }, + }); + await expect(archiveAndDeleteRun('test-config', fileName, 'a1', false)).rejects.toThrow('Run not found'); + expect(sentOf('ListObjectsV2Command')).toHaveLength(0); + expect(sentOf('DeleteObjectsCommand')).toHaveLength(0); + }); + }); }); \ No newline at end of file diff --git a/src/lib/storageService.ts b/src/lib/storageService.ts index 0de7e3d..2c7467d 100644 --- a/src/lib/storageService.ts +++ b/src/lib/storageService.ts @@ -1765,6 +1765,127 @@ export async function deleteResultByFileName(configId: string, fileName: string) } } +/** Where archiveAndDeleteRun keeps copies of the runs it removes. */ +export const DELETED_RUNS_ARCHIVE_DIR = path.join('archive', 'deleted-runs'); + +export interface ArchiveAndDeleteRunResult { + /** Storage keys that make up the run: its comparison file, then its artefacts. */ + keys: string[]; + /** Where the run was copied before deletion; null on a dry run. */ + archivePrefix: string | null; +} + +async function listFilesUnder(dir: string): Promise { + const files: string[] = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) files.push(...await listFilesUnder(entryPath)); + else files.push(entryPath); + } + return files; +} + +/** + * Removes one run of a blueprint: its comparison file and the folder of + * artefacts beside it (core.json, responses, coverage, histories). + * + * Every object is first copied to archive/deleted-runs// under its + * original key, so the run can be put back by copying those objects back, and + * nothing is deleted unless every copy succeeded. The archive sits outside + * backups/ on purpose: restore-data replaces all live data with what a backup + * holds, and a one-run archive must never be mistaken for a full backup. + * + * With dryRun, only lists the keys. Throws if the comparison file is missing. + * Summaries still list the run until they are rebuilt (actionBackfillSummary). + */ +export async function archiveAndDeleteRun( + configId: string, + fileName: string, + archiveName: string, + dryRun: boolean, +): Promise { + if (!/^[^/\\]+_comparison\.json$/.test(fileName)) { + throw new Error(`Not a run file name: ${fileName}`); + } + const basePath = getConfigBasePath(configId); + const runBase = fileName.replace(/_comparison\.json$/, ''); + const comparisonKey = path.join(basePath, fileName); + const artefactPrefix = path.join(basePath, runBase); + const archivePrefix = path.join(DELETED_RUNS_ARCHIVE_DIR, archiveName); + let keys: string[]; + + if (storageProvider === 's3' && s3Client && s3BucketName) { + const client = s3Client; + const bucket = s3BucketName; + try { + await client.send(new HeadObjectCommand({ Bucket: bucket, Key: comparisonKey })); + } catch (error: any) { + if (error?.name === 'NotFound' || error?.$metadata?.httpStatusCode === 404) { + throw new Error(`Run not found: ${comparisonKey}`); + } + throw error; + } + + const artefactKeys: string[] = []; + let continuationToken: string | undefined = undefined; + do { + const response: ListObjectsV2CommandOutput = await client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `${artefactPrefix}/`, + ContinuationToken: continuationToken, + })); + for (const obj of response.Contents || []) if (obj.Key) artefactKeys.push(obj.Key); + continuationToken = response.NextContinuationToken; + } while (continuationToken); + keys = [comparisonKey, ...artefactKeys]; + if (dryRun) return { keys, archivePrefix: null }; + + // CopySource must be URL-encoded; prompt and model IDs in artefact keys can hold any character. + const encodeKey = (key: string) => key.split('/').map(encodeURIComponent).join('/'); + const limit = pLimit(20); + await Promise.all(keys.map(key => limit(() => client.send(new CopyObjectCommand({ + Bucket: bucket, + CopySource: `${bucket}/${encodeKey(key)}`, + Key: path.join(archivePrefix, key), + }))))); + + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + const response = await client.send(new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: batch.map(Key => ({ Key })), Quiet: true }, + })); + if (response.Errors && response.Errors.length > 0) { + const first = response.Errors[0]; + throw new Error(`Failed to delete ${response.Errors.length} object(s) of ${comparisonKey}, e.g. ${first.Key}: ${first.Message}`); + } + } + } else if (storageProvider === 'local') { + const comparisonPath = path.join(RESULTS_DIR, comparisonKey); + if (!fsSync.existsSync(comparisonPath)) throw new Error(`Run not found: ${comparisonKey}`); + const artefactDir = path.join(RESULTS_DIR, artefactPrefix); + const artefactKeys = fsSync.existsSync(artefactDir) + ? (await listFilesUnder(artefactDir)).map(file => path.join(artefactPrefix, path.relative(artefactDir, file))) + : []; + keys = [comparisonKey, ...artefactKeys]; + if (dryRun) return { keys, archivePrefix: null }; + + for (const key of keys) { + const dest = path.join(RESULTS_DIR, archivePrefix, key); + await fs.mkdir(path.dirname(dest), { recursive: true }); + await fs.copyFile(path.join(RESULTS_DIR, key), dest); + } + await fs.unlink(comparisonPath); + await fs.rm(artefactDir, { recursive: true, force: true }); + } else { + throw new Error('No valid storage provider configured; nothing deleted.'); + } + + // Drop this server's cached copies of the run's artefacts. + await fs.rm(path.join(CACHE_DIR, configId, runBase), { recursive: true, force: true }).catch(() => {}); + return { keys, archivePrefix }; +} + export async function getLatestRunsSummary(): Promise { let fileContent: string | null = null; const s3Key = path.join(LIVE_DIR, 'aggregates', 'latest_runs_summary.json');