diff --git a/benchmarks/codex-mcp/package.json b/benchmarks/codex-mcp/package.json index f0643007..2621e83a 100644 --- a/benchmarks/codex-mcp/package.json +++ b/benchmarks/codex-mcp/package.json @@ -7,11 +7,14 @@ "node": ">=22.22.0" }, "scripts": { - "check": "promptfoo validate -c promptfooconfig.yaml", - "preflight": "node scripts/preflight.mjs", - "eval:smoke": "npm run preflight && promptfoo eval -c promptfooconfig.yaml --filter-first-n 2 --repeat 1 --no-cache --no-share", - "eval:pilot": "npm run preflight && promptfoo eval -c promptfooconfig.yaml --repeat 3 --no-cache --no-share", - "view": "promptfoo view" + "setup": "node scripts/setup.mjs", + "test:setup": "node --test scripts/setup.test.mjs", + "promptfoo": "node --env-file=.env node_modules/promptfoo/dist/src/entrypoint.js", + "check": "node scripts/require-node.mjs && node --env-file-if-exists=.env node_modules/promptfoo/dist/src/entrypoint.js validate -c promptfooconfig.yaml", + "preflight": "node --env-file=.env scripts/preflight.mjs", + "eval:smoke": "npm run preflight && npm run promptfoo -- eval -c promptfooconfig.yaml --filter-first-n 2 --repeat 1 --no-cache --no-share", + "eval:pilot": "npm run preflight && npm run promptfoo -- eval -c promptfooconfig.yaml --repeat 3 --no-cache --no-share", + "view": "npm run promptfoo -- view" }, "devDependencies": { "@openai/codex-sdk": "0.151.0", diff --git a/benchmarks/codex-mcp/scripts/require-node.mjs b/benchmarks/codex-mcp/scripts/require-node.mjs new file mode 100644 index 00000000..fa7a74de --- /dev/null +++ b/benchmarks/codex-mcp/scripts/require-node.mjs @@ -0,0 +1,8 @@ +import { REQUIRED_NODE_VERSION, versionAtLeast } from './setup-lib.mjs'; + +if (!versionAtLeast(process.versions.node)) { + process.stderr.write( + `Node.js ${REQUIRED_NODE_VERSION.join('.')} or newer is required; found ${process.versions.node}.\n`, + ); + process.exitCode = 1; +} diff --git a/benchmarks/codex-mcp/scripts/setup-lib.mjs b/benchmarks/codex-mcp/scripts/setup-lib.mjs new file mode 100644 index 00000000..d4316561 --- /dev/null +++ b/benchmarks/codex-mcp/scripts/setup-lib.mjs @@ -0,0 +1,87 @@ +import { homedir } from 'node:os'; +import { posix, win32 } from 'node:path'; + +export const REQUIRED_NODE_VERSION = [22, 22, 0]; + +export function versionAtLeast(actual, required = REQUIRED_NODE_VERSION) { + const parts = actual.split('.').map(Number); + return required.every((requiredPart, index) => { + const actualPart = parts[index] ?? 0; + const prefixMatches = required + .slice(0, index) + .every((part, prefixIndex) => (parts[prefixIndex] ?? 0) === part); + return !prefixMatches || actualPart >= requiredPart; + }); +} + +export function defaultEvaluationRoot(environment, platform = process.platform) { + const paths = platform === 'win32' ? win32 : posix; + if (environment.VIDXP_EVAL_ROOT) { + return paths.resolve(environment.VIDXP_EVAL_ROOT); + } + if (platform === 'win32') { + if (!environment.LOCALAPPDATA) { + throw new Error('LOCALAPPDATA is required when VIDXP_EVAL_ROOT is unset.'); + } + return paths.join(environment.LOCALAPPDATA, 'VidXP', 'benchmarks', 'codex-mcp'); + } + const dataHome = environment.XDG_DATA_HOME || paths.join(homedir(), '.local', 'share'); + return paths.join(dataHome, 'vidxp', 'benchmarks', 'codex-mcp'); +} + +export function evaluationEnvironment({ + benchmarkRoot, + repositoryRoot, + evaluationRoot, + environment = process.env, + platform = process.platform, +}) { + const paths = platform === 'win32' ? win32 : posix; + const executable = platform === 'win32' ? 'vidxp-mcp.exe' : 'vidxp-mcp'; + const scriptsDirectory = platform === 'win32' ? 'Scripts' : 'bin'; + return { + VIDXP_EVAL_CODEX_HOME: paths.join(evaluationRoot, 'codex-home'), + VIDXP_EVAL_WORKSPACE: paths.join(evaluationRoot, 'workspace'), + VIDXP_EVAL_DATA_DIR: paths.join(evaluationRoot, 'vidxp-data'), + VIDXP_EVAL_INDEX_DIR: paths.join(evaluationRoot, 'vidxp-index'), + VIDXP_MCP_COMMAND: paths.join(repositoryRoot, '.venv', scriptsDirectory, executable), + VIDXP_EVAL_REPOSITORY: environment.VIDXP_EVAL_REPOSITORY || 'default', + VIDXP_EVAL_DEVICE: environment.VIDXP_EVAL_DEVICE || 'cpu', + VIDXP_EVAL_MODEL: environment.VIDXP_EVAL_MODEL || 'gpt-5.6-sol', + VIDXP_EVAL_REASONING: environment.VIDXP_EVAL_REASONING || 'medium', + VIDXP_EVAL_ARTIFACT_DIR: paths.join(evaluationRoot, 'longvale-artifacts'), + VIDXP_EVAL_ENV_FILE: paths.join(benchmarkRoot, '.env'), + }; +} + +export function serializeEnvironment(environment) { + return Object.entries(environment) + .filter(([name]) => name !== 'VIDXP_EVAL_ARTIFACT_DIR' && name !== 'VIDXP_EVAL_ENV_FILE') + .map(([name, value]) => `${name}=${JSON.stringify(value.replaceAll('\\', '/'))}`) + .join('\n') + '\n'; +} + +export function indexContainsPilot(index, videoIds, modalities) { + if (!index) { + return false; + } + const filenames = new Set((index.items || []).map((item) => item.original_filename)); + const indexedModalities = new Set(index.modalities || []); + return videoIds.every((id) => filenames.has(`${id}.mp4`)) + && modalities.every((modality) => indexedModalities.has(modality)); +} + +export function libsqlBindingName(platform, architecture, glibcVersion = undefined) { + if (platform === 'win32' && architecture === 'x64') { + return '@libsql/win32-x64-msvc'; + } + if (platform === 'darwin' && ['arm64', 'x64'].includes(architecture)) { + return `@libsql/darwin-${architecture}`; + } + if (platform === 'linux' && ['arm', 'arm64', 'x64'].includes(architecture)) { + const libc = glibcVersion ? (architecture === 'arm' ? 'gnueabihf' : 'gnu') + : (architecture === 'arm' ? 'musleabihf' : 'musl'); + return `@libsql/linux-${architecture}-${libc}`; + } + throw new Error(`Promptfoo has no pinned libsql binding for ${platform}-${architecture}.`); +} diff --git a/benchmarks/codex-mcp/scripts/setup.mjs b/benchmarks/codex-mcp/scripts/setup.mjs new file mode 100644 index 00000000..416d5fd8 --- /dev/null +++ b/benchmarks/codex-mcp/scripts/setup.mjs @@ -0,0 +1,265 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + createReadStream, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + REQUIRED_NODE_VERSION, + defaultEvaluationRoot, + evaluationEnvironment, + indexContainsPilot, + libsqlBindingName, + serializeEnvironment, + versionAtLeast, +} from './setup-lib.mjs'; + +const benchmarkRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const repositoryRoot = resolve(benchmarkRoot, '..', '..'); +const manifestPath = join(benchmarkRoot, 'tasks', 'longvale-part9-pilot.json'); +const datasetRevision = '18889b01886e30c36b0d1c650ac4439ad460ee73'; +const archiveHash = 'c83d62557f102c6d41ea95c2c3b3581657481c8646cc70b1e12a85ead27a7ae3'; +const archiveRelativePath = join('raw_videos_test', 'LongVALE_test_1171_part_9.zip'); +const annotationFilename = 'longvale-annotations-eval.json'; +const modalities = ['scene', 'action', 'sound', 'speech']; + +function executableName(command) { + return process.platform === 'win32' && command === 'npm' ? 'npm.cmd' : command; +} + +function formatCommand(command, args) { + return [command, ...args] + .map((part) => (/\s/.test(part) ? JSON.stringify(part) : part)) + .join(' '); +} + +function run(command, args, { cwd = repositoryRoot, env = process.env, capture = false } = {}) { + process.stdout.write(`\n> ${formatCommand(command, args)}\n`); + const result = spawnSync(executableName(command), args, { + cwd, + env, + encoding: capture ? 'utf8' : undefined, + stdio: capture ? 'pipe' : 'inherit', + }); + if (result.error) { + throw new Error(`Could not run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const detail = capture ? `\n${result.stderr || result.stdout}` : ''; + throw new Error(`${command} exited with status ${result.status}.${detail}`); + } + return capture ? result.stdout : ''; +} + +async function sha256(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + } + return hash.digest('hex'); +} + +function readIndex(environment) { + try { + const output = run( + 'uv', + [ + 'run', '--no-sync', 'vidxp', + '--data-dir', environment.VIDXP_EVAL_DATA_DIR, + '--index-dir', environment.VIDXP_EVAL_INDEX_DIR, + 'index', 'list', '--json', + ], + { env: { ...process.env, ...environment }, capture: true }, + ); + return JSON.parse(output); + } catch { + return null; + } +} + +async function main() { + if (!versionAtLeast(process.versions.node)) { + throw new Error( + `Node.js ${REQUIRED_NODE_VERSION.join('.')} or newer is required; found ${process.versions.node}.`, + ); + } + + run('uv', ['--version'], { capture: true }); + run('codex', ['--version'], { capture: true }); + + const evaluationRoot = defaultEvaluationRoot(process.env); + const setupEnvironment = evaluationEnvironment({ + benchmarkRoot, + repositoryRoot, + evaluationRoot, + }); + const commandEnvironment = { ...process.env, ...setupEnvironment }; + const tasks = JSON.parse(readFileSync(manifestPath, 'utf8')); + const videoIds = [...new Set(tasks.map((task) => task.video_id))]; + + run( + 'uv', + ['sync', '--frozen', '--extra', 'local-worker', '--extra', 'mcp', '--extra', 'benchmarks'], + ); + run('npm', ['ci'], { cwd: benchmarkRoot }); + + const glibcVersion = process.report?.getReport().header.glibcVersionRuntime; + const bindingName = libsqlBindingName(process.platform, process.arch, glibcVersion); + const libsqlManifest = JSON.parse(readFileSync( + join(benchmarkRoot, 'node_modules', 'libsql', 'package.json'), + 'utf8', + )); + const bindingVersion = libsqlManifest.optionalDependencies?.[bindingName]; + if (!bindingVersion) { + throw new Error(`The Promptfoo lock does not declare ${bindingName}.`); + } + run( + 'npm', + [ + 'install', '--no-save', '--package-lock=false', '--omit=optional', + `${bindingName}@${bindingVersion}`, + ], + { cwd: benchmarkRoot }, + ); + run( + process.execPath, + [ + join(benchmarkRoot, 'node_modules', 'promptfoo', 'dist', 'src', 'entrypoint.js'), + 'validate', '-c', join(benchmarkRoot, 'promptfooconfig.yaml'), + ], + { cwd: benchmarkRoot, env: commandEnvironment }, + ); + + for (const directory of [ + setupEnvironment.VIDXP_EVAL_CODEX_HOME, + setupEnvironment.VIDXP_EVAL_WORKSPACE, + join(setupEnvironment.VIDXP_EVAL_WORKSPACE, 'media'), + setupEnvironment.VIDXP_EVAL_DATA_DIR, + setupEnvironment.VIDXP_EVAL_INDEX_DIR, + setupEnvironment.VIDXP_EVAL_ARTIFACT_DIR, + ]) { + mkdirSync(directory, { recursive: true }); + } + if (!existsSync(setupEnvironment.VIDXP_MCP_COMMAND)) { + throw new Error(`VidXP MCP executable was not created at ${setupEnvironment.VIDXP_MCP_COMMAND}.`); + } + writeFileSync( + setupEnvironment.VIDXP_EVAL_ENV_FILE, + serializeEnvironment(setupEnvironment), + 'utf8', + ); + + const authPath = join(setupEnvironment.VIDXP_EVAL_CODEX_HOME, 'auth.json'); + if (!existsSync(authPath)) { + process.stdout.write('\nSign in to the isolated Codex profile when prompted.\n'); + run('codex', ['login'], { + env: { ...commandEnvironment, CODEX_HOME: setupEnvironment.VIDXP_EVAL_CODEX_HOME }, + }); + } + if (!existsSync(authPath)) { + throw new Error('Codex login completed without creating auth.json in the isolated profile.'); + } + + process.stdout.write( + '\nDownloading the pinned LongVALE pilot files. Use of the dataset is subject to its published terms.\n', + ); + run( + 'uvx', + [ + 'hf', 'download', 'ttgeng233/LongVALE', + annotationFilename, + archiveRelativePath.replaceAll('\\', '/'), + '--repo-type', 'dataset', + '--revision', datasetRevision, + '--local-dir', setupEnvironment.VIDXP_EVAL_ARTIFACT_DIR, + ], + { env: commandEnvironment }, + ); + + const archivePath = join(setupEnvironment.VIDXP_EVAL_ARTIFACT_DIR, archiveRelativePath); + const actualHash = await sha256(archivePath); + if (actualHash !== archiveHash) { + throw new Error(`LongVALE archive hash mismatch: expected ${archiveHash}, found ${actualHash}.`); + } + + const sourceMedia = join(setupEnvironment.VIDXP_EVAL_ARTIFACT_DIR, 'video_test_1171'); + if (videoIds.some((videoId) => !existsSync(join(sourceMedia, `${videoId}.mp4`)))) { + run( + 'uv', + ['run', '--no-sync', 'python', '-m', 'zipfile', '-e', archivePath, setupEnvironment.VIDXP_EVAL_ARTIFACT_DIR], + { env: commandEnvironment }, + ); + } + for (const videoId of videoIds) { + const source = join(sourceMedia, `${videoId}.mp4`); + if (!existsSync(source)) { + throw new Error(`The LongVALE archive did not contain ${source}.`); + } + copyFileSync(source, join(setupEnvironment.VIDXP_EVAL_WORKSPACE, 'media', `${videoId}.mp4`)); + } + + run( + 'uv', + [ + 'run', '--no-sync', 'vidxp', + '--data-dir', setupEnvironment.VIDXP_EVAL_DATA_DIR, + '--index-dir', setupEnvironment.VIDXP_EVAL_INDEX_DIR, + 'prepare', '--modalities', modalities.join(','), '--yes', + ], + { env: commandEnvironment }, + ); + + if (!indexContainsPilot(readIndex(setupEnvironment), videoIds, modalities)) { + for (const videoId of videoIds) { + process.stdout.write(`\nIndexing ${videoId}.mp4\n`); + const mediaPath = join(setupEnvironment.VIDXP_EVAL_WORKSPACE, 'media', `${videoId}.mp4`); + const imported = JSON.parse(run( + 'uv', + [ + 'run', '--no-sync', 'vidxp', + '--data-dir', setupEnvironment.VIDXP_EVAL_DATA_DIR, + '--index-dir', setupEnvironment.VIDXP_EVAL_INDEX_DIR, + 'media', 'import', mediaPath, '--json', + ], + { env: commandEnvironment, capture: true }, + )); + run( + 'uv', + [ + 'run', '--no-sync', 'vidxp', + '--data-dir', setupEnvironment.VIDXP_EVAL_DATA_DIR, + '--index-dir', setupEnvironment.VIDXP_EVAL_INDEX_DIR, + 'index', 'create', imported.media_id, + ...modalities.flatMap((modality) => ['--modality', modality]), + ], + { env: commandEnvironment }, + ); + } + } else { + process.stdout.write('\nThe five pilot videos are already indexed; skipping indexing.\n'); + } + + run( + process.execPath, + [join(benchmarkRoot, 'scripts', 'preflight.mjs')], + { cwd: benchmarkRoot, env: commandEnvironment }, + ); + + process.stdout.write( + '\nSetup complete. Run:\n' + + ' npm --prefix benchmarks/codex-mcp run eval:smoke\n' + + ' npm --prefix benchmarks/codex-mcp run eval:pilot\n', + ); +} + +main().catch((error) => { + process.stderr.write(`\nSetup failed: ${error.message}\n`); + process.exitCode = 1; +}); diff --git a/benchmarks/codex-mcp/scripts/setup.test.mjs b/benchmarks/codex-mcp/scripts/setup.test.mjs new file mode 100644 index 00000000..0d0cf18e --- /dev/null +++ b/benchmarks/codex-mcp/scripts/setup.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + defaultEvaluationRoot, + evaluationEnvironment, + indexContainsPilot, + libsqlBindingName, + serializeEnvironment, + versionAtLeast, +} from './setup-lib.mjs'; + +test('checks the required Node version numerically', () => { + assert.equal(versionAtLeast('22.21.9'), false); + assert.equal(versionAtLeast('22.22.0'), true); + assert.equal(versionAtLeast('23.0.0'), true); +}); + +test('recognizes a complete pilot index regardless of extra media', () => { + const index = { + items: [ + { original_filename: 'alpha.mp4' }, + { original_filename: 'beta.mp4' }, + { original_filename: 'unrelated.mp4' }, + ], + modalities: ['scene', 'action', 'sound', 'speech'], + }; + + assert.equal( + indexContainsPilot(index, ['alpha', 'beta'], ['scene', 'action', 'sound', 'speech']), + true, + ); + assert.equal(indexContainsPilot(index, ['alpha', 'missing'], ['scene']), false); + assert.equal(indexContainsPilot(index, ['alpha'], ['scene', 'ocr']), false); +}); + +test('uses one optional root override for mutable setup state', () => { + assert.equal( + defaultEvaluationRoot({ VIDXP_EVAL_ROOT: 'C:/custom/eval' }, 'win32'), + 'C:\\custom\\eval', + ); + assert.equal( + defaultEvaluationRoot({ LOCALAPPDATA: 'C:/Users/test/AppData/Local' }, 'win32'), + 'C:\\Users\\test\\AppData\\Local\\VidXP\\benchmarks\\codex-mcp', + ); + assert.equal( + defaultEvaluationRoot({ XDG_DATA_HOME: '/tmp/data' }, 'linux'), + '/tmp/data/vidxp/benchmarks/codex-mcp', + ); +}); + +test('selects the required Promptfoo SQLite binding for the host', () => { + assert.equal(libsqlBindingName('win32', 'x64'), '@libsql/win32-x64-msvc'); + assert.equal(libsqlBindingName('darwin', 'arm64'), '@libsql/darwin-arm64'); + assert.equal(libsqlBindingName('linux', 'x64', '2.39'), '@libsql/linux-x64-gnu'); + assert.equal(libsqlBindingName('linux', 'x64'), '@libsql/linux-x64-musl'); + assert.throws(() => libsqlBindingName('win32', 'arm64'), /no pinned libsql binding/); +}); + +test('builds and serializes the environment consumed by Promptfoo', () => { + const environment = evaluationEnvironment({ + benchmarkRoot: 'C:/repo/benchmarks/codex-mcp', + repositoryRoot: 'C:/repo', + evaluationRoot: 'C:/eval', + environment: {}, + platform: 'win32', + }); + const serialized = serializeEnvironment(environment); + + assert.match(serialized, /VIDXP_EVAL_WORKSPACE="C:\/eval\/workspace"/); + assert.match(serialized, /VIDXP_MCP_COMMAND="C:\/repo\/\.venv\/Scripts\/vidxp-mcp\.exe"/); + assert.match(serialized, /VIDXP_EVAL_MODEL="gpt-5\.6-sol"/); + assert.doesNotMatch(serialized, /VIDXP_EVAL_ENV_FILE/); + assert.doesNotMatch(serialized, /VIDXP_EVAL_ARTIFACT_DIR/); +}); diff --git a/docs/benchmarking/agent_ablation.md b/docs/benchmarking/agent_ablation.md index 9ffdcf9c..d528de23 100644 --- a/docs/benchmarking/agent_ablation.md +++ b/docs/benchmarking/agent_ablation.md @@ -4,7 +4,7 @@ Collection index: [Benchmarking research](README.md) Status: Runnable scaffold; no agent results recorded -Last verified: 2026-08-30 +Last verified: 2026-09-01 This experiment measures whether access to VidXP through its local stdio MCP server improves a Codex agent's ability to find timestamped evidence in long @@ -100,128 +100,48 @@ therefore does not replace LongVALE in this ablation. Promptfoo 0.122.2 requires Node.js 22.22.0 or newer. The benchmark-local `.npmrc` enforces that requirement so an unsupported runtime fails during -installation instead of failing after Codex runs have begun. Install VidXP and -the local evaluation dependencies: +installation instead of failing after Codex runs have begun. You also need +`uv` and the Codex CLI on `PATH`. -```powershell -uv sync --frozen --extra local-worker --extra mcp --extra benchmarks -npm --prefix benchmarks/codex-mcp ci -``` - -The benchmark pins the Codex SDK directly and omits Promptfoo's unrelated -optional provider packages from this install. - -Create all mutable state outside the checkout. The paths below are examples; -keep the same values for both conditions: +From the repository root, run the automated setup: ```powershell -$evalRoot = Join-Path $env:LOCALAPPDATA 'VidXP\benchmarks\codex-mcp' -$env:VIDXP_EVAL_CODEX_HOME = Join-Path $evalRoot 'codex-home' -$env:VIDXP_EVAL_WORKSPACE = Join-Path $evalRoot 'workspace' -$env:VIDXP_EVAL_DATA_DIR = Join-Path $evalRoot 'vidxp-data' -$env:VIDXP_EVAL_INDEX_DIR = Join-Path $evalRoot 'vidxp-index' -$env:VIDXP_MCP_COMMAND = (Resolve-Path '.venv\Scripts\vidxp-mcp.exe').Path -$env:VIDXP_EVAL_REPOSITORY = 'default' -$env:VIDXP_EVAL_DEVICE = 'cpu' -$env:VIDXP_EVAL_MODEL = 'gpt-5.6-sol' -$env:VIDXP_EVAL_REASONING = 'medium' - -New-Item -ItemType Directory -Force ` - $env:VIDXP_EVAL_CODEX_HOME, ` - $env:VIDXP_EVAL_WORKSPACE, ` - (Join-Path $env:VIDXP_EVAL_WORKSPACE 'media'), ` - $env:VIDXP_EVAL_DATA_DIR, ` - $env:VIDXP_EVAL_INDEX_DIR | Out-Null - -$env:CODEX_HOME = $env:VIDXP_EVAL_CODEX_HOME -codex login -Remove-Item Env:CODEX_HOME +npm --prefix benchmarks/codex-mcp run setup ``` -Do not copy or commit `auth.json`. The preflight rejects an isolated Codex -configuration that declares any ambient `[mcp_servers]` section. - -## Fetch and index the pilot media - -Accept the LongVALE dataset terms before downloading. Fetch only the pinned -annotation and part-nine evaluation archive: - -The commands below require the Hugging Face `hf` CLI. Install it separately if -it is not already available; it is a dataset-transfer tool and is not part of -VidXP's runtime dependency set. +The command installs the pinned Python and Node dependencies, creates isolated +state outside the checkout, opens Codex login when authentication is absent, +downloads and verifies the pinned LongVALE archive, copies the five pilot +videos, prepares the four required capabilities, indexes the media, saves the +evaluation environment in the ignored `benchmarks/codex-mcp/.env` file, and +runs preflight. Accept the LongVALE dataset terms before running it. Do not copy +or commit the generated `auth.json`. -```powershell -$artifactRoot = Join-Path $evalRoot 'longvale-artifacts' -hf download ttgeng233/LongVALE ` - longvale-annotations-eval.json ` - raw_videos_test/LongVALE_test_1171_part_9.zip ` - --repo-type dataset ` - --revision 18889b01886e30c36b0d1c650ac4439ad460ee73 ` - --local-dir $artifactRoot - -$archive = Join-Path $artifactRoot ` - 'raw_videos_test\LongVALE_test_1171_part_9.zip' -(Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() -Expand-Archive -LiteralPath $archive -DestinationPath $artifactRoot -``` - -The printed hash must equal the pinned SHA-256 above. Copy the five selected -MP4s into `$env:VIDXP_EVAL_WORKSPACE\media`, preserving their filenames: +By default, mutable state goes under the operating system's user data +directory. Set only `VIDXP_EVAL_ROOT` when it needs to live elsewhere: ```powershell -$sourceMedia = Join-Path $artifactRoot 'video_test_1171' -$videoIds = @( - 'ZYTmgi1pAIE', - 'ZIdFAGJrlCw', - 'ZGXCr5n8Frg', - '_py1WXVX4oc', - 'ZVUAC3m48G0' -) -foreach ($videoId in $videoIds) { - Copy-Item -LiteralPath (Join-Path $sourceMedia "$videoId.mp4") ` - -Destination (Join-Path $env:VIDXP_EVAL_WORKSPACE 'media') -} +$env:VIDXP_EVAL_ROOT = 'D:\vidxp-eval' +npm --prefix benchmarks/codex-mcp run setup ``` -Prepare and index all four evidence paths: - -```powershell -uv run --no-sync vidxp ` - --data-dir $env:VIDXP_EVAL_DATA_DIR ` - --index-dir $env:VIDXP_EVAL_INDEX_DIR ` - prepare --modalities scene,action,sound,speech --yes - -foreach ($videoId in $videoIds) { - $mediaPath = Join-Path $env:VIDXP_EVAL_WORKSPACE "media\$videoId.mp4" - $asset = uv run --no-sync vidxp ` - --data-dir $env:VIDXP_EVAL_DATA_DIR ` - --index-dir $env:VIDXP_EVAL_INDEX_DIR ` - media import $mediaPath --json | ConvertFrom-Json - - uv run --no-sync vidxp ` - --data-dir $env:VIDXP_EVAL_DATA_DIR ` - --index-dir $env:VIDXP_EVAL_INDEX_DIR ` - index create $asset.media_id ` - --modality scene ` - --modality action ` - --modality sound ` - --modality speech -} -``` +The setup is safe to rerun. Cached downloads and prepared models are reused, +and indexing is skipped when all five videos and four modalities are already +present. The benchmark pins the Codex SDK directly and omits Promptfoo's +unrelated optional provider packages from the install. ## Validate before spending runs -The following commands perform no Codex inference: +Setup finishes by running preflight, which verifies the dedicated Codex +authentication, absence of ambient MCP configuration, all five media files, +the index paths, and a real VidXP MCP handshake. To repeat the configuration and +preflight checks without setup or Codex inference, run: ```powershell npm --prefix benchmarks/codex-mcp run check npm --prefix benchmarks/codex-mcp run preflight ``` -Preflight verifies the dedicated Codex authentication, absence of ambient MCP -configuration, all five media files, the index paths, and a real VidXP MCP -handshake. Do not run the matrix if it fails. - The first paid/allowance-consuming smoke is one task in both conditions: two Codex runs total.