diff --git a/.github/workflows/live-e2e.yml b/.github/workflows/live-e2e.yml index 6f2ef5f0a..861ddd72b 100644 --- a/.github/workflows/live-e2e.yml +++ b/.github/workflows/live-e2e.yml @@ -13,14 +13,27 @@ name: Live E2E # (#794). So a missing key warns, skips the six steps that need it, and leaves the run # green while the step summary says plainly that nothing ran and nothing was measured. A # real failure of the suite still fails the run: the skip is never a pass. +# +# The `mock` job below is the keyless half (#2565): the same bench, adapter and evidence +# path, with DEEPSEEK_BASE_URL pointed at the scripted mock (core/dev/mock-llm.mjs). It +# reads no secret, so it runs on every pull request that touches core/ or plugins/, not only +# on main — and its step summary names each check it ran, so its green is visibly different +# from the live job's skip. The `live` job is unchanged. on: schedule: - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z workflow_dispatch: + pull_request: + paths: + - "core/**" + - "plugins/**" + - "dev/live-e2e/**" + - ".github/workflows/live-e2e.yml" +# Per ref: a pull request's mock run must not sit in the nightly's queue (or start one). concurrency: - group: live-e2e + group: live-e2e-${{ github.ref }} cancel-in-progress: false permissions: @@ -29,6 +42,7 @@ permissions: jobs: window: name: window check + if: github.event_name != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -176,3 +190,126 @@ jobs: path: evidence/ if-no-files-found: ignore retention-days: 14 + + # The keyless browser suite (#2565). Same bench, web adapter and evidence path as the live + # job, with DEEPSEEK_BASE_URL on the scripted mock and no secret in the environment at all: + # the two #204 task checks cannot run here (a scripted model answers a task prompt with a + # canned acknowledgement and writes no files), so this job measures the class of regression + # the repository had no browser-level check for — a message goes in through the composer, a + # reply comes out, and both survive a reload. + mock: + name: mock · ubuntu · Node 24 · scripted model + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + WEB_PORT: '3197' + MOCK_PORT: '8765' + BENCH: live-e2e-mock + MODEL: deepseek-official/deepseek-flash + LIVE_E2E_MODE: mock + DEEPSEEK_BASE_URL: http://127.0.0.1:8765 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: core/package-lock.json + - name: npm ci (core) + working-directory: core + run: npm ci --no-fund --no-audit + timeout-minutes: 5 + # The runtime is the published package, not a source checkout: a checkout has no built + # client artifacts (dsh web exits with ENOENT on packages/client/*/lib/client.js), and + # this job must not spend ten minutes building DSH to prove a reply still renders. pnpm + # keeps the node_modules/.pnpm layout the adapter resolves playwright-core from. + - name: Install the pinned DSH runtime + run: | + DSH=$(node -p "require('./core/dev/harness/package.json').dependencies['@deepseek-ai/dsh']") + RUNTIME="$RUNNER_TEMP/dsh-runtime" + mkdir -p "$RUNTIME" + cd "$RUNTIME" + printf '{"name":"dsh-live-e2e-runtime","private":true,"packageManager":"pnpm@12.16.1"}\n' > package.json + # pnpm 12 refuses dependency build scripts by default; the pinned runtime's native + # helpers (subprocess spawn, pty, protobuf) are the same packages the live job runs. + printf 'dangerouslyAllowAllBuilds: true\n' > pnpm-workspace.yaml + corepack enable + corepack pnpm add "@deepseek-ai/dsh@$DSH" playwright-core@1.61.1 + echo "DSH_HARNESS=$RUNTIME" >> "$GITHUB_ENV" + timeout-minutes: 8 + # The headless shell is what the adapter launches. No --with-deps: the runner image + # already carries Chrome's system libraries, and an apt-get round there is the slow, + # flaky half (a run timed out on it); one retry covers the download itself. + - name: Chromium for the browser + run: | + for attempt in 1 2; do + node "$DSH_HARNESS/node_modules/playwright-core/cli.js" install chromium --only-shell && exit 0 + echo "::warning::chromium install attempt $attempt failed; retrying" + sleep 5 + done + exit 1 + timeout-minutes: 8 + - name: Start the scripted mock + run: | + # The bench home lives outside the checkout (#398), exactly as in the live job. + LIVE_E2E_HOME="$HOME/.cache/flybot-live-e2e/$BENCH" + echo "LIVE_E2E_HOME=$LIVE_E2E_HOME" >> "$GITHUB_ENV" + mkdir -p "$LIVE_E2E_HOME" + MOCK_LOG="$LIVE_E2E_HOME/mock.jsonl" setsid node core/dev/mock-llm.mjs \ + "$LIVE_E2E_HOME/mock.out" 2>&1 & echo $! >"$LIVE_E2E_HOME/mock.pid" + i=0 + until (exec 3<>"/dev/tcp/127.0.0.1/$MOCK_PORT") 2>/dev/null; do + i=$((i + 1)) + [ $i -ge 30 ] && { echo "::error::the scripted mock did not start"; cat "$LIVE_E2E_HOME/mock.out"; exit 1; } + sleep 1 + done + timeout-minutes: 2 + - name: Start the mock bench + run: node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset + timeout-minutes: 8 + # The summary carries the runner's own lines: one `ok`/`FAIL` per check it ran, the + # mode, and the outcome — so a green that measured nothing cannot look like this one. + - name: Run the keyless checks + id: run + run: | + set -o pipefail + { + echo "### Live E2E (mock, no key)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + # The fence closes and the step still fails on the runner's exit code: a summary + # that ends in a stray `echo` would make a red run look green. + status=0 + node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" "$WEB_PORT" \ + --model "$MODEL" --harness "$DSH_HARNESS" --evidence evidence --tz Asia/Shanghai \ + | tee -a "$GITHUB_STEP_SUMMARY" || status=$? + echo '```' >> "$GITHUB_STEP_SUMMARY" + exit $status + - name: Stop the bench and the mock + if: always() + run: | + node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true + if [ -f "$LIVE_E2E_HOME/mock.pid" ]; then + kill -TERM "-$(cat "$LIVE_E2E_HOME/mock.pid")" 2>/dev/null || true + fi + - name: Sanitized web log (diagnostics only, never uploaded raw) + if: failure() + run: tail -80 "$LIVE_E2E_HOME/web.log" | grep -v 'token=' || true + - name: Evidence gate + id: evidence-gate + if: always() && steps.run.outcome != 'skipped' + run: | + if [ ! -f evidence/sanitized.txt ]; then + echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded" + exit 1 + fi + - name: Upload evidence (allow-list, sanitizer-stamped) + if: always() && steps.evidence-gate.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: live-e2e-mock-evidence + path: evidence/ + if-no-files-found: ignore + retention-days: 14 diff --git a/changelog.d/2565-keyless-browser-e2e.added.md b/changelog.d/2565-keyless-browser-e2e.added.md new file mode 100644 index 000000000..f2ddfa1bf --- /dev/null +++ b/changelog.d/2565-keyless-browser-e2e.added.md @@ -0,0 +1 @@ +A keyless browser suite now runs in CI: the scripted mock drives the same bench and web adapter as the real-model suite, so the web UI's message-and-reply path is measured on every pull request that touches `core/` or `plugins/` instead of only existing as a nightly that skips without a secret. The job reads no secret, its step summary names each check it ran, and the real-model `live` job is unchanged. ([#2565](https://github.com/ThinkFlowLab/FlyBot/issues/2565)) diff --git a/dev/live-e2e/README.md b/dev/live-e2e/README.md index 653e380d3..4d8b3ca3f 100644 --- a/dev/live-e2e/README.md +++ b/dev/live-e2e/README.md @@ -14,8 +14,9 @@ only `adapter.mjs` knows FlyBot. Nothing here ships in a package or runs in | --- | --- | | `fixtures/orders.csv` | The fixed 5-row fixture the expected totals derive from. | | `scenarios.mjs` | The four checks, task texts, pure assertions (parsed JSON, SHA-256 immutability). | +| `smoke.mjs` | The keyless checks (#2565): a message sent through the composer gets a reply, and both survive a reload. Run by `LIVE_E2E_MODE=mock`; no files, no model wording asserted. | | `adapter.mjs` | The FlyBot web adapter: playwright-core from the harness checkout, UI task steps, read-only observation. | -| `bench.mjs` | A dsh web bench on this checkout with the real endpoint; refuses missing/mock keys, keeps its home outside the checkout, never touches `~/.dsh`. | +| `bench.mjs` | A dsh web bench on this checkout. `LIVE_E2E_MODE=live` refuses a missing/mock key, `LIVE_E2E_MODE=mock` needs the loopback mock and reads no key; both keep the home outside the checkout and never touch `~/.dsh`. | | `run.mjs` | The runner: guards, budgets, classification, report, evidence sanitizer. | | `report.mjs` | Outcome contract, guard logic, the tamper rule (`integrityOutcome`), usage/classification, report assembly (pure, tested keyless). | | `integrity.mjs` | The grader's own hash: `treeManifest` / `changedPaths` for the tamper check, and the path test the guards use (#398). | @@ -76,12 +77,35 @@ A run costs real tokens (the pilot used ~116K); budgets default to 180 s per che 250K tokens per run (`LIVE_E2E_CHECK_MS`, `LIVE_E2E_QUIET_MS`, `LIVE_E2E_MAX_TOKENS`), with zero retries. +## Run the keyless mock suite + +No key, no network to a model endpoint: the same bench and adapter, with the scripted mock +from `core/dev/mock-llm.mjs` answering (`LIVE_E2E_MODE=mock` also allows the loopback base +URL the live mode refuses). The checks are the two in `smoke.mjs`, not the four #204 task +checks — a scripted model does not write `summary.json`. + +```sh +export LIVE_E2E_MODE=mock +export DEEPSEEK_BASE_URL=http://127.0.0.1:8765 +MOCK_PORT=8765 node core/dev/mock-llm.mjs & # the scripted model +node dev/live-e2e/bench.mjs live-e2e 3197 reset +node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" 3197 --model deepseek-official/deepseek-flash +node dev/live-e2e/bench.mjs live-e2e 3197 stop +``` + +The runtime can be a published `@deepseek-ai/dsh` install instead of a source checkout: the +bench only needs the `dsh` command (a source tree has no built `lib/client.js`) and the +adapter `playwright-core` from `node_modules/.pnpm/node_modules`. + ## In CI `.github/workflows/live-e2e.yml` runs the same flow nightly at 00:30 Asia/Shanghai (and on dispatch) on `main` only, with a window gate so a quiet `main` spends no tokens: it clones the harness at the tag pinned in `core/dev/harness/package.json`, starts the tap, resets the bench, runs the four checks and uploads the sanitizer-cleared allow-list. +The keyless `mock` job runs the two checks above on every pull request that touches `core/` +or `plugins/` and on the same nightly schedule, with no `secrets.*` in its environment and +the runner's summary appended to the job's step summary ([#2565](https://github.com/ThinkFlowLab/FlyBot/issues/2565)). The four decisions behind this setup are recorded in [#204](https://github.com/ThinkFlowLab/issues/204). "Stable" means five consecutive scheduled runs green at one recorded configuration; until then the suite is evidence, diff --git a/dev/live-e2e/adapter.mjs b/dev/live-e2e/adapter.mjs index 185822556..aa2a3b982 100644 --- a/dev/live-e2e/adapter.mjs +++ b/dev/live-e2e/adapter.mjs @@ -151,9 +151,16 @@ export async function openFlyBotWeb({ harness, home, port, model, outDir, quietM await page.waitForTimeout(1200) }, // Who said what is app-specific to render; for the scenarios it is enough that the - // task texts are still visible and that some reply beside them is. + // task texts are still visible and that some reply beside them is. The reading is + // scoped to the conversation flow, never the whole page: the sidebar's own text (Bot + // names, previews, group lines) would otherwise count as "a reply", and a chat that + // rendered nothing would still look non-empty (#2565). async conversationLines() { - const text = await page.evaluate(() => document.body.innerText) + const text = await page.evaluate(() => { + const flow = document.querySelector('[data-conversation-scroll] [data-chat-flow]') + ?? document.querySelector('[data-conversation-scroll]') + return (flow ?? document.body).innerText + }) const lines = text.split('\n').map(line => line.trim()).filter(line => line.length > 0) return lines.map(line => ({ who: sent.some(task => task.slice(0, 24) && line.includes(task.slice(0, 24))) ? 'user' : 'app', text: line })) }, diff --git a/dev/live-e2e/bench.mjs b/dev/live-e2e/bench.mjs index 507ccb3ff..9db4757b3 100644 --- a/dev/live-e2e/bench.mjs +++ b/dev/live-e2e/bench.mjs @@ -1,7 +1,8 @@ -// Real-model test bench for the live E2E suite (#204): a dsh web whose profile links -// this checkout, talking to the real DeepSeek endpoint with DEEPSEEK_API_KEY from the -// caller's environment. The mock bench stays core/dev/bench.sh; this one never starts a -// mock and refuses to run without a real key, so the two suites cannot mix. +// Test bench for the E2E suites: a dsh web whose profile links this checkout, talking to +// whatever DEEPSEEK_BASE_URL names. LIVE_E2E_MODE=live (#204) is the real DeepSeek endpoint +// with a real key from the caller's environment; LIVE_E2E_MODE=mock (#2565) is the scripted +// mock already listening on loopback and reads no key at all. This bench never starts a +// mock itself, and each mode refuses the other's configuration, so the two cannot mix. // // node dev/live-e2e/bench.mjs [start|stop|reset|status] // @@ -10,14 +11,16 @@ // checkout is refused: that workspace would hold the grader (#398). Environment: DEEPSEEK_API_KEY // (required; fails loudly when the suite's own reading refuses it — see key.mjs), // DEEPSEEK_BASE_URL (optional; defaults to the official endpoint — point it at -// dev/llm-tap.mjs to record per-call usage), DSH_HARNESS (a deepseek-harness checkout with -// its dependencies installed). +// dev/llm-tap.mjs to record per-call usage, or at core/dev/mock-llm.mjs in mock mode), +// DSH_HARNESS (a deepseek-harness checkout with its dependencies installed), LIVE_E2E_MODE +// (live, the default, or mock). import { spawn, spawnSync } from 'node:child_process' import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { benchHome } from './integrity.mjs' -import { keyProblem, keyReading } from './key.mjs' +import { keyProblem, keyReading, MOCK_KEY } from './key.mjs' +import { readMode } from './report.mjs' const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) const [NAME, PORT_TEXT, ACTION = 'start'] = process.argv.slice(2) @@ -30,9 +33,18 @@ if (WEB_PORT === 3080) die('port 3080 belongs to the real service') const { home: HOME_DIR, problem: homeProblem } = benchHome({ name: NAME, root: ROOT }) if (homeProblem) die(`${homeProblem}\n(the bench home must be outside ${ROOT}; export LIVE_E2E_HOME to choose it)`) +const MODE = readMode(process.env.LIVE_E2E_MODE) +if (MODE !== 'live' && MODE !== 'mock') die(`LIVE_E2E_MODE is ${MODE}; use live or mock`) +const loopbackEndpoint = () => { + try { return /^(localhost|127\.|0\.0\.0\.0|\[::1\]|::1)/.test(new URL(process.env.DEEPSEEK_BASE_URL ?? '').host) } catch { return false } +} const apiKey = process.env.DEEPSEEK_API_KEY ?? '' -const key = keyReading(apiKey) -if (!key.usable) die(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}; a real key is required (never printed)`) +if (MODE === 'live') { + const key = keyReading(apiKey) + if (!key.usable) die(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}; a real key is required (never printed)`) +} else if (!loopbackEndpoint()) { + die('mock mode needs DEEPSEEK_BASE_URL on loopback, where the scripted mock listens; nothing was started') +} const harness = process.env.DSH_HARNESS ?? join(ROOT, 'deepseek-harness') if (!existsSync(join(harness, 'package.json'))) die(`no deepseek-harness checkout at ${harness}; set DSH_HARNESS (see CONTRIBUTING.md)`) @@ -87,14 +99,21 @@ async function start() { // anything is uploaded. Detached, the child is its own process group: the pid file // names the group, and a later stop reaches the whole tree. const out = openSync(logFile, 'a') - const child = spawn('pnpm', ['-s', 'dsh', '--profile', 'web', '--patch', 'apps/web/tests/pin-browse-picker.overlay.yml', '--patch', join(HOME_DIR, 'bench.patch.yml'), '--no-open', '--port', String(WEB_PORT)], { + // The overlay pins the in-browser directory picker. A source checkout carries it; the + // published runtime the mock job installs has no apps/ tree, and the smoke opens no + // picker, so its absence is not a failure. + const overlay = join(harness, 'apps/web/tests/pin-browse-picker.overlay.yml') + const patches = [...(existsSync(overlay) ? ['--patch', 'apps/web/tests/pin-browse-picker.overlay.yml'] : []), '--patch', join(HOME_DIR, 'bench.patch.yml')] + const child = spawn('pnpm', ['-s', 'dsh', '--profile', 'web', ...patches, '--no-open', '--port', String(WEB_PORT)], { cwd: harness, detached: true, stdio: ['ignore', out, out], env: { ...process.env, DSH_HOME: HOME_DIR, - DEEPSEEK_API_KEY: apiKey, + // In mock mode the bench never sees a key from this environment: the scripted mock + // ignores auth, and a real key has no business in a run that measures no model. + DEEPSEEK_API_KEY: MODE === 'mock' ? MOCK_KEY : apiKey, ...(process.env.DEEPSEEK_BASE_URL ? { DEEPSEEK_BASE_URL: process.env.DEEPSEEK_BASE_URL } : {}), }, }) @@ -120,7 +139,7 @@ function tailSanitized() { function status() { const running = existsSync(webPidFile) && alive(Number(readFileSync(webPidFile, 'utf8'))) console.log(`web: ${running ? `running (pid ${readFileSync(webPidFile, 'utf8').trim()})` : 'stopped'}`) - console.log(`home: ${HOME_DIR} web: http://127.0.0.1:${WEB_PORT} model: real (key from the environment)`) + console.log(`home: ${HOME_DIR} web: http://127.0.0.1:${WEB_PORT} model: ${MODE === 'mock' ? 'scripted mock (no key)' : 'real (key from the environment)'}`) } switch (ACTION) { diff --git a/dev/live-e2e/report.mjs b/dev/live-e2e/report.mjs index ea929c91c..e072cf8a5 100644 --- a/dev/live-e2e/report.mjs +++ b/dev/live-e2e/report.mjs @@ -29,14 +29,22 @@ export function budgetsFrom(env) { } } +// The two ways to drive the same bench: a real key against the real endpoint, or the +// scripted mock on loopback with no key at all (#2565). The mode is explicit because the +// two must never look like each other — a live run that reaches a loopback endpoint, or a +// mock run that needs a secret, is a configuration mistake, not a variant. +export const MODES = ['live', 'mock'] +export const readMode = value => (value === undefined || value === '' ? 'live' : value) + // Fail-closed checks on the run's inputs. A loopback base URL is the mock bench (or the // usage tap, which names the real upstream in LIVE_E2E_TAP_UPSTREAM); anything else // without a usable key (dev/live-e2e/key.mjs, the reading the preflight and the bench start share) -// is refused rather than run. -export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome, repoRoot, evidence, cwd }) { +// is refused rather than run. Mock mode swaps both halves: it needs the loopback endpoint +// and refuses to be the live mode's accident, since it measures a scripted model, not a key. +export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome, repoRoot, evidence, cwd, mode = 'live' }) { const problems = [] - const key = keyReading(apiKey) - if (!key.usable) problems.push(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}`) + const mock = mode === 'mock' + if (!MODES.includes(mode)) problems.push(`LIVE_E2E_MODE is ${mode}; use ${MODES.join(' or ')}`) let host = 'api.deepseek.com' let origin = 'default official endpoint' if (baseUrl) { @@ -46,7 +54,10 @@ export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome, repoRoo } catch { problems.push(`DEEPSEEK_BASE_URL is not a URL: ${baseUrl}`) } } const loopback = /^(localhost|127\.|0\.0\.0\.0|\[::1\]|::1)/.test(host) - if (loopback) { + if (mock) { + if (!baseUrl) problems.push('mock mode needs DEEPSEEK_BASE_URL pointing at the scripted mock') + else if (!loopback) problems.push(`mock mode only drives a loopback endpoint; ${host} is not loopback`) + } else if (loopback) { if (tapUpstream) { try { const upstream = new URL(tapUpstream).host @@ -56,6 +67,10 @@ export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome, repoRoo problems.push(`DEEPSEEK_BASE_URL is loopback (${host}); the live suite never runs against the mock`) } } + if (!mock) { + const key = keyReading(apiKey) + if (!key.usable) problems.push(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}`) + } // Both path guards compare canonical paths: a relative `--home`, a symlinked `$HOME` or a // differently-cased existing segment must not make a directory read as somewhere it is not. if (home && userHome) { @@ -74,7 +89,7 @@ export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome, repoRoo if (evidence && repoRoot && isInside(evidence, join(repoRoot, 'dev/live-e2e'), cwd)) { problems.push(`the evidence directory ${evidence} is inside the grader ${join(repoRoot, 'dev/live-e2e')}: the run would record its own writes as tampering`) } - return { problems, endpoint: { host, origin, viaTap: loopback } } + return { problems, endpoint: { host, origin, viaTap: loopback && !mock, mock } } } // The run's last word (#398). Whatever the checks and the evidence scanner said, a grader that @@ -152,8 +167,10 @@ export function buildReport({ runId, revision, config, endpoint, budgets, checks } export function summaryLines(report) { + const mode = report.config.mode ?? 'live' const lines = [ `run ${report.runId} (${report.revision})`, + `mode ${mode}${mode === 'mock' ? ' (scripted model, no key)' : ''}`, `model ${report.config.model} via ${report.endpoint.viaTap ? `tap ${report.endpoint.origin} -> ${report.endpoint.host}` : report.endpoint.origin}`, `started ${report.startedAt.iso} (UTC${report.startedAt.offset}) / ${report.humanTimes.startedAt} ${report.humanTimes.zone}`, `ended ${report.endedAt.iso} (UTC${report.endedAt.offset}) / ${report.humanTimes.endedAt} ${report.humanTimes.zone}`, diff --git a/dev/live-e2e/run.mjs b/dev/live-e2e/run.mjs index 67cadf547..9464484bc 100644 --- a/dev/live-e2e/run.mjs +++ b/dev/live-e2e/run.mjs @@ -22,7 +22,8 @@ import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' import { randomUUID } from 'node:crypto' import { runScenarios } from './scenarios.mjs' -import { budgetsFrom, buildReport, guardRun, integrityOutcome, OUTCOMES, providerFailures, summaryLines, usageOf } from './report.mjs' +import { runSmoke } from './smoke.mjs' +import { budgetsFrom, buildReport, guardRun, integrityOutcome, OUTCOMES, providerFailures, readMode, summaryLines, usageOf } from './report.mjs' import { changedPaths, canonicalPath, treeManifest } from './integrity.mjs' import { scanTree } from './sanitize.mjs' import { openFlyBotWeb } from './adapter.mjs' @@ -38,6 +39,9 @@ const { values: args, positionals } = parseArgs({ tz: { type: 'string', default: process.env.TZ || 'Asia/Shanghai' }, }, }) +// live: a real key against the real endpoint. mock: the scripted mock on loopback, no key +// (#2565). One variable, so the bench, the guard and this report cannot disagree. +const MODE = readMode(process.env.LIVE_E2E_MODE) const [HOME_TEXT, PORT] = positionals if (!HOME_TEXT || !PORT) { console.error('usage: node dev/live-e2e/run.mjs [--model ref] [--harness dir] [--tap file] [--evidence dir] [--tz zone]') @@ -67,7 +71,7 @@ const conclude = (report) => { const stoppedEarly = (outcome, problems) => { const report = buildReport({ runId, revision, - config: { node: process.version, dsh: dshVersion, model: args.model }, + config: { node: process.version, dsh: dshVersion, model: args.model, mode: MODE }, endpoint: { host: 'not reached', origin: 'not reached', viaTap: false }, budgets, checks: [], usage: usageOf(tapLines()), providerFailures: providerFailures(tapLines()), startedAt, endedAt: new Date(), tz: args.tz, @@ -80,7 +84,7 @@ const stoppedEarly = (outcome, problems) => { // Guards first: never open a browser — or write anything — against a bench that should // not exist, so a guard failure prints and exits without touching the passed home. -const guard = guardRun({ apiKey: process.env.DEEPSEEK_API_KEY, baseUrl: process.env.DEEPSEEK_BASE_URL, tapUpstream: process.env.LIVE_E2E_TAP_UPSTREAM, home: HOME, userHome: homedir(), repoRoot: ROOT, evidence }) +const guard = guardRun({ apiKey: process.env.DEEPSEEK_API_KEY, baseUrl: process.env.DEEPSEEK_BASE_URL, tapUpstream: process.env.LIVE_E2E_TAP_UPSTREAM, home: HOME, userHome: homedir(), repoRoot: ROOT, evidence, mode: MODE }) if (guard.problems.length > 0) { console.error(guard.problems.join('\n')) console.error(`outcome infra_failed (exit ${OUTCOMES.infra_failed})`) @@ -105,7 +109,8 @@ try { let checks = [] let override try { - const { results, stopped } = await runScenarios(adapter, { + const runChecks = MODE === 'mock' ? runSmoke : runScenarios + const { results, stopped } = await runChecks(adapter, { checkMs: budgets.checkMs, betweenChecks: () => { const usage = usageOf(tapLines()) @@ -127,7 +132,7 @@ try { const report = buildReport({ runId, revision, - config: { node: process.version, dsh: dshVersion, model: adapter.modelId ?? args.model }, + config: { node: process.version, dsh: dshVersion, model: adapter.modelId ?? args.model, mode: MODE }, endpoint: guard.endpoint, budgets, checks, usage: usageOf(tapLines()), providerFailures: providerFailures(tapLines()), diff --git a/dev/live-e2e/smoke.mjs b/dev/live-e2e/smoke.mjs new file mode 100644 index 000000000..4e1c4111e --- /dev/null +++ b/dev/live-e2e/smoke.mjs @@ -0,0 +1,55 @@ +// The keyless browser smoke (#2565): the same FlyBot web adapter, the same bench and the +// same evidence path as the #204 suite, against the scripted mock instead of a real model. +// The four #204 tasks cannot run here — the mock answers a task prompt with a canned +// acknowledgement and writes no files — so these checks assert only what a scripted model +// can drive: a message goes in through the composer, a reply comes out, and both survive a +// reload. That is the class of regression the repository had no browser-level check for. +// +// The message is one of the mock's scripted commands (`想想`), whose reply does not echo +// the message: that keeps "a reply appeared" distinguishable from "the user's own bubble is +// rendered", which is the difference between a UI that delivered and a UI that only echoed. +// Assertions never compare the reply's wording, so a mock edit cannot make this pass or fail. +export class SmokeError extends Error {} + +const fail = message => { throw new SmokeError(message) } + +export const MESSAGE = '想想' + +// Same result shape as scenarios.mjs (id, title, outcome, problems, durationMs) so the +// runner's report, budgets and classification stay one code path. +export async function runSmoke(adapter, { checkMs, betweenChecks }) { + const checks = [] + let stopped + const step = async (id, title, run) => { + const entry = { id, title, problems: [] } + checks.push(entry) + const started = performance.now() + try { await run() } catch (error) { entry.problems.push(error.message) } + entry.durationMs = performance.now() - started + const verdict = betweenChecks ? await betweenChecks(entry) : undefined + if (verdict?.stop) { stopped = verdict.reason ?? 'stopped between checks'; return false } + return true + } + const replyLines = lines => lines.filter(line => line.who !== 'user') + const mentionsMessage = lines => lines.some(line => line.who === 'user' && line.text.includes(MESSAGE)) + + if (!await step('message-and-reply', 'a message sent through the composer gets a reply beside it', async () => { + await adapter.sendTask(MESSAGE) + await adapter.settle(checkMs) + const lines = await adapter.conversationLines() + if (!mentionsMessage(lines)) fail(`the sent message is not in the conversation: ${MESSAGE}`) + if (replyLines(lines).length === 0) fail('no reply is visible beside the message') + })) return { results: finish(), stopped } + + await step('survive-reload', 'a reload keeps the message and the reply', async () => { + await adapter.reload() + const lines = await adapter.conversationLines() + if (!mentionsMessage(lines)) fail('after reload the sent message is gone from the conversation') + if (replyLines(lines).length === 0) fail('after reload no reply is visible') + }) + + function finish() { + return checks.map(({ problems, ...entry }) => ({ ...entry, outcome: problems.length === 0 ? 'pass' : 'assertion_failed', problems })) + } + return { results: finish(), stopped } +} diff --git a/dev/live-e2e/test/fake-adapter.mjs b/dev/live-e2e/test/fake-adapter.mjs index 0b139ecd1..0a98014e1 100644 --- a/dev/live-e2e/test/fake-adapter.mjs +++ b/dev/live-e2e/test/fake-adapter.mjs @@ -14,7 +14,9 @@ const paidJson = rows => { } export class FakeAdapter { - constructor({ breakIds, mutateSummary, stalePreview, neverSettle } = {}) { + // chatOnly serves the keyless smoke (#2565): a message and a reply, no files. noReply and + // forgetOnReload break exactly the two things the smoke asserts. + constructor({ breakIds, mutateSummary, stalePreview, neverSettle, chatOnly, noReply, forgetOnReload } = {}) { this.files = new Map([['orders.csv', fixtureCsv()]]) this.conversation = [] this.delivered = [] @@ -22,12 +24,19 @@ export class FakeAdapter { this.mutateSummary = mutateSummary this.stalePreview = stalePreview this.neverSettle = neverSettle + this.chatOnly = chatOnly + this.noReply = noReply + this.forgetOnReload = forgetOnReload } async uploadFile(name, bytes) { this.files.set(name, bytes) } async sendTask(text) { this.conversation.push({ who: 'user', text }) + if (this.chatOnly) { + if (!this.noReply) this.conversation.push({ who: 'app', text: '想了一下,建议分三步走:先定目标,再看约束,最后定下一步。' }) + return + } this.conversation.push({ who: 'app', text: '好的,我来处理。' }) if (this.neverSettle) return const rows = this.files.get('orders.csv').toString('utf8').split('\n').slice(1).filter(Boolean) @@ -55,7 +64,7 @@ export class FakeAdapter { async openPreview(name) { return this.stalePreview && name === 'summary.json' ? '{"orders": [], "count": 0, "total": 0}' : this.files.get(name)?.toString('utf8') ?? '' } - async reload() { this.reloaded = true } + async reload() { this.reloaded = true; if (this.forgetOnReload) this.conversation = [] } async conversationLines() { return this.conversation.map(line => ({ ...line })) } diff --git a/dev/live-e2e/test/mock-mode.test.mjs b/dev/live-e2e/test/mock-mode.test.mjs new file mode 100644 index 000000000..fdacb0329 --- /dev/null +++ b/dev/live-e2e/test/mock-mode.test.mjs @@ -0,0 +1,67 @@ +// Keyless coverage of the mock mode and the browser smoke (#2565): the mode split in the +// guards, the smoke's two checks against a fake app, and the report line that tells a run +// that measured something from one that did not. No network, no browser, no key. +import { strict as assert } from 'node:assert' +import test from 'node:test' +import { FakeAdapter } from './fake-adapter.mjs' +import { runSmoke } from '../smoke.mjs' +import { buildReport, budgetsFrom, guardRun, readMode, summaryLines } from '../report.mjs' + +const MOCK_ENDPOINT = 'http://127.0.0.1:8765' + +test('mock mode needs no key and takes the scripted mock endpoint', () => { + const guard = guardRun({ apiKey: '', baseUrl: MOCK_ENDPOINT, mode: 'mock' }) + assert.deepEqual(guard.problems, []) + assert.equal(guard.endpoint.mock, true) + assert.equal(guard.endpoint.viaTap, false) +}) + +test('mock mode refuses an endpoint that is not the loopback mock', () => { + assert.match(guardRun({ apiKey: '', baseUrl: 'https://api.deepseek.com', mode: 'mock' }).problems.join('; '), /loopback/) + assert.match(guardRun({ apiKey: '', baseUrl: undefined, mode: 'mock' }).problems.join('; '), /DEEPSEEK_BASE_URL/) +}) + +test('the live guard still refuses a keyless loopback run', () => { + const guard = guardRun({ apiKey: '', baseUrl: MOCK_ENDPOINT }) + assert.match(guard.problems.join('; '), /DEEPSEEK_API_KEY is missing/) + assert.match(guard.problems.join('; '), /loopback/) + assert.match(guardRun({ apiKey: 'a'.repeat(32), baseUrl: MOCK_ENDPOINT }).problems.join('; '), /loopback/) + assert.deepEqual(guardRun({ apiKey: 'a'.repeat(32), baseUrl: undefined }).problems, []) +}) + +test('an unknown mode is a problem, and no variable means live', () => { + assert.equal(readMode(undefined), 'live') + assert.equal(readMode('mock'), 'mock') + assert.match(guardRun({ apiKey: 'a'.repeat(32), baseUrl: undefined, mode: 'nope' }).problems.join('; '), /LIVE_E2E_MODE is nope/) +}) + +test('the smoke passes against an app that replies', async () => { + const { results, stopped } = await runSmoke(new FakeAdapter({ chatOnly: true }), { checkMs: 1000 }) + assert.equal(stopped, undefined) + assert.deepEqual(results.map(result => result.id), ['message-and-reply', 'survive-reload']) + assert.deepEqual(results.map(result => result.outcome), ['pass', 'pass']) +}) + +test('the smoke fails when the reply never arrives', async () => { + const { results } = await runSmoke(new FakeAdapter({ chatOnly: true, noReply: true }), { checkMs: 1000 }) + assert.equal(results[0].outcome, 'assertion_failed') + assert.match(results[0].problems.join('; '), /no reply/) +}) + +test('the reload check fails when the conversation does not come back', async () => { + const { results } = await runSmoke(new FakeAdapter({ chatOnly: true, forgetOnReload: true }), { checkMs: 1000 }) + assert.equal(results[0].outcome, 'pass') + assert.equal(results[1].outcome, 'assertion_failed') + assert.match(results[1].problems.join('; '), /message is gone/) +}) + +test('the report says which mode ran, so a mock green is not a live green', () => { + const report = buildReport({ + runId: 'r', revision: 'abc', + config: { node: 'v22', dsh: '0.2.1-rc.2', model: 'deepseek-official/deepseek-flash', mode: 'mock' }, + endpoint: { host: '127.0.0.1:8765', origin: MOCK_ENDPOINT, viaTap: false, mock: true }, + budgets: budgetsFrom({}), checks: [], usage: { calls: 0, prompt: 0, cached: 0, output: 0 }, + startedAt: new Date(), endedAt: new Date(), tz: 'UTC', + }) + assert.match(summaryLines(report).join('\n'), /mode mock \(scripted model, no key\)/) +}) diff --git a/docs/subsystems/live-e2e.md b/docs/subsystems/live-e2e.md index 8ae279d0b..e25e329c5 100644 --- a/docs/subsystems/live-e2e.md +++ b/docs/subsystems/live-e2e.md @@ -13,6 +13,13 @@ DeepSeek endpoint. Commands, budgets and the evidence rules are in [the suite README](../../dev/live-e2e/README.md); the adapter contract is in [docs/interop/live-e2e-adapter.md](../interop/live-e2e-adapter.md). +Its keyless sibling (#2565) runs the same bench, adapter and evidence path with +`LIVE_E2E_MODE=mock`, against `core/dev/mock-llm.mjs` and no key at all. A scripted model +cannot do the #204 tasks (it answers a task prompt with a canned acknowledgement and writes +no files), so that mode runs the two `smoke.mjs` checks — a message gets a reply, and both +survive a reload — in a `mock` job that reads no `secrets.*` and runs on pull requests +touching `core/` or `plugins/`, not only on `main`. + Which user journeys this suite is meant to cover, and what covers each one today, is the [user-scenario matrix](../testing/user-scenarios.md) (#358). The matrix is the map; this suite is one of its columns, not the map's owner. @@ -20,8 +27,8 @@ suite is one of its columns, not the map's owner. ## Owns - `dev/live-e2e/`: the fixture and scenario/assertion modules, the FlyBot web adapter, - the real-model bench launcher, the runner (guards, budgets, report, evidence - sanitizer, grader hash) and the keyless fake-adapter tests. + the bench launcher (real-model and mock modes), the runner (guards, budgets, report, + evidence sanitizer, grader hash), the keyless smoke checks and their fake-adapter tests. - `scripts/lint-live-e2e.mjs`: lint using core's installed harness tooling. ## Invariants @@ -43,8 +50,11 @@ suite is one of its columns, not the map's owner. bench start and the run guard (`dev/live-e2e/key.mjs`): a key under 16 characters or the mock's is skipped with its own warning, never passed to a bench that refuses it. A key long enough but not real is not detected here; only the endpoint can say that (#2214). -- The mock suites stay separate: this bench never starts the mock model, and the mock - benches never gain a real key. +- The two modes never mix: `LIVE_E2E_MODE=live` requires a real key and refuses a loopback + endpoint (except the usage tap with a real upstream); `LIVE_E2E_MODE=mock` requires the + loopback scripted mock, reads no key (a key in the environment is replaced with the mock's + own before the bench starts) and refuses an endpoint that is not loopback. The bench never + starts the mock model itself, and mock mode never gains a real key. - Evidence is an explicit allow-list that passed the sanitizer (no `token=` URLs, no key material, no cookies); a sanitizer finding is an infra failure, never a pass. - The grader is hashed before the first turn and again before the report (#398): a change