From 70a72fae04bec767a99e37e4bc1b6e0df4849f3c Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sat, 7 Nov 2026 00:16:56 +0800 Subject: [PATCH 1/5] Add a measurable performance baseline on the mock bench Record turn end-to-end, tool-call latency, resident RSS and plugin load time from the bench, so a later slowdown has a number to be compared with. Metrics read the same Session events the activity feed folds; the script starts nothing, prints its table and writes --out JSON. No thresholds: this is a baseline, not a gate. Refs #356 --- core/dev/perf-baseline.mjs | 352 ++++++++++++++++++++++++++++++++ core/test/perf-baseline.test.js | 130 ++++++++++++ docs/performance-baseline.md | 98 +++++++++ docs/subsystems/ds-bot-core.md | 1 + 4 files changed, 581 insertions(+) create mode 100644 core/dev/perf-baseline.mjs create mode 100644 core/test/perf-baseline.test.js create mode 100644 docs/performance-baseline.md diff --git a/core/dev/perf-baseline.mjs b/core/dev/perf-baseline.mjs new file mode 100644 index 000000000..2963e3ed3 --- /dev/null +++ b/core/dev/perf-baseline.mjs @@ -0,0 +1,352 @@ +// Performance baseline on the mock bench (dev/bench.sh): the four numbers #356 fixed, measured +// from the same Session events the activity feed folds, plus the web process's resident RSS. +// +// dev/bench.sh perf 3097 8797 reset +// node dev/perf-baseline.mjs perf 3097 --out /tmp/perf.json +// dev/bench.sh perf 3097 8797 stop +// +// A baseline, not a gate: there are no thresholds. It is a baseline only for this box and this +// scripted model, and the doc says so (docs/performance-baseline.md). Tooling only: it reads the +// bench, writes the file named by --out, and leaves no process of its own behind. +// +// The Session events are read through dev/turn-split.mjs, which already decodes the zstd +// `.jsonl.zstd` frames, so a measurement here means the same events the feed folds. +import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { cpus, totalmem, type as osType, release as osRelease } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { performance } from 'node:perf_hooks' + +import { findSessionFiles, readSessionFile, sessionIdFromPath } from './turn-split.mjs' + +const USAGE = 'usage: node dev/perf-baseline.mjs --out ' + + '[--turns 20] [--warmup 3] [--timeout 120000] [--message ]' + +// The mock model is scripted; this is a prompt the bench answers with one tool call, which is +// what makes the tool-call latency sample exist at all. +const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段，然后告诉我它讲了什么。' + +export function nearestRank(sortedAsc, p) { + if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') + if (!(p > 0 && p <= 100)) throw new Error(`nearestRank: p out of range: ${p}`) + const rank = Math.ceil((p / 100) * sortedAsc.length) + return sortedAsc[Math.min(sortedAsc.length, Math.max(1, rank)) - 1] +} + +export function percentile(values, p) { + if (!Array.isArray(values) || values.length === 0) throw new Error('percentile: no samples') + const numbers = values.map(Number) + if (numbers.some((value) => !Number.isFinite(value))) throw new Error('percentile: non-finite sample') + return nearestRank([...numbers].sort((a, b) => a - b), p) +} + +// One scenario row: p50/p95 by nearest rank (no interpolation, so a number is always a sample), +// `peak` the largest sample, `samples` how many the row rests on. +export function summarize(metric, unit, values) { + if (!Array.isArray(values) || values.length === 0) return { metric, unit, samples: 0, p50: null, p95: null, peak: null } + const rounded = values.map((value) => round1(Number(value))) + return { + metric, + unit, + samples: rounded.length, + p50: round1(percentile(rounded, 50)), + p95: round1(percentile(rounded, 95)), + peak: round1(Math.max(...rounded)), + } +} + +export function round1(value) { + return Math.round(Number(value) * 10) / 10 +} + +export function machineLine() { + const cpu = cpus()[0] + const cores = cpus().length + const ramGb = round1(totalmem() / 1024 ** 3) + return `${osType()} ${osRelease()}; ${cpu?.model?.trim() ?? 'unknown CPU'}; ${cores} cores; ${ramGb} GB RAM` +} + +export function buildReport({ machine, commit, node, date, scenarios }) { + return { machine, commit, node, date, scenarios } +} + +export const REPORT_KEYS = ['machine', 'commit', 'node', 'scenarios'] +export const SCENARIO_KEYS = ['metric', 'unit', 'samples', 'p50', 'p95', 'peak'] + +// The shape the doc and the diff rely on: two runs must be comparable field by field. +export function validateReport(report) { + const problems = [] + if (!report || typeof report !== 'object') return ['report is not an object'] + for (const key of REPORT_KEYS) if (!(key in report)) problems.push(`missing ${key}`) + if (typeof report.commit !== 'string' || !/^[0-9a-f]{40}$/.test(report.commit)) problems.push('commit is not a 40-character sha') + if (typeof report.machine !== 'string' || report.machine.length < 5) problems.push('machine is not a line') + if (typeof report.node !== 'string' || !report.node.startsWith('v')) problems.push('node is not a version') + if (!Array.isArray(report.scenarios) || report.scenarios.length === 0) problems.push('scenarios is empty') + for (const [index, assumed] of (report.scenarios ?? []).entries()) { + for (const key of SCENARIO_KEYS) if (!(key in (assumed ?? {}))) problems.push(`scenarios[${index}] missing ${key}`) + if (!(assumed?.samples >= 0)) problems.push(`scenarios[${index}] samples is not a count`) + } + return problems +} + +// One markdown table: a header, then a row per scenario. A run pins itself by commit and date, +// so two runs diff row by row instead of being read as prose. +export function markdownTable(report) { + const lines = [ + ``, + '', + `Machine: ${report.machine}. Commit \`${report.commit.slice(0, 12)}\`, Node ${report.node}, run ${report.date ?? 'unknown'}.`, + '', + '| metric | unit | samples | p50 | p95 | peak |', + '| --- | --- | --- | --- | --- | --- |', + ] + for (const scenario of report.scenarios) { + lines.push(`| ${scenario.metric} | ${scenario.unit} | ${scenario.samples} | ${scenario.p50 ?? 'n/a'} | ${scenario.p95 ?? 'n/a'} | ${scenario.peak ?? 'n/a'} |`) + } + return lines.join('\n') +} + +// VmRSS of the web process the bench recorded, plus its children: `web.pid` is the pnpm that +// starts `dsh`, so the page's memory lives one level down. Peak over the run is what is reported. +export function rssMb(pid) { + const kb = rssKb(pid, 0) + return kb === null ? null : round1(kb / 1024) +} + +function rssKb(pid, depth) { + if (!pid || depth > 5) return null + let status + try { + status = readFileSync(`/proc/${pid}/status`, 'utf8') + } catch { + return null + } + const own = Number(/VmRSS:\s+(\d+) kB/.exec(status)?.[1] ?? 0) + let children = 0 + let childPids = [] + try { + childPids = readFileSync(`/proc/${pid}/task/${pid}/children`, 'utf8').trim().split(/\s+/).filter(Boolean).map(Number) + } catch { + childPids = [] + } + for (const child of childPids) children += rssKb(child, depth + 1) ?? 0 + return own + children +} + +// Plugin load time: the bench truncates web.log right before it boots `dsh`, so the log's birth +// is the process start; the ready line is `dsh web: http`, the same line bench.sh waits on. +// When the file system does not report a birth time, `null` is returned rather than a guess. +export function pluginLoadMs(home) { + const logPath = join(home, 'web.log') + let log + try { + log = readFileSync(logPath, 'utf8') + } catch { + return null + } + if (!/dsh web: http/.test(log)) return null + let start + try { + const stat = statSync(logPath, { bigint: true }) + start = stat.birthtimeMs > 0n ? Number(stat.birthtimeMs) : Number(stat.ctimeMs) + } catch { + return null + } + // The ready line is appended once, when the server is up: the file's mtime is that moment as + // long as nothing wrote afterwards. A later write only makes the number an over-estimate, and + // the doc says the number is an upper bound for the boot. + const end = Number(statSync(logPath).mtimeMs) + const ms = end - start + return Number.isFinite(ms) && ms >= 0 ? round1(ms) : null +} + +export function parseArgs(argv) { + const flags = { name: null, port: null, out: null, turns: 20, warmup: 3, timeout: 120_000, message: DEFAULT_MESSAGE } + const rest = [...argv] + while (rest.length > 0) { + const value = rest.shift() + if (value === '--out') flags.out = rest.shift() ?? null + else if (value === '--turns') flags.turns = Number(rest.shift()) + else if (value === '--warmup') flags.warmup = Number(rest.shift()) + else if (value === '--timeout') flags.timeout = Number(rest.shift()) + else if (value === '--message') flags.message = rest.shift() ?? DEFAULT_MESSAGE + else if (value.startsWith('--')) throw new Error(`unknown flag: ${value}`) + else if (flags.name === null) flags.name = value + else if (flags.port === null) flags.port = Number(value) + else throw new Error(`unexpected argument: ${value}`) + } + if (!flags.name || !flags.port) throw new Error('bench name and port are required') + if (!flags.out) throw new Error('--out is required') + if (!(flags.turns > 0) || !(flags.warmup >= 0)) throw new Error('--turns must be positive and --warmup not negative') + return flags +} + +// The bench's own login: web.log carries a one-time URL, and the cookie it sets is the session. +async function login(home, port) { + let log = '' + try { + log = readFileSync(join(home, 'web.log'), 'utf8') + } catch { + throw new Error(`bench is down: no ${join(home, 'web.log')}`) + } + const url = log.match(new RegExp(`http://127\\.0\\.0\\.1:${port}/\\?token=[^\\s]+`))?.[0] + if (!url) throw new Error(`bench is down: no login URL in web.log for port ${port}`) + const response = await fetch(url, { redirect: 'manual' }) + const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(';')[0]) + if (cookies.length === 0) throw new Error(`login returned no cookie (${response.status})`) + return cookies.join('; ') +} + +function sessionFilePath(home, sessionId) { + const root = join(home, 'sessions') + if (!existsSync(root)) return null + for (const file of findSessionFiles(root)) { + if (sessionIdFromPath(file).startsWith(sessionId)) return file + } + return null +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +async function main(argv) { + let flags + try { + flags = parseArgs(argv) + } catch (error) { + console.error(`${error.message}\n${USAGE}`) + return 2 + } + const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) + const HOME = join(ROOT, `.dsh-test-${flags.name}`) + const BASE = `http://127.0.0.1:${flags.port}` + + const cookie = await login(HOME, flags.port) + const api = async (endpoint, payload = {}) => { + const response = await fetch(`${BASE}/api/bot`, { + method: 'POST', + headers: { 'content-type': 'application/json', cookie }, + body: JSON.stringify({ endpoint, payload }), + }) + const result = await response.json() + if (!result.ok) throw new Error(`${endpoint}: ${result.error?.message ?? JSON.stringify(result)}`) + return result.value + } + + const state = await api('state') + const bots = Object.values(state?.bots ?? {}) + const chief = bots.find((bot) => bot.name === 'Chief') ?? bots.find((bot) => (state.mainBotIds ?? []).includes(bot.id)) + if (!chief?.sessionId) throw new Error('no Chief Session on this bench: reset it with bench.sh') + const file = sessionFilePath(HOME, chief.sessionId) + if (!file) throw new Error(`no Session file for ${chief.name} under ${join(HOME, 'sessions')}`) + + const rssSamples = [] + const sampleRss = () => { + let pid = null + try { + pid = Number(readFileSync(join(HOME, 'web.pid'), 'utf8').trim()) + } catch { + return + } + const value = rssMb(pid) + if (value !== null) rssSamples.push(value) + } + sampleRss() + + const turnSamples = [] + const toolSamples = [] + const seen = new Set() + const turnText = flags.message + const total = flags.warmup + flags.turns + + for (let index = 0; index < total; index += 1) { + const before = readSessionFile(file) + const beforeKeys = new Set(before.map((event) => `${event.time}:${event.type}:${JSON.stringify(event.data ?? {}).slice(0, 40)}`)) + const started = performance.now() + await api('send', { sessionId: chief.sessionId, text: turnText }) + const deadline = Date.now() + flags.timeout + let events = null + while (Date.now() < deadline) { + await sleep(50) + sampleRss() + const current = readSessionFile(file) + const fresh = current.filter((event) => !beforeKeys.has(`${event.time}:${event.type}:${JSON.stringify(event.data ?? {}).slice(0, 40)}`)) + const ended = fresh.find((event) => event.type === 'turn/end') ?? null + if (ended) { + events = { ended, fresh } + break + } + } + if (!events) throw new Error(`turn ${index + 1}/${total} timed out after ${flags.timeout} ms`) + const elapsed = performance.now() - started + sampleRss() + + const callTimes = new Map() + let toolsThisTurn = 0 + for (const event of events.fresh) { + if (event.type === 'tool/call' && event.data?.callId) callTimes.set(event.data.callId, Number(event.time)) + if (event.type === 'tool/result') { + const callId = event.data?.message?.toolCallId ?? event.data?.message?.source?.callId + const startedAt = callId === undefined ? undefined : callTimes.get(callId) + if (startedAt !== undefined && Number.isFinite(startedAt) && Number(event.time) >= startedAt) { + toolsThisTurn += 1 + const latency = Number(event.time) - startedAt + if (index >= flags.warmup) toolSamples.push(latency) + } + } + } + if (index >= flags.warmup) { + turnSamples.push(round1(elapsed)) + seen.add(toolsThisTurn) + } + if (toolsThisTurn === 0) console.error(`warning: turn ${index + 1}/${total} carried no tool call; the mock prompt did not trigger one`) + } + + if (turnSamples.length === 0) throw new Error('no measured turn') + if (toolSamples.length === 0) throw new Error('no measured tool call: the mock prompt triggered none, see the warnings') + + const load = pluginLoadMs(HOME) + const report = buildReport({ + machine: machineLine(), + commit: process.env.PERF_COMMIT ?? benchCommit(ROOT), + node: process.version, + date: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), + scenarios: [ + summarize('turn end-to-end', 'ms', turnSamples), + summarize('tool-call latency', 'ms', toolSamples), + summarize('resident RSS', 'MB', rssSamples), + summarize('plugin load time', 'ms', load === null ? [] : [load]), + ], + }) + const problems = validateReport(report) + if (problems.length > 0) throw new Error(`report is not the agreed shape: ${problems.join('; ')}`) + + writeFileSync(flags.out, `${JSON.stringify(report, null, 2)}\n`) + console.log(markdownTable(report)) + return 0 +} + +// The commit of the checkout the bench links. The bench runs without git on this host, so the +// caller may pass PERF_COMMIT instead of asking the file system. +function benchCommit(root) { + const fromEnv = process.env.PERF_COMMIT + if (fromEnv && /^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv + try { + const head = readFileSync(join(root, '.git', 'HEAD'), 'utf8').trim() + if (/^[0-9a-f]{40}$/.test(head)) return head + const ref = /^ref:\s+(\S+)$/.exec(head)?.[1] + if (ref) { + const sha = readFileSync(join(root, '.git', ref), 'utf8').trim() + if (/^[0-9a-f]{40}$/.test(sha)) return sha + } + } catch { + // no git metadata next to the checkout: a baseline still records that it cannot name one. + } + return '0'.repeat(40) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then((code) => process.exit(code)).catch((error) => { + console.error(`perf-baseline: ${error.message}`) + process.exit(1) + }) +} diff --git a/core/test/perf-baseline.test.js b/core/test/perf-baseline.test.js new file mode 100644 index 000000000..6f4fc13ef --- /dev/null +++ b/core/test/perf-baseline.test.js @@ -0,0 +1,130 @@ +// The baseline numbers are only worth recording if the code that turns samples into p50/p95 and +// into a diffable row is itself tested, and the JSON shape two runs are compared by is fixed here. +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + buildReport, + markdownTable, + nearestRank, + parseArgs, + percentile, + pluginLoadMs, + round1, + summarize, + validateReport, +} from '../dev/perf-baseline.mjs' + +test('nearest rank: one sample is its own p50 and p95', () => { + assert.equal(nearestRank([7], 50), 7) + assert.equal(nearestRank([7], 95), 7) + assert.equal(percentile([42], 50), 42) +}) + +test('nearest rank: odd n picks a real sample, never an interpolated one', () => { + const values = [10, 20, 30, 40, 50] + assert.equal(nearestRank(values, 50), 30) + assert.equal(nearestRank(values, 95), 50) + assert.equal(nearestRank(values, 20), 10) +}) + +test('nearest rank: even n uses ceil(p/100 * n), the higher of the middle pair at p50', () => { + const values = [10, 20, 30, 40] + assert.equal(nearestRank(values, 50), 20) + assert.equal(nearestRank(values, 75), 30) + assert.equal(nearestRank(values, 95), 40) + assert.equal(nearestRank(values, 100), 40) +}) + +test('percentile sorts its input and rejects an empty sample', () => { + assert.equal(percentile([30, 10, 20], 50), 20) + assert.throws(() => percentile([], 50), /no samples/) + assert.throws(() => nearestRank([1, 2], 0), /out of range/) + assert.throws(() => percentile([1, Number.NaN], 50), /non-finite/) +}) + +test('summarize keeps the agreed scenario shape, peak included', () => { + const row = summarize('turn end-to-end', 'ms', [3, 1, 2, 4]) + assert.deepEqual(Object.keys(row), ['metric', 'unit', 'samples', 'p50', 'p95', 'peak']) + assert.equal(row.metric, 'turn end-to-end') + assert.equal(row.unit, 'ms') + assert.equal(row.samples, 4) + assert.equal(row.p50, 2) + assert.equal(row.p95, 4) + assert.equal(row.peak, 4) + assert.deepEqual(summarize('plugin load time', 'ms', []), { + metric: 'plugin load time', unit: 'ms', samples: 0, p50: null, p95: null, peak: null, + }) +}) + +test('round1 keeps one decimal so two runs diff instead of drifting', () => { + assert.equal(round1(12.34), 12.3) + assert.equal(round1(12.35), 12.4) + assert.equal(round1(0), 0) +}) + +const sample = () => buildReport({ + machine: 'Linux 6.8; test CPU; 8 cores; 31.3 GB RAM', + commit: 'a'.repeat(40), + node: 'v22.23.3', + date: '2026-11-06T16:20:00Z', + scenarios: [ + summarize('turn end-to-end', 'ms', [500, 520, 480]), + summarize('tool-call latency', 'ms', [120, 130]), + summarize('resident RSS', 'MB', [512.5, 530]), + summarize('plugin load time', 'ms', [4200]), + ], +}) + +test('the report carries machine, commit, node and every scenario', () => { + const report = sample() + assert.deepEqual(validateReport(report), []) + assert.deepEqual(Object.keys(report).filter((key) => key !== 'date'), ['machine', 'commit', 'node', 'scenarios']) + assert.equal(report.scenarios.length, 4) +}) + +test('validateReport refuses what would make two runs incomparable', () => { + const missing = sample() + delete missing.commit + assert.match(validateReport(missing).join(';'), /missing commit/) + + const shortSha = sample() + shortSha.commit = 'abc123' + assert.match(validateReport(shortSha).join(';'), /40-character sha/) + + const noScenarios = sample() + noScenarios.scenarios = [] + assert.match(validateReport(noScenarios).join(';'), /scenarios is empty/) + + const unnamed = sample() + delete unnamed.scenarios[0].unit + assert.match(validateReport(unnamed).join(';'), /scenarios\[0\] missing unit/) +}) + +test('the markdown table is one header plus one row per scenario, pinned by commit', () => { + const table = markdownTable(sample()) + const lines = table.split('\n') + assert.match(lines[0], /a{40}/) + assert.match(table, /\| metric \| unit \| samples \| p50 \| p95 \| peak \|/) + assert.match(table, /\| turn end-to-end \| ms \| 3 \| 500 \| 520 \| 520 \|/) + assert.match(table, /\| plugin load time \| ms \| 1 \| 4200 \| 4200 \| 4200 \|/) + assert.equal(lines.filter((line) => line.startsWith('| ') && !line.startsWith('| ---') && !line.startsWith('| metric')).length, 4) +}) + +test('arguments: bench name, port and --out are required; flags parse by name', () => { + assert.throws(() => parseArgs([]), /required/) + assert.throws(() => parseArgs(['perf', '3097']), /--out is required/) + assert.throws(() => parseArgs(['perf', '3097', '--out', 'x', '--nope']), /unknown flag/) + const flags = parseArgs(['perf', '3097', '--out', '/tmp/perf.json', '--turns', '5', '--warmup', '1', '--message', 'hi']) + assert.equal(flags.name, 'perf') + assert.equal(flags.port, 3097) + assert.equal(flags.out, '/tmp/perf.json') + assert.equal(flags.turns, 5) + assert.equal(flags.warmup, 1) + assert.equal(flags.message, 'hi') + assert.equal(flags.timeout, 120_000) +}) + +test('plugin load time is null when the bench never logged a ready line', () => { + assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) +}) diff --git a/docs/performance-baseline.md b/docs/performance-baseline.md new file mode 100644 index 000000000..026e7f648 --- /dev/null +++ b/docs/performance-baseline.md @@ -0,0 +1,98 @@ +# Performance baseline (#356) + +Numbers for how smooth FlyBot is, so a later regression has something to be compared against. +This is a **baseline, not a gate**: nothing here is a threshold, and no check reads it. + +It measures the plugin on the **mock bench** (`core/dev/bench.sh` + the scripted model), not a +real model and not a person's machine. The container numbers answer "did the plugin get slower +at its own work"; they do not describe how long a real reply takes, because the model's time is +not in them. + +## What is measured + +| metric | unit | definition | +| --- | --- | --- | +| turn end-to-end | ms | A message POST to that turn's `turn/end`, read from the same Session events the activity feed folds. It includes the mock delay (`MOCK_DELAY_MS`, 300 ms by default), which is constant, so a change in the number is the plugin's. | +| tool-call latency | ms | `tool/call` to its `tool/result`, per call, in the same events. | +| resident RSS | MB | `VmRSS` of the `dsh` web process started by the bench (`web.pid`, plus its children — the pid is the `pnpm` that starts `dsh`), sampled after ready, during each turn and at the end; the row reports the **peak**. | +| plugin load time | ms | Process start to the `dsh web: http` line in `web.log`, which is the same line `bench.sh` waits on, so it covers every plugin's install. `web.log` is truncated right before the boot, so its birth time is the start; the ready line's moment is the file's mtime, which makes this number an **upper bound**, not an exact figure. | + +Samples: 3 warm-up turns and 20 measured turns. Percentiles are nearest rank (`ceil(p/100·n)`, +clamped), so every reported number is a real sample and no interpolation is invented. + +The manual items from the issue — scroll feel, background-tab recovery, a 500+ message session — +are **not** measured by the script and are not claimed here. They need a person in front of the +page; this script does not pretend to see them. + +## How to run + +```sh +cd core +MOCK_DELAY_MS=300 dev/bench.sh perf 3097 8797 reset +node dev/perf-baseline.mjs perf 3097 --out /tmp/perf.json +dev/bench.sh perf 3097 8797 stop +``` + +Requirements: the bench must stay up (see `docs/sandbox.md`), Node 22+, and a `deepseek-harness` +checkout for `bench.sh` (`DSH_HARNESS` or `../deepseek-harness`). The script exits non-zero when +the bench is down, when a turn times out, or when no turn produced a tool call, and it starts no +process of its own; it only reads the bench and writes the `--out` file. + +`--turns` (default 20), `--warmup` (3), `--timeout` (120000 ms) and `--message` (the prompt that +the scripted model answers with one tool call) are overridable. `PERF_COMMIT=` pins the +commit line when the bench checkout has no `.git` (the bench host runs without git). + +## Output + +`--out` is JSON and the same table is printed: + +```json +{ + "machine": "Linux ...; ; 8 cores; 31.3 GB RAM", + "commit": "<40-character sha>", + "node": "v22.23.3", + "date": "2026-11-06T16:20:00Z", + "scenarios": [ + { "metric": "turn end-to-end", "unit": "ms", "samples": 20, "p50": 0, "p95": 0, "peak": 0 } + ] +} +``` + +Scenario rows are `turn end-to-end` (ms), `tool-call latency` (ms), `resident RSS` (MB) and +`plugin load time` (ms). Two runs diff cleanly: same keys, same order, one decimal. + +## First baseline on the maintenance box + +**Not measured yet.** The delivery workflow that wrote this page runs on a host with neither a +`deepseek-harness` checkout nor `pnpm`, so `core/dev/bench.sh` cannot boot a bench there and the +four rows would be invented rather than measured. Recording invented numbers would be worse than +an empty table: the point of this page is that a later run can be compared with it. + +The command above is the run to do, and the commit to record is the one that added +`core/dev/perf-baseline.mjs`. The table to fill is: + + + +| metric | unit | samples | p50 | p95 | peak | +| --- | --- | --- | --- | --- | --- | +| turn end-to-end | ms | — | — | — | — | +| tool-call latency | ms | — | — | — | — | +| resident RSS | MB | — | — | — | — | +| plugin load time | ms | — | — | — | — | + +Machine: —. Commit: —. Node: —. Date: —. + +## Metrics to record on each daily tag (#351) + +Four numbers, the ones above — they are cheap to produce and each one moves for a different +reason: + +1. **turn end-to-end p50/p95** — the plugin's own work per turn, with the mock delay constant. +2. **tool-call latency p50/p95** — the tool plumbing, which is where a slow permission check or a + blocking file read would show up first. +3. **resident RSS peak** — memory that grows across turns even though the Session was reset. +4. **plugin load time** — every plugin's install bundled into one number, the cost a user pays + once per launch. + +No thresholds: the daily tag records the numbers, and a person decides when a move is a +regression. diff --git a/docs/subsystems/ds-bot-core.md b/docs/subsystems/ds-bot-core.md index 05b45a397..7bc657c48 100644 --- a/docs/subsystems/ds-bot-core.md +++ b/docs/subsystems/ds-bot-core.md @@ -43,6 +43,7 @@ holds. Task bookkeeping does not imply runtime execution or tool permission. | `core/dev/eval/` | User-scenario runner (#358): pure rubrics (`rubric.mjs`), canonical journey specs (`specs.mjs`), optional model judge (`judge.mjs`), grader hashing (#398, `guard.mjs`) and a CLI (`run.mjs`). Dev only, not part of `npm test`. | | `core/dev/token-report.mjs` | Reads the usage ledger into per-Bot, per-Session and per-day token numbers and each Session's prompt growth, keeping fresh input, cache read and cache write apart (#799). Read-only; `--db`, `--from`, `--to`, `--tz`, `--bot`, `--top`, `--growth`, `--json`. | | `core/dev/turn-split.mjs` | Reads Session logs into the split of one turn's prompt: the fixed block (system prompt plus tool schemas) measured as each Session's fit intercept, then history, tool results and task by character share (#796). Read-only; `--sessions`, `--session`, `--turn`, `--tz`, `--days`, `--top`, `--stats`, `--room`, `--json`. | +| `core/dev/perf-baseline.mjs` | Measures the four baseline numbers (#356) on the mock bench: turn end-to-end p50/p95, tool-call latency p50/p95, resident RSS peak, plugin load time — read from the Session events the activity feed folds, never from the page. Prints its markdown table and writes the `--out` JSON; no thresholds (see [performance-baseline](../performance-baseline.md)). Tooling only: reads the bench and starts nothing. | | `core/dev/experience/` | Offline experience library (#874 Step 2): JSON lessons under `repos///` and `stacks//`. The read API is `core/src/host/experience.js` (see [core-host](core-host.md)). `check.mjs` validates the tree and can add one entry file. Not part of the npm `files` list, and not injected into a harness tier. | | `core/src/host/hooks.js` (approvals) | Held sends bound to their owner — provenance, owner-bound decisions, stale revalidation. Chief replacement invalidates sender-side holds eagerly on `setMain` (not via `staleReason`). The authority contract lives in [team](team.md) (#52). | From 449743530439b6acbbcaddfc1910b8d6f886a263 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sat, 7 Nov 2026 00:18:10 +0800 Subject: [PATCH 2/5] Drop an unused import from the baseline script The linter runs over core/dev, and the script no longer reads the directory itself. --- core/dev/perf-baseline.mjs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/dev/perf-baseline.mjs b/core/dev/perf-baseline.mjs index 2963e3ed3..147dea4f5 100644 --- a/core/dev/perf-baseline.mjs +++ b/core/dev/perf-baseline.mjs @@ -11,7 +11,7 @@ // // The Session events are read through dev/turn-split.mjs, which already decodes the zstd // `.jsonl.zstd` frames, so a measurement here means the same events the feed folds. -import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs' import { cpus, totalmem, type as osType, release as osRelease } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' @@ -254,7 +254,6 @@ async function main(argv) { const turnSamples = [] const toolSamples = [] - const seen = new Set() const turnText = flags.message const total = flags.warmup + flags.turns @@ -296,7 +295,6 @@ async function main(argv) { } if (index >= flags.warmup) { turnSamples.push(round1(elapsed)) - seen.add(toolsThisTurn) } if (toolsThisTurn === 0) console.error(`warning: turn ${index + 1}/${total} carried no tool call; the mock prompt did not trigger one`) } From abea5f7ed7d87d061eb4c153e6384f6faa5ad592 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sat, 7 Nov 2026 00:22:23 +0800 Subject: [PATCH 3/5] Carry the baseline script in the patch split The package job holds core/ to ds-bot/ plus patches, so the two new files also live verbatim under patches/ds-bot/files/. `node scripts/ds-bot-patches.mjs check` passes. --- patches/ds-bot/files/dev/perf-baseline.mjs | 350 ++++++++++++++++++ .../ds-bot/files/test/perf-baseline.test.js | 130 +++++++ 2 files changed, 480 insertions(+) create mode 100644 patches/ds-bot/files/dev/perf-baseline.mjs create mode 100644 patches/ds-bot/files/test/perf-baseline.test.js diff --git a/patches/ds-bot/files/dev/perf-baseline.mjs b/patches/ds-bot/files/dev/perf-baseline.mjs new file mode 100644 index 000000000..147dea4f5 --- /dev/null +++ b/patches/ds-bot/files/dev/perf-baseline.mjs @@ -0,0 +1,350 @@ +// Performance baseline on the mock bench (dev/bench.sh): the four numbers #356 fixed, measured +// from the same Session events the activity feed folds, plus the web process's resident RSS. +// +// dev/bench.sh perf 3097 8797 reset +// node dev/perf-baseline.mjs perf 3097 --out /tmp/perf.json +// dev/bench.sh perf 3097 8797 stop +// +// A baseline, not a gate: there are no thresholds. It is a baseline only for this box and this +// scripted model, and the doc says so (docs/performance-baseline.md). Tooling only: it reads the +// bench, writes the file named by --out, and leaves no process of its own behind. +// +// The Session events are read through dev/turn-split.mjs, which already decodes the zstd +// `.jsonl.zstd` frames, so a measurement here means the same events the feed folds. +import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { cpus, totalmem, type as osType, release as osRelease } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { performance } from 'node:perf_hooks' + +import { findSessionFiles, readSessionFile, sessionIdFromPath } from './turn-split.mjs' + +const USAGE = 'usage: node dev/perf-baseline.mjs --out ' + + '[--turns 20] [--warmup 3] [--timeout 120000] [--message ]' + +// The mock model is scripted; this is a prompt the bench answers with one tool call, which is +// what makes the tool-call latency sample exist at all. +const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段，然后告诉我它讲了什么。' + +export function nearestRank(sortedAsc, p) { + if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') + if (!(p > 0 && p <= 100)) throw new Error(`nearestRank: p out of range: ${p}`) + const rank = Math.ceil((p / 100) * sortedAsc.length) + return sortedAsc[Math.min(sortedAsc.length, Math.max(1, rank)) - 1] +} + +export function percentile(values, p) { + if (!Array.isArray(values) || values.length === 0) throw new Error('percentile: no samples') + const numbers = values.map(Number) + if (numbers.some((value) => !Number.isFinite(value))) throw new Error('percentile: non-finite sample') + return nearestRank([...numbers].sort((a, b) => a - b), p) +} + +// One scenario row: p50/p95 by nearest rank (no interpolation, so a number is always a sample), +// `peak` the largest sample, `samples` how many the row rests on. +export function summarize(metric, unit, values) { + if (!Array.isArray(values) || values.length === 0) return { metric, unit, samples: 0, p50: null, p95: null, peak: null } + const rounded = values.map((value) => round1(Number(value))) + return { + metric, + unit, + samples: rounded.length, + p50: round1(percentile(rounded, 50)), + p95: round1(percentile(rounded, 95)), + peak: round1(Math.max(...rounded)), + } +} + +export function round1(value) { + return Math.round(Number(value) * 10) / 10 +} + +export function machineLine() { + const cpu = cpus()[0] + const cores = cpus().length + const ramGb = round1(totalmem() / 1024 ** 3) + return `${osType()} ${osRelease()}; ${cpu?.model?.trim() ?? 'unknown CPU'}; ${cores} cores; ${ramGb} GB RAM` +} + +export function buildReport({ machine, commit, node, date, scenarios }) { + return { machine, commit, node, date, scenarios } +} + +export const REPORT_KEYS = ['machine', 'commit', 'node', 'scenarios'] +export const SCENARIO_KEYS = ['metric', 'unit', 'samples', 'p50', 'p95', 'peak'] + +// The shape the doc and the diff rely on: two runs must be comparable field by field. +export function validateReport(report) { + const problems = [] + if (!report || typeof report !== 'object') return ['report is not an object'] + for (const key of REPORT_KEYS) if (!(key in report)) problems.push(`missing ${key}`) + if (typeof report.commit !== 'string' || !/^[0-9a-f]{40}$/.test(report.commit)) problems.push('commit is not a 40-character sha') + if (typeof report.machine !== 'string' || report.machine.length < 5) problems.push('machine is not a line') + if (typeof report.node !== 'string' || !report.node.startsWith('v')) problems.push('node is not a version') + if (!Array.isArray(report.scenarios) || report.scenarios.length === 0) problems.push('scenarios is empty') + for (const [index, assumed] of (report.scenarios ?? []).entries()) { + for (const key of SCENARIO_KEYS) if (!(key in (assumed ?? {}))) problems.push(`scenarios[${index}] missing ${key}`) + if (!(assumed?.samples >= 0)) problems.push(`scenarios[${index}] samples is not a count`) + } + return problems +} + +// One markdown table: a header, then a row per scenario. A run pins itself by commit and date, +// so two runs diff row by row instead of being read as prose. +export function markdownTable(report) { + const lines = [ + ``, + '', + `Machine: ${report.machine}. Commit \`${report.commit.slice(0, 12)}\`, Node ${report.node}, run ${report.date ?? 'unknown'}.`, + '', + '| metric | unit | samples | p50 | p95 | peak |', + '| --- | --- | --- | --- | --- | --- |', + ] + for (const scenario of report.scenarios) { + lines.push(`| ${scenario.metric} | ${scenario.unit} | ${scenario.samples} | ${scenario.p50 ?? 'n/a'} | ${scenario.p95 ?? 'n/a'} | ${scenario.peak ?? 'n/a'} |`) + } + return lines.join('\n') +} + +// VmRSS of the web process the bench recorded, plus its children: `web.pid` is the pnpm that +// starts `dsh`, so the page's memory lives one level down. Peak over the run is what is reported. +export function rssMb(pid) { + const kb = rssKb(pid, 0) + return kb === null ? null : round1(kb / 1024) +} + +function rssKb(pid, depth) { + if (!pid || depth > 5) return null + let status + try { + status = readFileSync(`/proc/${pid}/status`, 'utf8') + } catch { + return null + } + const own = Number(/VmRSS:\s+(\d+) kB/.exec(status)?.[1] ?? 0) + let children = 0 + let childPids = [] + try { + childPids = readFileSync(`/proc/${pid}/task/${pid}/children`, 'utf8').trim().split(/\s+/).filter(Boolean).map(Number) + } catch { + childPids = [] + } + for (const child of childPids) children += rssKb(child, depth + 1) ?? 0 + return own + children +} + +// Plugin load time: the bench truncates web.log right before it boots `dsh`, so the log's birth +// is the process start; the ready line is `dsh web: http`, the same line bench.sh waits on. +// When the file system does not report a birth time, `null` is returned rather than a guess. +export function pluginLoadMs(home) { + const logPath = join(home, 'web.log') + let log + try { + log = readFileSync(logPath, 'utf8') + } catch { + return null + } + if (!/dsh web: http/.test(log)) return null + let start + try { + const stat = statSync(logPath, { bigint: true }) + start = stat.birthtimeMs > 0n ? Number(stat.birthtimeMs) : Number(stat.ctimeMs) + } catch { + return null + } + // The ready line is appended once, when the server is up: the file's mtime is that moment as + // long as nothing wrote afterwards. A later write only makes the number an over-estimate, and + // the doc says the number is an upper bound for the boot. + const end = Number(statSync(logPath).mtimeMs) + const ms = end - start + return Number.isFinite(ms) && ms >= 0 ? round1(ms) : null +} + +export function parseArgs(argv) { + const flags = { name: null, port: null, out: null, turns: 20, warmup: 3, timeout: 120_000, message: DEFAULT_MESSAGE } + const rest = [...argv] + while (rest.length > 0) { + const value = rest.shift() + if (value === '--out') flags.out = rest.shift() ?? null + else if (value === '--turns') flags.turns = Number(rest.shift()) + else if (value === '--warmup') flags.warmup = Number(rest.shift()) + else if (value === '--timeout') flags.timeout = Number(rest.shift()) + else if (value === '--message') flags.message = rest.shift() ?? DEFAULT_MESSAGE + else if (value.startsWith('--')) throw new Error(`unknown flag: ${value}`) + else if (flags.name === null) flags.name = value + else if (flags.port === null) flags.port = Number(value) + else throw new Error(`unexpected argument: ${value}`) + } + if (!flags.name || !flags.port) throw new Error('bench name and port are required') + if (!flags.out) throw new Error('--out is required') + if (!(flags.turns > 0) || !(flags.warmup >= 0)) throw new Error('--turns must be positive and --warmup not negative') + return flags +} + +// The bench's own login: web.log carries a one-time URL, and the cookie it sets is the session. +async function login(home, port) { + let log = '' + try { + log = readFileSync(join(home, 'web.log'), 'utf8') + } catch { + throw new Error(`bench is down: no ${join(home, 'web.log')}`) + } + const url = log.match(new RegExp(`http://127\\.0\\.0\\.1:${port}/\\?token=[^\\s]+`))?.[0] + if (!url) throw new Error(`bench is down: no login URL in web.log for port ${port}`) + const response = await fetch(url, { redirect: 'manual' }) + const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(';')[0]) + if (cookies.length === 0) throw new Error(`login returned no cookie (${response.status})`) + return cookies.join('; ') +} + +function sessionFilePath(home, sessionId) { + const root = join(home, 'sessions') + if (!existsSync(root)) return null + for (const file of findSessionFiles(root)) { + if (sessionIdFromPath(file).startsWith(sessionId)) return file + } + return null +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +async function main(argv) { + let flags + try { + flags = parseArgs(argv) + } catch (error) { + console.error(`${error.message}\n${USAGE}`) + return 2 + } + const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) + const HOME = join(ROOT, `.dsh-test-${flags.name}`) + const BASE = `http://127.0.0.1:${flags.port}` + + const cookie = await login(HOME, flags.port) + const api = async (endpoint, payload = {}) => { + const response = await fetch(`${BASE}/api/bot`, { + method: 'POST', + headers: { 'content-type': 'application/json', cookie }, + body: JSON.stringify({ endpoint, payload }), + }) + const result = await response.json() + if (!result.ok) throw new Error(`${endpoint}: ${result.error?.message ?? JSON.stringify(result)}`) + return result.value + } + + const state = await api('state') + const bots = Object.values(state?.bots ?? {}) + const chief = bots.find((bot) => bot.name === 'Chief') ?? bots.find((bot) => (state.mainBotIds ?? []).includes(bot.id)) + if (!chief?.sessionId) throw new Error('no Chief Session on this bench: reset it with bench.sh') + const file = sessionFilePath(HOME, chief.sessionId) + if (!file) throw new Error(`no Session file for ${chief.name} under ${join(HOME, 'sessions')}`) + + const rssSamples = [] + const sampleRss = () => { + let pid = null + try { + pid = Number(readFileSync(join(HOME, 'web.pid'), 'utf8').trim()) + } catch { + return + } + const value = rssMb(pid) + if (value !== null) rssSamples.push(value) + } + sampleRss() + + const turnSamples = [] + const toolSamples = [] + const turnText = flags.message + const total = flags.warmup + flags.turns + + for (let index = 0; index < total; index += 1) { + const before = readSessionFile(file) + const beforeKeys = new Set(before.map((event) => `${event.time}:${event.type}:${JSON.stringify(event.data ?? {}).slice(0, 40)}`)) + const started = performance.now() + await api('send', { sessionId: chief.sessionId, text: turnText }) + const deadline = Date.now() + flags.timeout + let events = null + while (Date.now() < deadline) { + await sleep(50) + sampleRss() + const current = readSessionFile(file) + const fresh = current.filter((event) => !beforeKeys.has(`${event.time}:${event.type}:${JSON.stringify(event.data ?? {}).slice(0, 40)}`)) + const ended = fresh.find((event) => event.type === 'turn/end') ?? null + if (ended) { + events = { ended, fresh } + break + } + } + if (!events) throw new Error(`turn ${index + 1}/${total} timed out after ${flags.timeout} ms`) + const elapsed = performance.now() - started + sampleRss() + + const callTimes = new Map() + let toolsThisTurn = 0 + for (const event of events.fresh) { + if (event.type === 'tool/call' && event.data?.callId) callTimes.set(event.data.callId, Number(event.time)) + if (event.type === 'tool/result') { + const callId = event.data?.message?.toolCallId ?? event.data?.message?.source?.callId + const startedAt = callId === undefined ? undefined : callTimes.get(callId) + if (startedAt !== undefined && Number.isFinite(startedAt) && Number(event.time) >= startedAt) { + toolsThisTurn += 1 + const latency = Number(event.time) - startedAt + if (index >= flags.warmup) toolSamples.push(latency) + } + } + } + if (index >= flags.warmup) { + turnSamples.push(round1(elapsed)) + } + if (toolsThisTurn === 0) console.error(`warning: turn ${index + 1}/${total} carried no tool call; the mock prompt did not trigger one`) + } + + if (turnSamples.length === 0) throw new Error('no measured turn') + if (toolSamples.length === 0) throw new Error('no measured tool call: the mock prompt triggered none, see the warnings') + + const load = pluginLoadMs(HOME) + const report = buildReport({ + machine: machineLine(), + commit: process.env.PERF_COMMIT ?? benchCommit(ROOT), + node: process.version, + date: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), + scenarios: [ + summarize('turn end-to-end', 'ms', turnSamples), + summarize('tool-call latency', 'ms', toolSamples), + summarize('resident RSS', 'MB', rssSamples), + summarize('plugin load time', 'ms', load === null ? [] : [load]), + ], + }) + const problems = validateReport(report) + if (problems.length > 0) throw new Error(`report is not the agreed shape: ${problems.join('; ')}`) + + writeFileSync(flags.out, `${JSON.stringify(report, null, 2)}\n`) + console.log(markdownTable(report)) + return 0 +} + +// The commit of the checkout the bench links. The bench runs without git on this host, so the +// caller may pass PERF_COMMIT instead of asking the file system. +function benchCommit(root) { + const fromEnv = process.env.PERF_COMMIT + if (fromEnv && /^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv + try { + const head = readFileSync(join(root, '.git', 'HEAD'), 'utf8').trim() + if (/^[0-9a-f]{40}$/.test(head)) return head + const ref = /^ref:\s+(\S+)$/.exec(head)?.[1] + if (ref) { + const sha = readFileSync(join(root, '.git', ref), 'utf8').trim() + if (/^[0-9a-f]{40}$/.test(sha)) return sha + } + } catch { + // no git metadata next to the checkout: a baseline still records that it cannot name one. + } + return '0'.repeat(40) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then((code) => process.exit(code)).catch((error) => { + console.error(`perf-baseline: ${error.message}`) + process.exit(1) + }) +} diff --git a/patches/ds-bot/files/test/perf-baseline.test.js b/patches/ds-bot/files/test/perf-baseline.test.js new file mode 100644 index 000000000..6f4fc13ef --- /dev/null +++ b/patches/ds-bot/files/test/perf-baseline.test.js @@ -0,0 +1,130 @@ +// The baseline numbers are only worth recording if the code that turns samples into p50/p95 and +// into a diffable row is itself tested, and the JSON shape two runs are compared by is fixed here. +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + buildReport, + markdownTable, + nearestRank, + parseArgs, + percentile, + pluginLoadMs, + round1, + summarize, + validateReport, +} from '../dev/perf-baseline.mjs' + +test('nearest rank: one sample is its own p50 and p95', () => { + assert.equal(nearestRank([7], 50), 7) + assert.equal(nearestRank([7], 95), 7) + assert.equal(percentile([42], 50), 42) +}) + +test('nearest rank: odd n picks a real sample, never an interpolated one', () => { + const values = [10, 20, 30, 40, 50] + assert.equal(nearestRank(values, 50), 30) + assert.equal(nearestRank(values, 95), 50) + assert.equal(nearestRank(values, 20), 10) +}) + +test('nearest rank: even n uses ceil(p/100 * n), the higher of the middle pair at p50', () => { + const values = [10, 20, 30, 40] + assert.equal(nearestRank(values, 50), 20) + assert.equal(nearestRank(values, 75), 30) + assert.equal(nearestRank(values, 95), 40) + assert.equal(nearestRank(values, 100), 40) +}) + +test('percentile sorts its input and rejects an empty sample', () => { + assert.equal(percentile([30, 10, 20], 50), 20) + assert.throws(() => percentile([], 50), /no samples/) + assert.throws(() => nearestRank([1, 2], 0), /out of range/) + assert.throws(() => percentile([1, Number.NaN], 50), /non-finite/) +}) + +test('summarize keeps the agreed scenario shape, peak included', () => { + const row = summarize('turn end-to-end', 'ms', [3, 1, 2, 4]) + assert.deepEqual(Object.keys(row), ['metric', 'unit', 'samples', 'p50', 'p95', 'peak']) + assert.equal(row.metric, 'turn end-to-end') + assert.equal(row.unit, 'ms') + assert.equal(row.samples, 4) + assert.equal(row.p50, 2) + assert.equal(row.p95, 4) + assert.equal(row.peak, 4) + assert.deepEqual(summarize('plugin load time', 'ms', []), { + metric: 'plugin load time', unit: 'ms', samples: 0, p50: null, p95: null, peak: null, + }) +}) + +test('round1 keeps one decimal so two runs diff instead of drifting', () => { + assert.equal(round1(12.34), 12.3) + assert.equal(round1(12.35), 12.4) + assert.equal(round1(0), 0) +}) + +const sample = () => buildReport({ + machine: 'Linux 6.8; test CPU; 8 cores; 31.3 GB RAM', + commit: 'a'.repeat(40), + node: 'v22.23.3', + date: '2026-11-06T16:20:00Z', + scenarios: [ + summarize('turn end-to-end', 'ms', [500, 520, 480]), + summarize('tool-call latency', 'ms', [120, 130]), + summarize('resident RSS', 'MB', [512.5, 530]), + summarize('plugin load time', 'ms', [4200]), + ], +}) + +test('the report carries machine, commit, node and every scenario', () => { + const report = sample() + assert.deepEqual(validateReport(report), []) + assert.deepEqual(Object.keys(report).filter((key) => key !== 'date'), ['machine', 'commit', 'node', 'scenarios']) + assert.equal(report.scenarios.length, 4) +}) + +test('validateReport refuses what would make two runs incomparable', () => { + const missing = sample() + delete missing.commit + assert.match(validateReport(missing).join(';'), /missing commit/) + + const shortSha = sample() + shortSha.commit = 'abc123' + assert.match(validateReport(shortSha).join(';'), /40-character sha/) + + const noScenarios = sample() + noScenarios.scenarios = [] + assert.match(validateReport(noScenarios).join(';'), /scenarios is empty/) + + const unnamed = sample() + delete unnamed.scenarios[0].unit + assert.match(validateReport(unnamed).join(';'), /scenarios\[0\] missing unit/) +}) + +test('the markdown table is one header plus one row per scenario, pinned by commit', () => { + const table = markdownTable(sample()) + const lines = table.split('\n') + assert.match(lines[0], /a{40}/) + assert.match(table, /\| metric \| unit \| samples \| p50 \| p95 \| peak \|/) + assert.match(table, /\| turn end-to-end \| ms \| 3 \| 500 \| 520 \| 520 \|/) + assert.match(table, /\| plugin load time \| ms \| 1 \| 4200 \| 4200 \| 4200 \|/) + assert.equal(lines.filter((line) => line.startsWith('| ') && !line.startsWith('| ---') && !line.startsWith('| metric')).length, 4) +}) + +test('arguments: bench name, port and --out are required; flags parse by name', () => { + assert.throws(() => parseArgs([]), /required/) + assert.throws(() => parseArgs(['perf', '3097']), /--out is required/) + assert.throws(() => parseArgs(['perf', '3097', '--out', 'x', '--nope']), /unknown flag/) + const flags = parseArgs(['perf', '3097', '--out', '/tmp/perf.json', '--turns', '5', '--warmup', '1', '--message', 'hi']) + assert.equal(flags.name, 'perf') + assert.equal(flags.port, 3097) + assert.equal(flags.out, '/tmp/perf.json') + assert.equal(flags.turns, 5) + assert.equal(flags.warmup, 1) + assert.equal(flags.message, 'hi') + assert.equal(flags.timeout, 120_000) +}) + +test('plugin load time is null when the bench never logged a ready line', () => { + assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) +}) From c55205cbc1910152f4dcb18089f8a8d226374f50 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sat, 7 Nov 2026 00:31:37 +0800 Subject: [PATCH 4/5] Refuse a baseline report without a named commit A zero-filled commit made two runs look comparable when neither named the code that was measured. PERF_COMMIT now has to be a real sha on a checkout without .git, the test pins the refusal (red before the fix: 11 pass 1 fail), and the doc names Node 24 as the spec version. --- core/dev/perf-baseline.mjs | 18 +++++++++++------- core/test/perf-baseline.test.js | 8 ++++++++ docs/performance-baseline.md | 11 +++++++---- patches/ds-bot/files/dev/perf-baseline.mjs | 18 +++++++++++------- .../ds-bot/files/test/perf-baseline.test.js | 8 ++++++++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/core/dev/perf-baseline.mjs b/core/dev/perf-baseline.mjs index 147dea4f5..cb1cc95b4 100644 --- a/core/dev/perf-baseline.mjs +++ b/core/dev/perf-baseline.mjs @@ -305,7 +305,7 @@ async function main(argv) { const load = pluginLoadMs(HOME) const report = buildReport({ machine: machineLine(), - commit: process.env.PERF_COMMIT ?? benchCommit(ROOT), + commit: benchCommit(ROOT), node: process.version, date: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), scenarios: [ @@ -324,10 +324,14 @@ async function main(argv) { } // The commit of the checkout the bench links. The bench runs without git on this host, so the -// caller may pass PERF_COMMIT instead of asking the file system. -function benchCommit(root) { - const fromEnv = process.env.PERF_COMMIT - if (fromEnv && /^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv +// caller may pass PERF_COMMIT instead of asking the file system. A baseline that cannot name its +// commit is not a baseline: refuse rather than record a placeholder nobody can diff against. +export function benchCommit(root, env = process.env) { + const fromEnv = env.PERF_COMMIT + if (fromEnv !== undefined) { + if (/^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv + throw new Error(`PERF_COMMIT must be a 40-character sha, got ${JSON.stringify(fromEnv)}`) + } try { const head = readFileSync(join(root, '.git', 'HEAD'), 'utf8').trim() if (/^[0-9a-f]{40}$/.test(head)) return head @@ -337,9 +341,9 @@ function benchCommit(root) { if (/^[0-9a-f]{40}$/.test(sha)) return sha } } catch { - // no git metadata next to the checkout: a baseline still records that it cannot name one. + throw new Error('PERF_COMMIT=<40-character sha> is required on a checkout without .git') } - return '0'.repeat(40) + throw new Error('PERF_COMMIT=<40-character sha> is required when .git holds no commit') } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/core/test/perf-baseline.test.js b/core/test/perf-baseline.test.js index 6f4fc13ef..6dcb527d5 100644 --- a/core/test/perf-baseline.test.js +++ b/core/test/perf-baseline.test.js @@ -4,6 +4,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { + benchCommit, buildReport, markdownTable, nearestRank, @@ -128,3 +129,10 @@ test('arguments: bench name, port and --out are required; flags parse by name', test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) }) + +test('a report never records a placeholder commit', () => { + const sha = 'b'.repeat(40) + assert.equal(benchCommit('/nonexistent-checkout', { PERF_COMMIT: sha }), sha) + assert.throws(() => benchCommit('/nonexistent-checkout', {}), /PERF_COMMIT=<40-character sha> is required/) + assert.throws(() => benchCommit('/nonexistent-checkout', { PERF_COMMIT: 'abc123' }), /must be a 40-character sha/) +}) diff --git a/docs/performance-baseline.md b/docs/performance-baseline.md index 026e7f648..4c2a47721 100644 --- a/docs/performance-baseline.md +++ b/docs/performance-baseline.md @@ -33,10 +33,13 @@ node dev/perf-baseline.mjs perf 3097 --out /tmp/perf.json dev/bench.sh perf 3097 8797 stop ``` -Requirements: the bench must stay up (see `docs/sandbox.md`), Node 22+, and a `deepseek-harness` -checkout for `bench.sh` (`DSH_HARNESS` or `../deepseek-harness`). The script exits non-zero when -the bench is down, when a turn times out, or when no turn produced a tool call, and it starts no -process of its own; it only reads the bench and writes the `--out` file. +Requirements: the bench must stay up (see `docs/sandbox.md`), Node 24 (the version the spec and CI +use; Node 22+ also runs it), and a `deepseek-harness` checkout for `bench.sh` (`DSH_HARNESS` or +`../deepseek-harness`). The script exits non-zero when +the bench is down, when a turn times out, when no turn produced a tool call, or when it cannot +name the commit (`.git` absent and `PERF_COMMIT` unset) — a baseline with a placeholder commit is +refused, not recorded. It starts no process of its own; it only reads the bench and writes the +`--out` file. `--turns` (default 20), `--warmup` (3), `--timeout` (120000 ms) and `--message` (the prompt that the scripted model answers with one tool call) are overridable. `PERF_COMMIT=` pins the diff --git a/patches/ds-bot/files/dev/perf-baseline.mjs b/patches/ds-bot/files/dev/perf-baseline.mjs index 147dea4f5..cb1cc95b4 100644 --- a/patches/ds-bot/files/dev/perf-baseline.mjs +++ b/patches/ds-bot/files/dev/perf-baseline.mjs @@ -305,7 +305,7 @@ async function main(argv) { const load = pluginLoadMs(HOME) const report = buildReport({ machine: machineLine(), - commit: process.env.PERF_COMMIT ?? benchCommit(ROOT), + commit: benchCommit(ROOT), node: process.version, date: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), scenarios: [ @@ -324,10 +324,14 @@ async function main(argv) { } // The commit of the checkout the bench links. The bench runs without git on this host, so the -// caller may pass PERF_COMMIT instead of asking the file system. -function benchCommit(root) { - const fromEnv = process.env.PERF_COMMIT - if (fromEnv && /^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv +// caller may pass PERF_COMMIT instead of asking the file system. A baseline that cannot name its +// commit is not a baseline: refuse rather than record a placeholder nobody can diff against. +export function benchCommit(root, env = process.env) { + const fromEnv = env.PERF_COMMIT + if (fromEnv !== undefined) { + if (/^[0-9a-f]{40}$/.test(fromEnv)) return fromEnv + throw new Error(`PERF_COMMIT must be a 40-character sha, got ${JSON.stringify(fromEnv)}`) + } try { const head = readFileSync(join(root, '.git', 'HEAD'), 'utf8').trim() if (/^[0-9a-f]{40}$/.test(head)) return head @@ -337,9 +341,9 @@ function benchCommit(root) { if (/^[0-9a-f]{40}$/.test(sha)) return sha } } catch { - // no git metadata next to the checkout: a baseline still records that it cannot name one. + throw new Error('PERF_COMMIT=<40-character sha> is required on a checkout without .git') } - return '0'.repeat(40) + throw new Error('PERF_COMMIT=<40-character sha> is required when .git holds no commit') } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/patches/ds-bot/files/test/perf-baseline.test.js b/patches/ds-bot/files/test/perf-baseline.test.js index 6f4fc13ef..6dcb527d5 100644 --- a/patches/ds-bot/files/test/perf-baseline.test.js +++ b/patches/ds-bot/files/test/perf-baseline.test.js @@ -4,6 +4,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { + benchCommit, buildReport, markdownTable, nearestRank, @@ -128,3 +129,10 @@ test('arguments: bench name, port and --out are required; flags parse by name', test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) }) + +test('a report never records a placeholder commit', () => { + const sha = 'b'.repeat(40) + assert.equal(benchCommit('/nonexistent-checkout', { PERF_COMMIT: sha }), sha) + assert.throws(() => benchCommit('/nonexistent-checkout', {}), /PERF_COMMIT=<40-character sha> is required/) + assert.throws(() => benchCommit('/nonexistent-checkout', { PERF_COMMIT: 'abc123' }), /must be a 40-character sha/) +}) From 855b5e70916ee34f6316458c308019f288658d8f Mon Sep 17 00:00:00 2001 From: Birfy Date: Sat, 7 Nov 2026 01:08:52 +0800 Subject: [PATCH 5/5] Record why the baseline table is empty, and where the run is tracked docs/performance-baseline.md now names the real blocker: this host cannot build the deepseek-harness checkout (no git for its version step, no C compiler for the native/system addon), so dev/bench.sh cannot boot a bench. The attempt is tracked in #2571 and the four rows stay unmeasured rather than invented. Fixes #356 --- docs/performance-baseline.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/performance-baseline.md b/docs/performance-baseline.md index 4c2a47721..d2ffe68be 100644 --- a/docs/performance-baseline.md +++ b/docs/performance-baseline.md @@ -66,10 +66,7 @@ Scenario rows are `turn end-to-end` (ms), `tool-call latency` (ms), `resident RS ## First baseline on the maintenance box -**Not measured yet.** The delivery workflow that wrote this page runs on a host with neither a -`deepseek-harness` checkout nor `pnpm`, so `core/dev/bench.sh` cannot boot a bench there and the -four rows would be invented rather than measured. Recording invented numbers would be worse than -an empty table: the point of this page is that a later run can be compared with it. +**Not measured — this host cannot build the deepseek-harness checkout (no `git`, no C compiler); tracked in #2571.** The checkout present here is pristine source, and building it needs `git` for its version step and a C compiler for the `native/system` addon, so `core/dev/bench.sh` cannot boot a bench and the four rows would be invented rather than measured. Recording invented numbers would be worse than an empty table: the point of this page is that a later run can be compared with it. The command above is the run to do, and the commit to record is the one that added `core/dev/perf-baseline.mjs`. The table to fill is: