diff --git a/core/dev/perf-baseline.mjs b/core/dev/perf-baseline.mjs index cb1cc95b..362873cd 100644 --- a/core/dev/perf-baseline.mjs +++ b/core/dev/perf-baseline.mjs @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs --out ]' // The mock model is scripted; this is a prompt the bench answers with one tool call, which is -// what makes the tool-call latency sample exist at all. -const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段,然后告诉我它讲了什么。' +// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes +// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string +// through `decide()` so a change on either side fails instead of warning once per turn. +export const DEFAULT_MESSAGE = '看看团队现在在忙什么。' export function nearestRank(sortedAsc, p) { if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') diff --git a/core/test/perf-baseline.test.js b/core/test/perf-baseline.test.js index 6dcb527d..26593e2a 100644 --- a/core/test/perf-baseline.test.js +++ b/core/test/perf-baseline.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { decide } from '../dev/mock-llm.mjs' + import { + DEFAULT_MESSAGE, benchCommit, buildReport, markdownTable, @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name', assert.equal(flags.timeout, 120_000) }) +test('the default message makes the scripted model call a tool (#2577)', () => { + // The bench answers only a message the scripted model routes to a tool; on any other message + // every turn warns "carried no tool call" and the run refuses to record. This is the seam, so + // a change to the default message or to the model's branches fails here instead of at run time. + const out = decide({ + tools: [{ name: 'list_bots' }, { name: 'send_message' }], + messages: [{ role: 'user', content: DEFAULT_MESSAGE }], + }) + const called = out.filter((block) => block.type === 'tool_use') + assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`) + assert.equal(called[0].name, 'list_bots') + // The message is the flag's default, so --message is the only way to override it. + assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE) +}) + test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) }) diff --git a/patches/ds-bot/files/dev/perf-baseline.mjs b/patches/ds-bot/files/dev/perf-baseline.mjs index cb1cc95b..362873cd 100644 --- a/patches/ds-bot/files/dev/perf-baseline.mjs +++ b/patches/ds-bot/files/dev/perf-baseline.mjs @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs --out ]' // The mock model is scripted; this is a prompt the bench answers with one tool call, which is -// what makes the tool-call latency sample exist at all. -const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段,然后告诉我它讲了什么。' +// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes +// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string +// through `decide()` so a change on either side fails instead of warning once per turn. +export const DEFAULT_MESSAGE = '看看团队现在在忙什么。' export function nearestRank(sortedAsc, p) { if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') diff --git a/patches/ds-bot/files/test/perf-baseline.test.js b/patches/ds-bot/files/test/perf-baseline.test.js index 6dcb527d..26593e2a 100644 --- a/patches/ds-bot/files/test/perf-baseline.test.js +++ b/patches/ds-bot/files/test/perf-baseline.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { decide } from '../dev/mock-llm.mjs' + import { + DEFAULT_MESSAGE, benchCommit, buildReport, markdownTable, @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name', assert.equal(flags.timeout, 120_000) }) +test('the default message makes the scripted model call a tool (#2577)', () => { + // The bench answers only a message the scripted model routes to a tool; on any other message + // every turn warns "carried no tool call" and the run refuses to record. This is the seam, so + // a change to the default message or to the model's branches fails here instead of at run time. + const out = decide({ + tools: [{ name: 'list_bots' }, { name: 'send_message' }], + messages: [{ role: 'user', content: DEFAULT_MESSAGE }], + }) + const called = out.filter((block) => block.type === 'tool_use') + assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`) + assert.equal(called[0].name, 'list_bots') + // The message is the flag's default, so --message is the only way to override it. + assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE) +}) + test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) })