From f558c471fcb591e2606f867a77d4dca899f3c4a5 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sat, 7 Nov 2026 14:04:16 +0800 Subject: [PATCH] fix(dev): make the perf baseline's default message trigger a tool call (#2577) DEFAULT_MESSAGE asked a README question the scripted model answers with plain text, so every turn warned "carried no tool call" and the run refused to record: the baseline could not be taken in its default configuration. The default is now a prompt the mock routes to list_bots, and it is exported so a test feeds that exact string through decide() and fails if either side drifts. The two files are carried whole in patches/ds-bot/files/, so those mirrors move with them. Docs-not-needed: the default message is not stated in docs/subsystems/ds-bot-core.md; the perf baseline row still describes the script correctly. --- core/dev/perf-baseline.mjs | 6 ++++-- core/test/perf-baseline.test.js | 18 ++++++++++++++++++ patches/ds-bot/files/dev/perf-baseline.mjs | 6 ++++-- .../ds-bot/files/test/perf-baseline.test.js | 18 ++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/core/dev/perf-baseline.mjs b/core/dev/perf-baseline.mjs index cb1cc95b..362873cd 100644 --- a/core/dev/perf-baseline.mjs +++ b/core/dev/perf-baseline.mjs @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs --out ]' // The mock model is scripted; this is a prompt the bench answers with one tool call, which is -// what makes the tool-call latency sample exist at all. -const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段，然后告诉我它讲了什么。' +// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes +// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string +// through `decide()` so a change on either side fails instead of warning once per turn. +export const DEFAULT_MESSAGE = '看看团队现在在忙什么。' export function nearestRank(sortedAsc, p) { if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') diff --git a/core/test/perf-baseline.test.js b/core/test/perf-baseline.test.js index 6dcb527d..26593e2a 100644 --- a/core/test/perf-baseline.test.js +++ b/core/test/perf-baseline.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { decide } from '../dev/mock-llm.mjs' + import { + DEFAULT_MESSAGE, benchCommit, buildReport, markdownTable, @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name', assert.equal(flags.timeout, 120_000) }) +test('the default message makes the scripted model call a tool (#2577)', () => { + // The bench answers only a message the scripted model routes to a tool; on any other message + // every turn warns "carried no tool call" and the run refuses to record. This is the seam, so + // a change to the default message or to the model's branches fails here instead of at run time. + const out = decide({ + tools: [{ name: 'list_bots' }, { name: 'send_message' }], + messages: [{ role: 'user', content: DEFAULT_MESSAGE }], + }) + const called = out.filter((block) => block.type === 'tool_use') + assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`) + assert.equal(called[0].name, 'list_bots') + // The message is the flag's default, so --message is the only way to override it. + assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE) +}) + test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) }) diff --git a/patches/ds-bot/files/dev/perf-baseline.mjs b/patches/ds-bot/files/dev/perf-baseline.mjs index cb1cc95b..362873cd 100644 --- a/patches/ds-bot/files/dev/perf-baseline.mjs +++ b/patches/ds-bot/files/dev/perf-baseline.mjs @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs --out ]' // The mock model is scripted; this is a prompt the bench answers with one tool call, which is -// what makes the tool-call latency sample exist at all. -const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段，然后告诉我它讲了什么。' +// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes +// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string +// through `decide()` so a change on either side fails instead of warning once per turn. +export const DEFAULT_MESSAGE = '看看团队现在在忙什么。' export function nearestRank(sortedAsc, p) { if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples') diff --git a/patches/ds-bot/files/test/perf-baseline.test.js b/patches/ds-bot/files/test/perf-baseline.test.js index 6dcb527d..26593e2a 100644 --- a/patches/ds-bot/files/test/perf-baseline.test.js +++ b/patches/ds-bot/files/test/perf-baseline.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { decide } from '../dev/mock-llm.mjs' + import { + DEFAULT_MESSAGE, benchCommit, buildReport, markdownTable, @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name', assert.equal(flags.timeout, 120_000) }) +test('the default message makes the scripted model call a tool (#2577)', () => { + // The bench answers only a message the scripted model routes to a tool; on any other message + // every turn warns "carried no tool call" and the run refuses to record. This is the seam, so + // a change to the default message or to the model's branches fails here instead of at run time. + const out = decide({ + tools: [{ name: 'list_bots' }, { name: 'send_message' }], + messages: [{ role: 'user', content: DEFAULT_MESSAGE }], + }) + const called = out.filter((block) => block.type === 'tool_use') + assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`) + assert.equal(called[0].name, 'list_bots') + // The message is the flag's default, so --message is the only way to override it. + assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE) +}) + test('plugin load time is null when the bench never logged a ready line', () => { assert.equal(pluginLoadMs('/nonexistent-bench-home'), null) })