Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions core/dev/perf-baseline.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs <bench-name> <port> --out <file
+ '[--turns 20] [--warmup 3] [--timeout 120000] [--message <text>]'

// The mock model is scripted; this is a prompt the bench answers with one tool call, which is
// what makes the tool-call latency sample exist at all.
const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段,然后告诉我它讲了什么。'
// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes
// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string
// through `decide()` so a change on either side fails instead of warning once per turn.
export const DEFAULT_MESSAGE = '看看团队现在在忙什么。'

export function nearestRank(sortedAsc, p) {
if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples')
Expand Down
18 changes: 18 additions & 0 deletions core/test/perf-baseline.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'

import { decide } from '../dev/mock-llm.mjs'

import {
DEFAULT_MESSAGE,
benchCommit,
buildReport,
markdownTable,
Expand Down Expand Up @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name',
assert.equal(flags.timeout, 120_000)
})

test('the default message makes the scripted model call a tool (#2577)', () => {
// The bench answers only a message the scripted model routes to a tool; on any other message
// every turn warns "carried no tool call" and the run refuses to record. This is the seam, so
// a change to the default message or to the model's branches fails here instead of at run time.
const out = decide({
tools: [{ name: 'list_bots' }, { name: 'send_message' }],
messages: [{ role: 'user', content: DEFAULT_MESSAGE }],
})
const called = out.filter((block) => block.type === 'tool_use')
assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`)
assert.equal(called[0].name, 'list_bots')
// The message is the flag's default, so --message is the only way to override it.
assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE)
})

test('plugin load time is null when the bench never logged a ready line', () => {
assert.equal(pluginLoadMs('/nonexistent-bench-home'), null)
})
Expand Down
6 changes: 4 additions & 2 deletions patches/ds-bot/files/dev/perf-baseline.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@ const USAGE = 'usage: node dev/perf-baseline.mjs <bench-name> <port> --out <file
+ '[--turns 20] [--warmup 3] [--timeout 120000] [--message <text>]'

// The mock model is scripted; this is a prompt the bench answers with one tool call, which is
// what makes the tool-call latency sample exist at all.
const DEFAULT_MESSAGE = '请读一下项目根目录的 README.md 第一段,然后告诉我它讲了什么。'
// what makes the tool-call latency sample exist at all. It has to be a message `decide()` routes
// to a tool (`/团队|班底|team/i` answers with `list_bots`), and a test feeds this exact string
// through `decide()` so a change on either side fails instead of warning once per turn.
export const DEFAULT_MESSAGE = '看看团队现在在忙什么。'

export function nearestRank(sortedAsc, p) {
if (!Array.isArray(sortedAsc) || sortedAsc.length === 0) throw new Error('nearestRank: no samples')
Expand Down
18 changes: 18 additions & 0 deletions patches/ds-bot/files/test/perf-baseline.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'

import { decide } from '../dev/mock-llm.mjs'

import {
DEFAULT_MESSAGE,
benchCommit,
buildReport,
markdownTable,
Expand Down Expand Up @@ -126,6 +129,21 @@ test('arguments: bench name, port and --out are required; flags parse by name',
assert.equal(flags.timeout, 120_000)
})

test('the default message makes the scripted model call a tool (#2577)', () => {
// The bench answers only a message the scripted model routes to a tool; on any other message
// every turn warns "carried no tool call" and the run refuses to record. This is the seam, so
// a change to the default message or to the model's branches fails here instead of at run time.
const out = decide({
tools: [{ name: 'list_bots' }, { name: 'send_message' }],
messages: [{ role: 'user', content: DEFAULT_MESSAGE }],
})
const called = out.filter((block) => block.type === 'tool_use')
assert.ok(called.length > 0, `the perf default message carried no tool_use: ${JSON.stringify(out)}`)
assert.equal(called[0].name, 'list_bots')
// The message is the flag's default, so --message is the only way to override it.
assert.equal(parseArgs(['perf', '3097', '--out', '/tmp/perf.json']).message, DEFAULT_MESSAGE)
})

test('plugin load time is null when the bench never logged a ready line', () => {
assert.equal(pluginLoadMs('/nonexistent-bench-home'), null)
})
Expand Down
Loading