diff --git a/changelog.d/2429-workflow-text-bounds-code-points.fixed.md b/changelog.d/2429-workflow-text-bounds-code-points.fixed.md new file mode 100644 index 000000000..2318de6ed --- /dev/null +++ b/changelog.d/2429-workflow-text-bounds-code-points.fixed.md @@ -0,0 +1 @@ +A Workflow node's prompt and the step-result body kept from a Bot's reply now cut on code points, so a bound that lands inside an emoji stores the whole character instead of half a surrogate pair that every reader draws as `οΏ½` ([#2429](https://github.com/ThinkFlowLab/FlyBot/issues/2429)): the 4,000 and 2,000 character bounds also count what a person counts, so an emoji-heavy prompt keeps its full length. diff --git a/plugins/workflow/src/graph.js b/plugins/workflow/src/graph.js index 1308a9df5..0ab689af7 100644 --- a/plugins/workflow/src/graph.js +++ b/plugins/workflow/src/graph.js @@ -20,10 +20,10 @@ function stripControls(value) { // Cut on code points, never on UTF-16 units: a unit offset can split a surrogate // pair and leave a lone surrogate, which renders as a replacement character. +const cut = (text, max) => (!Number.isFinite(max) || text.length <= max ? text : Array.from(text).slice(0, max).join('')) + export function cleanLine(value, max = 300) { - const text = stripControls(value).replace(/\s+/g, ' ').trim() - if (!Number.isFinite(max) || text.length <= max) return text - return Array.from(text).slice(0, max).join('') + return cut(stripControls(value).replace(/\s+/g, ' ').trim(), max) } // A node prompt keeps line breaks. Other controls become spaces. @@ -34,7 +34,7 @@ export function cleanPrompt(value, max = 4000) { if (code === 0x0a || code === 0x09) { out += ch; continue } out += code <= 0x1f || code === 0x7f ? ' ' : ch } - return out.replace(/[ \t]+\n/g, '\n').trim().slice(0, max) + return cut(out.replace(/[ \t]+\n/g, '\n').trim(), max) } export function cleanTools(value) { @@ -78,7 +78,7 @@ export function readStepResult(text, node) { if (!match) { kept.push(line); continue } outputs[match[1].toLowerCase()] = cleanLine(match[2], 2000) } - const body = kept.join('\n').trim().slice(0, 2000) + const body = cut(kept.join('\n').trim(), 2000) const declared = node?.outputs ?? [] if (declared.length === 1 && !outputs[declared[0].name] && body) outputs[declared[0].name] = body if (declared.length > 0) { diff --git a/plugins/workflow/test/prompt-bounds.test.js b/plugins/workflow/test/prompt-bounds.test.js new file mode 100644 index 000000000..af2d358ed --- /dev/null +++ b/plugins/workflow/test/prompt-bounds.test.js @@ -0,0 +1,27 @@ +// The prompt and step-result text bounds of graph.js cut on code points, like cleanLine +// above them in the same file (#1705). A bound that lands between the halves of a UTF-16 +// surrogate pair stores a lone surrogate, which every encoder turns into U+FFFD and every +// reader draws as a replacement character. +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { cleanPrompt, readStepResult } from '../src/graph.js' + +test('a prompt bound lands on a code point boundary, not inside an emoji', () => { + const kept = cleanPrompt(`${'a'.repeat(3999)}πŸš€tail`) + assert.equal(kept.isWellFormed(), true, 'the clipped prompt keeps half a surrogate pair') + assert.deepEqual([...kept].length, 4000, 'the prompt keeps its 4000-character bound') + assert.equal([...kept].at(-1), 'πŸš€', 'the last whole character is the emoji, not half of it') +}) + +test('the prompt bound counts code points, so whole emoji are never cut in half', () => { + const kept = cleanPrompt('πŸš€'.repeat(4000)) + assert.deepEqual([...kept].length, 4000, 'all 4000 code points survive the 4000 bound') + assert.equal(kept.length, 8000, 'the bound is counted in code points, so no emoji is cut in half') +}) + +test('a step result body bound lands on a code point boundary, not inside an emoji', () => { + const result = readStepResult(`${'x'.repeat(1999)}πŸš€y`, { outputs: [] }) + assert.equal(result.text.isWellFormed(), true, 'the stored step result keeps half a surrogate pair') + assert.deepEqual([...result.text].length, 2000, 'the body keeps its 2000-character bound') + assert.equal([...result.text].at(-1), 'πŸš€', 'the last whole character is the emoji, not half of it') +})