workflow: cut a node prompt and a step-result body on code points

changelog.d/2429-workflow-text-bounds-code-points.fixed.md

@@ -0,0 +1 @@
+A Workflow node's prompt and the step-result body kept from a Bot's reply now cut on code points, so a bound that lands inside an emoji stores the whole character instead of half a surrogate pair that every reader draws as `�` ([#2429](https://github.com/ThinkFlowLab/FlyBot/issues/2429)): the 4,000 and 2,000 character bounds also count what a person counts, so an emoji-heavy prompt keeps its full length.

plugins/workflow/src/graph.js

@@ -20,10 +20,10 @@ function stripControls(value) {
 
 // Cut on code points, never on UTF-16 units: a unit offset can split a surrogate
 // pair and leave a lone surrogate, which renders as a replacement character.
+const cut = (text, max) => (!Number.isFinite(max) || text.length <= max ? text : Array.from(text).slice(0, max).join(''))
+
 export function cleanLine(value, max = 300) {
-  const text = stripControls(value).replace(/\s+/g, ' ').trim()
-  if (!Number.isFinite(max) || text.length <= max) return text
-  return Array.from(text).slice(0, max).join('')
+  return cut(stripControls(value).replace(/\s+/g, ' ').trim(), max)
 }
 
 // A node prompt keeps line breaks. Other controls become spaces.
@@ -34,7 +34,7 @@ export function cleanPrompt(value, max = 4000) {
     if (code === 0x0a || code === 0x09) { out += ch; continue }
     out += code <= 0x1f || code === 0x7f ? ' ' : ch
   }
-  return out.replace(/[ \t]+\n/g, '\n').trim().slice(0, max)
+  return cut(out.replace(/[ \t]+\n/g, '\n').trim(), max)
 }
 
 export function cleanTools(value) {
@@ -78,7 +78,7 @@ export function readStepResult(text, node) {
     if (!match) { kept.push(line); continue }
     outputs[match[1].toLowerCase()] = cleanLine(match[2], 2000)
   }
-  const body = kept.join('\n').trim().slice(0, 2000)
+  const body = cut(kept.join('\n').trim(), 2000)
   const declared = node?.outputs ?? []
   if (declared.length === 1 && !outputs[declared[0].name] && body) outputs[declared[0].name] = body
   if (declared.length > 0) {

plugins/workflow/test/prompt-bounds.test.js

@@ -0,0 +1,27 @@
+// The prompt and step-result text bounds of graph.js cut on code points, like cleanLine
+// above them in the same file (#1705). A bound that lands between the halves of a UTF-16
+// surrogate pair stores a lone surrogate, which every encoder turns into U+FFFD and every
+// reader draws as a replacement character.
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { cleanPrompt, readStepResult } from '../src/graph.js'
+
+test('a prompt bound lands on a code point boundary, not inside an emoji', () => {
+  const kept = cleanPrompt(`${'a'.repeat(3999)}🚀tail`)
+  assert.equal(kept.isWellFormed(), true, 'the clipped prompt keeps half a surrogate pair')
+  assert.deepEqual([...kept].length, 4000, 'the prompt keeps its 4000-character bound')
+  assert.equal([...kept].at(-1), '🚀', 'the last whole character is the emoji, not half of it')
+})
+
+test('the prompt bound counts code points, so whole emoji are never cut in half', () => {
+  const kept = cleanPrompt('🚀'.repeat(4000))
+  assert.deepEqual([...kept].length, 4000, 'all 4000 code points survive the 4000 bound')
+  assert.equal(kept.length, 8000, 'the bound is counted in code points, so no emoji is cut in half')
+})
+
+test('a step result body bound lands on a code point boundary, not inside an emoji', () => {
+  const result = readStepResult(`${'x'.repeat(1999)}🚀y`, { outputs: [] })
+  assert.equal(result.text.isWellFormed(), true, 'the stored step result keeps half a surrogate pair')
+  assert.deepEqual([...result.text].length, 2000, 'the body keeps its 2000-character bound')
+  assert.equal([...result.text].at(-1), '🚀', 'the last whole character is the emoji, not half of it')
+})