Cut task cues, failure text, shares and labels on code points

changelog.d/2581-utf16-cut.fixed.md

@@ -0,0 +1 @@
+Workflow failure text, task cues, co-seat room shares, workflow avatars and role lines, and timeline labels now cut on character boundaries instead of UTF-16 units, so an emoji is never split into a half character and rendered as a replacement character ([#2581](https://github.com/ThinkFlowLab/FlyBot/issues/2581)).

core/src/host/task-drive.js

@@ -5,6 +5,7 @@
 // open a group Session and does not post the reply into the room. Assigning,
 // progress, and the next assignee are board writes; chat is not the scheduler.
 import { compareForQueue, compareForWork, occupiesSlot, sameRole } from '../shared/task-board.js'
+import { cut } from '../shared/text-boundary.js'
 import { formatRoomBoard } from './room-context.js'
 import { retiredBot } from './store.js'
 
@@ -34,7 +35,7 @@ const fingerprint = (tasks) => tasks.map(task => [
 // A queue item is offered, not owned: this cue names the item and the claim, never its note —
 // the note is the body, and reading it is the turn's job, not the wake's (#1247).
 const queueCueFor = (task) => {
-  const title = String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim().slice(0, 200)
+  const title = cut(String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim(), 200).text
   const priority = task.priority ?? 'normal'
   return `Take ${task.id} [${priority}] ${title} — queued for ${task.queueRole}. Claim it with track_tasks {claim: true, id: '${task.id}'}, then set its column to in-progress. Do not post a status in the group. Stop when it is done or waiting on an answer. Do not invent other work.`
 }
@@ -51,7 +52,7 @@ const queueFingerprint = (task) => (task === undefined ? '' : [
 ].join('\u0000'))
 
 const cueFor = (task) => {
-  const title = String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim().slice(0, 200)
+  const title = cut(String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim(), 200).text
   const priority = task.priority ?? 'normal'
   const post = task.writeBack?.state === 'pending'
     ? ` Then post the decision comment on ${task.writeBack.target} with your GitHub connector and record it with track_tasks postResult {task: '${task.id}', ok: true}, or ok: false with the error.`

core/src/shared/text-boundary.js

@@ -23,6 +23,11 @@ export const fits = (text, max) => {
 }
 
 // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped.
+// `max` stays a UTF-16 unit budget, on purpose: its callers bound the units a stored field may cost
+// (a capability description at 2000, the open-page snapshot's address and title, a busy-ack prompt
+// at 600), and those shapes are pinned by tests. `fits` above is the *character* judgment, for a
+// consumer whose budget is characters (#2321); the two questions differ on purpose, and what must
+// never differ is that the kept prefix ends on a code point boundary (#1963, #2581).
 // The input is flattened first (#1963): a string that already carried an unpaired surrogate went
 // straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit
 // for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag.

core/test/task-drive.test.js

@@ -636,3 +636,18 @@ test('the reminder cap never silences the task own work (#1172)', async (t) => {
   assert.equal(app.calls('s').length, 2, 'a working task wakes even with the comment reminder used up')
   assert.match(app.calls('s')[1].content.map(block => block.text).join('\n'), new RegExp(`Continue ${id} \\[normal\\] Ship the fix`))
 })
+
+// #2581: the cue re-cuts the title that tasks.js already bounded, and the second bound must land on
+// a code point boundary. A title of emoji is two UTF-16 units per character, so the old 200-unit
+// slice ended inside one and the other Bot's turn read a lone surrogate as U+FFFD.
+test('#2581: a title of emoji is cut on a code point boundary in the cue', async (t) => {
+  const app = await start(t)
+  // Five ASCII units plus 300 emoji units: the 200-unit offset lands inside a face.
+  const title = `Draft${'🚀'.repeat(150)}`
+  const body = await app.track('s', { tasks: [{ title, column: 'not-started' }] })
+  const id = app.idOf(body, 'Draft')
+  const cue = app.calls('s').at(-1).content.map(block => block.text ?? '').join('\n')
+  assert.equal(cue.isWellFormed(), true, 'the cue carries no lone surrogate')
+  assert.equal(cue.includes('\uFFFD'), false, 'the cue carries no replacement character')
+  assert.match(cue, new RegExp(`Continue ${id} \\[normal\\] Draft`))
+})

patches/ds-bot/files/src/host/task-drive.js

@@ -5,6 +5,7 @@
 // open a group Session and does not post the reply into the room. Assigning,
 // progress, and the next assignee are board writes; chat is not the scheduler.
 import { compareForQueue, compareForWork, occupiesSlot, sameRole } from '../shared/task-board.js'
+import { cut } from '../shared/text-boundary.js'
 import { formatRoomBoard } from './room-context.js'
 import { retiredBot } from './store.js'
 
@@ -34,7 +35,7 @@ const fingerprint = (tasks) => tasks.map(task => [
 // A queue item is offered, not owned: this cue names the item and the claim, never its note —
 // the note is the body, and reading it is the turn's job, not the wake's (#1247).
 const queueCueFor = (task) => {
-  const title = String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim().slice(0, 200)
+  const title = cut(String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim(), 200).text
   const priority = task.priority ?? 'normal'
   return `Take ${task.id} [${priority}] ${title} — queued for ${task.queueRole}. Claim it with track_tasks {claim: true, id: '${task.id}'}, then set its column to in-progress. Do not post a status in the group. Stop when it is done or waiting on an answer. Do not invent other work.`
 }
@@ -51,7 +52,7 @@ const queueFingerprint = (task) => (task === undefined ? '' : [
 ].join('\u0000'))
 
 const cueFor = (task) => {
-  const title = String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim().slice(0, 200)
+  const title = cut(String(task.title ?? '').replace(/[\r\n]+/g, ' ').trim(), 200).text
   const priority = task.priority ?? 'normal'
   const post = task.writeBack?.state === 'pending'
     ? ` Then post the decision comment on ${task.writeBack.target} with your GitHub connector and record it with track_tasks postResult {task: '${task.id}', ok: true}, or ok: false with the error.`

patches/ds-bot/files/src/shared/text-boundary.js

@@ -23,6 +23,11 @@ export const fits = (text, max) => {
 }
 
 // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped.
+// `max` stays a UTF-16 unit budget, on purpose: its callers bound the units a stored field may cost
+// (a capability description at 2000, the open-page snapshot's address and title, a busy-ack prompt
+// at 600), and those shapes are pinned by tests. `fits` above is the *character* judgment, for a
+// consumer whose budget is characters (#2321); the two questions differ on purpose, and what must
+// never differ is that the kept prefix ends on a code point boundary (#1963, #2581).
 // The input is flattened first (#1963): a string that already carried an unpaired surrogate went
 // straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit
 // for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag.

patches/ds-bot/files/test/task-drive.test.js

@@ -636,3 +636,18 @@ test('the reminder cap never silences the task own work (#1172)', async (t) => {
   assert.equal(app.calls('s').length, 2, 'a working task wakes even with the comment reminder used up')
   assert.match(app.calls('s')[1].content.map(block => block.text).join('\n'), new RegExp(`Continue ${id} \\[normal\\] Ship the fix`))
 })
+
+// #2581: the cue re-cuts the title that tasks.js already bounded, and the second bound must land on
+// a code point boundary. A title of emoji is two UTF-16 units per character, so the old 200-unit
+// slice ended inside one and the other Bot's turn read a lone surrogate as U+FFFD.
+test('#2581: a title of emoji is cut on a code point boundary in the cue', async (t) => {
+  const app = await start(t)
+  // Five ASCII units plus 300 emoji units: the 200-unit offset lands inside a face.
+  const title = `Draft${'🚀'.repeat(150)}`
+  const body = await app.track('s', { tasks: [{ title, column: 'not-started' }] })
+  const id = app.idOf(body, 'Draft')
+  const cue = app.calls('s').at(-1).content.map(block => block.text ?? '').join('\n')
+  assert.equal(cue.isWellFormed(), true, 'the cue carries no lone surrogate')
+  assert.equal(cue.includes('\uFFFD'), false, 'the cue carries no replacement character')
+  assert.match(cue, new RegExp(`Continue ${id} \\[normal\\] Draft`))
+})

plugins/social/src/social.js

@@ -101,8 +101,16 @@
   const kb = (n / 1024).toFixed(1)
   return Number(kb) >= 1024 ? `${(n / 1048576).toFixed(1)} MB` : `${kb} KB`
 }
+// Cut on code points, never on UTF-16 units: the share envelope leaves this Bot as JSON and a
+// slice that ends inside a surrogate pair reaches the other home as U+FFFD (#2581).
+export const cutCodePoints = (value, max) => {
+  const text = String(value ?? '').toWellFormed()
+  if (!Number.isFinite(max) || text.length <= max) return text
+  return Array.from(text).slice(0, max).join('')
+}
+
 // Remote strings that reach model-visible text: one line, no brackets or markup, capped.
 export const cleanText = (value, fallback = '') => String(value ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"<>`]/g, '').replace(/\s+/g, ' ').trim().slice(0, 60) || fallback
 // Owner-only draft body: keep punctuation the friend will see; strip controls and cap.
 export const draftText = (value) => String(value ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 800)
 // Owner cards never show the FLYBOT-FACE-HUMAN wire envelope (#159).
@@ -1838,7 +1846,7 @@
       who: cleanText(flybot.bot.get(msg.from)?.name, 'member'),
       kind: 'message',
       authorKind: 'human',
-      text: String(msg.text ?? '').slice(0, 18_000),
+      text: cutCodePoints(msg.text, 18_000),
       calls: [to],
     }
     return `FLYBOT-ROOM-SHARE\n${JSON.stringify(body)}`

plugins/social/test/social.test.js

@@ -1336,6 +1336,41 @@ test('#1141: a co-member reaches a shared-room Bot without being the friendship
   assert.equal((room.participants ?? []).some(p => p.handle === '@coder.bob.home' && p.leftSeq === undefined), false)
 })
 
+// #2581: the co-seat envelope leaves this Bot as JSON, and its 18 000 is a character budget.
+// A UTF-16 slice there can end inside an emoji, and the other home reads half a pair as U+FFFD.
+test('#2581: a co-seat room line keeps its emoji whole at the envelope bound', async (t) => {
+  const rooms = {
+    delivery: {
+      id: 'delivery', name: '交付组', members: ['a', 'b'], admin: 'a', named: true, log: [], seq: 0, seen: {},
+      participants: [
+        { kind: 'bot', id: 'a', role: 'admin', displayName: 'Chief', joinedSeq: 0, joinedAt: 1 },
+        { kind: 'bot', id: 'b', role: 'member', displayName: 'Writer', joinedSeq: 0, joinedAt: 1 },
+        { kind: 'bot', remote: true, id: '@coder.bob.home', handle: '@coder.bob.home', ownerHome: '@bob.home', ownerName: 'Bob', displayName: 'Coder', role: 'member', joinedSeq: 1, joinedAt: 1 },
+      ],
+    },
+  }
+  const { ctx, stop } = await team({ rooms })
+  t.after(stop)
+  const server = mockServer()
+  server.tools.list_remote_bots = () => ({ homes: [{ relationship_id: 'r1', home: { bot_id: 'h2', handle: '@bob.home', display_name: 'Bob', owner_display_name: 'Bob' }, bots: [{ bot_id: 'x1', handle: '@coder.bob.home', display_name: 'Coder' }], scope: 'member', via: ['@writer.danyang.home'] }] })
+  const social = createSocial(ctx.flybot, { autoAccept: true, inboxPollMs: 0 }, { client: createMcpClient({ server: 'https://s', token: 'owner-token', fetch: server.fetch }), home: { id: 'h1', handle: '@danyang.home' } })
+  t.after(() => social.dispose())
+  await grantTeam(ctx.flybot, '@bob.home')
+  await social.registerAll()
+  await social.remoteBots({ fresh: true })
+  await ctx.flybot.grants.set('a', [{ grantee: '@bob.home' }])
+  // 18 001 characters, 36 001 UTF-16 units: the old 18 000-unit slice kept 8 999 faces and half of the next.
+  const text = `A${'🚀'.repeat(18_000)}`
+  const sent = await ctx.flybot.bot.send({ from: 'a', to: '@coder.bob.home', text, role: 'reply', via: 'auto-reply' })
+  assert.equal(sent.action, 'allow', JSON.stringify(sent))
+  const call = server.calls.filter(item => item.name === 'send_as_member').at(-1)
+  const body = JSON.parse(String(call.args.text).slice('FLYBOT-ROOM-SHARE\n'.length))
+  assert.equal(body.text.isWellFormed(), true, 'the envelope text carries no lone surrogate')
+  assert.equal(body.text.includes('\uFFFD'), false)
+  assert.equal(Array.from(body.text).length, 18_000, 'the budget is characters, the shared text included')
+  assert.equal(body.text, `A${'🚀'.repeat(17_999)}`)
+})
+
 // A server that holds every send_as_member and lists the holds. `reasons` names what Bot Social
 // held it for: the plain approval requirement, or a finding of its own (a DLP category).
 function holdingServer({ reasons = ['mvp.all_outbound_requires_approval'], riskFlags = [], holdReason } = {}) {

plugins/timeline/src/sanitize.js

@@ -1,15 +1,25 @@
 // Everything the recorder stores and the panel shows that did not come from this plugin
 // passes through these two functions: a Bot or group name, a tool name, an id from an
 // event. The core's feed cleans its own strings as well; the plugin does not rely on that.
 const CONTROLS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g
 // Bidirectional overrides and isolates reorder text on screen; zero-width characters hide it.
 const INVISIBLE = /[\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u2069\ufeff]/g
 
+// Cut on code points, never on UTF-16 units: `max` is a character budget and a unit offset can
+// end inside a surrogate pair, so the panel or the exported JSON would carry half a character
+// (#2581). The input is flattened first, so a value that already carried an unpaired surrogate is
+// never stored as one.
+const cutCodePoints = (text, max) => {
+  const value = text.toWellFormed()
+  if (!Number.isFinite(max) || value.length <= max) return value
+  return Array.from(value).slice(0, max).join('')
+}
+
 // One line of plain text, no controls, no fence markers, no angle brackets, capped.
 export const cleanLabel = (value, max = 60) => {
   let out = String(value ?? '').replace(CONTROLS, ' ').replace(INVISIBLE, '')
   for (let previous; previous !== out;) { previous = out; out = out.replace(/<\s*\/?\s*untrusted-?/gi, '') }
-  return out.replace(/[<>`"]/g, '').replace(/\s+/g, ' ').trim().slice(0, max)
+  return cutCodePoints(out.replace(/[<>`"]/g, '').replace(/\s+/g, ' ').trim(), max)
 }
 
 const ID = /^[A-Za-z0-9][A-Za-z0-9:_#.@-]{0,159}$/

plugins/timeline/test/sanitize.test.js

@@ -54,3 +54,16 @@ test('cleanCode keeps an enumerated error code and nothing else', () => {
   assert.equal(cleanCode('AUTH'), 'AUTH')
   for (const bad of ['sk-live-0123456789abcdef', 'quota exceeded for key abc', 'rate_limit', '<b>', '', 'A'.repeat(41), 5, null, undefined]) assert.equal(cleanCode(bad), 'UNKNOWN', String(bad))
 })
+
+// #2581: 60 is a character budget. A label of emoji is 120 UTF-16 units, and a unit slice at 60
+// ends inside the 31st face, so the panel and the exported JSON carried half a character.
+test('#2581: cleanLabel cuts emoji on a code point boundary', () => {
+  const out = cleanLabel('🚀'.repeat(61))
+  assert.equal(out.isWellFormed(), true)
+  assert.equal(out.includes('\uFFFD'), false)
+  assert.equal(Array.from(out).length, 60)
+  assert.equal(out, '🚀'.repeat(60))
+  // A label already inside the budget is untouched, however many units it costs.
+  const short = '🚀'.repeat(30)
+  assert.equal(cleanLabel(short), short)
+})

plugins/workflow/client.js

@@ -295,6 +295,14 @@ window.__ModuleLoader__.load({
     const COLUMN_KEY = { 'not-started': 'Not started', 'in-progress': 'In progress', done: 'Done' }
     const COLUMN_OF = { open: 'not-started', claimed: 'not-started', working: 'in-progress', blocked: 'in-progress', 'result-ready': 'in-progress', accepted: 'done', cancelled: 'done', failed: 'done' }
     const COLORS = { Planner: '#8b5cf6', Writer: '#f0509a', Coder: '#1fbf6a', Reviewer: '#f5a524', Tester: '#14b8a6', Deployer: '#2f7cf6', '研究员': '#a78bfa', '数据分析师': '#ec4899' }
+    // One code point, not one UTF-16 unit: half a surrogate pair renders as a replacement
+    // character in the avatar circle (#2017, #2581).
+    const firstGlyph = name => Array.from(String(name ?? '').trim())[0] ?? '?'
+    // A cut in code points, so a label never ends inside a surrogate pair (#2581).
+    const cutCodePoints = (value, max) => {
+      const text = String(value ?? '')
+      return text.length <= max ? text : Array.from(text).slice(0, max).join('')
+    }
     // Zero-width plane center line: shift wire coordinates so strokes stay inside the SVG box.
     const WIRE_SPAN = 480
     const NODE_WIRE_OUT = 56
@@ -397,7 +405,7 @@ window.__ModuleLoader__.load({
       const Avatar = globalThis.__flybot__?.BotAvatar
       if (bot && typeof Avatar === 'function') return h(Avatar, { bot, size, badge: false, live: false })
       const color = COLORS[node?.name] ?? '#4d6bfe'
-      return h('span', { className: 'wf-av', style: { background: color, width: size, height: size, fontSize: Math.max(11, size * 0.42) } }, (node?.name ?? '?').slice(0, 1))
+      return h('span', { className: 'wf-av', style: { background: color, width: size, height: size, fontSize: Math.max(11, size * 0.42) } }, firstGlyph(node?.name))
     }
 
     const BOT_TINTS = ['#4d6bfe', '#8b5cf6', '#14b8a6', '#f5a524', '#ec4899', '#1fbf6a', '#2f7cf6']
@@ -410,7 +418,7 @@ window.__ModuleLoader__.load({
     function botPaletteSubtitle(bot, peers) {
       const role = String(bot?.role ?? '').replace(/\s+/g, ' ').trim()
       const dupes = (peers ?? []).filter(item => item.name === bot.name).length > 1
-      if (role) return role.length > 28 ? `${role.slice(0, 27)}…` : role
+      if (role) return Array.from(role).length > 28 ? `${cutCodePoints(role, 27)}…` : role
       if (dupes && bot?.id) return `…${String(bot.id).slice(-6)}`
       return ''
     }
@@ -422,7 +430,7 @@ window.__ModuleLoader__.load({
       return h('span', {
         className: 'wf-palette-av',
         style: { background: color, width: size, height: size, fontSize: Math.max(11, size * 0.42) },
-      }, (bot?.name ?? '?').slice(0, 1))
+      }, firstGlyph(bot?.name))
     }
 
     function PaletteBotChip({ bot, peers, onPointerDown, dragging }) {
@@ -515,7 +523,7 @@ window.__ModuleLoader__.load({
               fill: color,
               fontSize: 12,
               fontWeight: 600,
-            }, (node.name ?? '?').slice(0, 1)))
+            }, firstGlyph(node.name)))
         })))
     }
 
@@ -2690,7 +2698,7 @@ window.__ModuleLoader__.load({
       runHistoryRows, liveStageRuns, resolveFocusedRun, newestAttempt, parallelFrameStyle, fitPlane,
       editPlane, flowLayout, flowRows,
       DRAWER_FLOW_W, NODE_W, NODE_H,
-      botPaletteSubtitle, botTint, settledCall,
+      botPaletteSubtitle, botTint, settledCall, firstGlyph, MiniFlowchart,
     }
   },
 })

plugins/workflow/src/graph.js

@@ -18,6 +18,16 @@
   return out
 }
 
+// Cut on code points, never on UTF-16 units: a unit offset can split a surrogate
+// pair and leave a lone surrogate, which renders as a replacement character. `max` is a
+// code-point budget (the rule of core's shared/text-boundary.js), and the input is flattened
+// first so a value that already carried an unpaired surrogate is never stored as one.
+export function cutCodePoints(value, max) {
+  const text = String(value ?? '').toWellFormed()
+  if (!Number.isFinite(max) || text.length <= max) return text
+  return Array.from(text).slice(0, max).join('')
+}
+
 // Cut on code points, never on UTF-16 units: a unit offset can split a surrogate
 // pair and leave a lone surrogate, which renders as a replacement character.
 export function cleanLine(value, max = 300) {
@@ -687,8 +697,8 @@
   const index = layerIndex(graph, nodeId)
   const layer = graph.layers[index]
   const outputs = ctx.outputs && typeof ctx.outputs === 'object' ? ctx.outputs : {}
   const loops = { ...(ctx.loops ?? {}) }
   const fired = { ...(ctx.fired ?? {}) }
 
   // Check if this completion should end the run early (#1537, #1538)
   const earlyEnd = checkEarlyEnd(outputs, ctx.text)
@@ -767,14 +777,14 @@
       started: [],
       joined: false,
       finished: true,
       failAttempts: { ...(ctx.failAttempts ?? {}) },
       recovered: false,
       reset: [],
       earlyEnd: earlyEnd.reason,
     }
   }
 
   const failAttempts = { ...(ctx.failAttempts ?? {}) }
   const recovery = failRecoveryDecision(graph, nodeId, failAttempts)
 
   if (recovery) {