host: cut the remaining code-unit bounds on a code point boundary

changelog.d/1916-code-unit-cuts.fixed.md

@@ -0,0 +1 @@
+Eight more places stopped storing or showing half of an emoji ([#1916](https://github.com/ThinkFlowLab/FlyBot/issues/1916)): the claim cue's fallback reference, the task lead's name in a member's prompt, a failed group Session's error text, `read_own_chat`'s query and anchor, an open page's address and title, a diagnostics browser-log line, and a friend's Bot name now cut on a code point boundary. The rule itself moved to `core/src/shared/text-boundary.js`, so the shared label code and the host's `cut` use one implementation instead of two.

core/client.js

@@ -1718,10 +1718,18 @@ var call = async (endpoint, payload, signal) => {
   return result.value;
 };
 
+// src/shared/text-boundary.js
+var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
+var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
+var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at;
+var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end;
+var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max });
+
 // src/shared/remote-label.js
 function cleanPeerName(name, fallback = "") {
-  const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim().slice(0, 60);
-  return text2 || fallback;
+  const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim();
+  const label = cut(text2, 60).text;
+  return label || fallback;
 }
 function normRemoteHandle(handle) {
   return `@${String(handle ?? "").trim().replace(/^@+/, "").toLowerCase()}`;

core/src/host/browser.js

@@ -251,7 +251,7 @@ export function install(rt) {
     if (value === null || typeof value !== 'object') return open
     for (const [botId, page] of Object.entries(value).slice(0, OPEN_MAX)) {
       if (rt.botOf(botId) === undefined || typeof page?.url !== 'string') continue
-      open[botId] = { url: page.url.slice(0, 2000), title: String(page.title ?? '').slice(0, 300) }
+      open[botId] = { url: cut(page.url, 2000).text, title: cut(String(page.title ?? ''), 300).text }
     }
     return open
   }

core/src/host/diagnostics.js

@@ -208,7 +208,7 @@ export function install(rt) {
   }
 
   const sanitizeClient = (client) => {
-    const pick = (value, max) => (typeof value === 'string' ? clean(value.slice(0, max)) : undefined)
+    const pick = (value, max) => (typeof value === 'string' ? clean(cut(value, max).text) : undefined)
     return {
       surface: pick(client?.surface, 40),
       language: pick(client?.language, 20),
@@ -217,7 +217,7 @@ export function install(rt) {
   }
   const sanitizeClientEntries = list => (Array.isArray(list) ? list : []).slice(-20).flatMap((item) => {
     if (typeof item?.text !== 'string' || typeof item.time !== 'number') return []
-    return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(item.text.slice(0, 500)), ...(typeof item.code === 'string' ? { code: clean(item.code.slice(0, 40)) } : {}) }]
+    return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(cut(item.text, 500).text), ...(typeof item.code === 'string' ? { code: clean(cut(item.code, 40).text) } : {}) }]
   })
 
   async function report(body = {}) {

core/src/host/groups.js

@@ -48,7 +48,7 @@ export function claimCueText(task) {
   const title = claimLine(task?.title, 300)
   if (title === '') return ''
   const found = title.match(/^#(\d{1,7})\b/) ?? title.match(/#(\d{1,7})\b/)
-  const ref = found ? `#${found[1]}` : title.slice(0, CLAIM_CUE_REF_MAX)
+  const ref = found ? `#${found[1]}` : cut(title, CLAIM_CUE_REF_MAX).text
   const rest = found ? title.slice(title.indexOf(found[0]) + found[0].length).trim() : ''
   const step = claimLine(task?.note, CLAIM_CUE_STEP_MAX) || claimLine(rest, CLAIM_CUE_STEP_MAX)
   return step === '' ? `接了 ${ref}` : `接了 ${ref}，第一步是 ${step}`
@@ -327,7 +327,7 @@ export function install(rt) {
     if (leadId === member.id) {
       return 'You are this room\'s task lead (总负责人) for the board. When the group is given a job, assign it on the board with track_tasks: one overall task, and a subtask for each part (assignee is that Bot\'s name). Nobody else plans. A chat line does not assign. A card you assign is claimed: that Bot starts the work in its own chat and does not wait for a confirmation. Progress is that card\'s column and progress. A question a person has to decide is one short post in this group; do not @ a Bot to hand the work on.'
     }
-    const name = String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' ').slice(0, 80)
+    const name = cut(String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' '), 80).text
     return `${name} is this room's task lead (总负责人) for the board: it assigns the parts on the board. Claim the card it assigns you by setting its column to in-progress with track_tasks, and update that card's column and progress as the work moves. Do not record a new plan, assign work, or post a status in the group. @Name only wakes someone; it does not assign. Start the work in your own chat; do not wait for a confirmation. A question a person has to decide is one short post in this group; do not @ a Bot to hand the work on.`
   }
 
@@ -719,7 +719,7 @@ export function install(rt) {
         round.speaking = null
         slot.status = 'failed'
         round.failed.push(member.name)
-        round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: String(error?.message ?? error).slice(0, 300), time: Date.now() })
+        round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: cut(String(error?.message ?? error), 300).text, time: Date.now() })
         continue
       }
       const prompt = roomPrompt(room, member, round)
@@ -961,8 +961,8 @@ export function install(rt) {
       if (botId === undefined) return 'Only Bots on the team can use read_own_chat.'
       const member = rt.botOf(botId)
       const room = rt.roomOf(owner?.roomId)
-      const query = String(args.query ?? '').trim().slice(0, 80)
-      const around = String(args.around ?? '').trim().slice(0, 40)
+      const query = cut(String(args.query ?? '').trim(), 80).text
+      const around = cut(String(args.around ?? '').trim(), 40).text
       const parts = rt.partsOf(botId)
       const number = Number.isInteger(args.part) ? args.part : undefined
       if (room !== undefined && member !== undefined) {

core/src/host/text.js

@@ -1,6 +1,9 @@
 // Pure text helpers shared by the host modules: message text, names, model refs,
 // cutting and estimating text, and the clock format of chat anchors.
 import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js'
+import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js'
+
+export { cut }
 
 // A color id saved under its legacy name reads as the current one.
 export const currentColor = color => (typeof color === 'string' && Object.hasOwn(LEGACY_COLORS, color) ? LEGACY_COLORS[color] : color)
@@ -34,18 +37,10 @@ export const codePoints = (text) => {
   return n
 }
 
-const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff
-const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff
-
-// A cut moves back onto a code point boundary: half of a surrogate pair reaches a UTF-8 encoder as
-// U+FFFD, so a clipped emoji would show up as a replacement character at every exit (#1642 for the
-// client side, here for the host).
-const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at)
-const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end)
-
-// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own
-// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped.
-export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max })
+// A cut moves back onto a code point boundary (#1642 client side, #1912 here, #1916 shared): half of
+// a surrogate pair reaches a UTF-8 encoder as U+FFFD, so a clipped emoji shows up as a replacement
+// character at every exit. The boundary rule itself lives in `shared/text-boundary.js`, because a
+// shared label (`shared/remote-label.js`) bounds a peer name too and must not import this module.
 
 // The part of a long message around the first match, so a search hit stays visible.
 export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => {

core/src/shared/remote-label.js

@@ -1,10 +1,12 @@
 // Display labels for a friend's published Bot and for a room the owner did not start.
 // Peer-controlled strings (names, homes, handles) are untrusted: strip controls and
 // markup so a name cannot look like instructions when it later reaches a model.
+import { cut } from './text-boundary.js'
 
 export function cleanPeerName(name, fallback = '') {
-  const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim().slice(0, 60)
-  return text || fallback
+  const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim()
+  const label = cut(text, 60).text
+  return label || fallback
 }
 
 export function normRemoteHandle(handle) {

core/src/shared/text-boundary.js

@@ -0,0 +1,16 @@
+// Where a bounded prefix may end. Both sides bound text (#1908/#1912/#1916): the host cuts what a
+// Bot reads back and what it stores, and a shared label cuts a peer's name. A cut that lands between
+// the halves of a surrogate pair reaches a UTF-8 encoder as U+FFFD, so every bound goes through here
+// instead of `slice`, and `host/text.js` builds its `cut` on this one implementation.
+
+const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff
+const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff
+
+// The start of a bounded window, moved back onto a code point boundary.
+export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at)
+
+// The end of a bounded prefix, moved back onto a code point boundary.
+export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end)
+
+// A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped.
+export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max })

core/test/browser-channel.test.js

@@ -1,6 +1,6 @@
 import { test } from 'node:test'
 import assert from 'node:assert/strict'
-import { createBrowserChannel, pageText, partEnd } from '../src/host/browser.js'
+import { createBrowserChannel, install, pageText, partEnd } from '../src/host/browser.js'
 
 const ids = items => items.map(item => item.id)
 
@@ -141,3 +141,27 @@ test('partEnd prefers a blank line, then a line break, and cuts hard only withou
   assert.equal(partEnd('a\n' + 'b'.repeat(20), 0, 10), 10)
   assert.equal(partEnd('short', 0, 10), 5)
 })
+
+// #1916: the open-page snapshot is what a Bot reads back from outside the team, and both bounds
+// are code-unit ones: 2000 for an address and 300 for a title must not land inside an emoji.
+test('the open-page snapshot never cuts a title or an address in half (#1916)', () => {
+  const registered = []
+  const rt = {
+    ctx: { get: () => ({ name: 'desktop' }), effect: () => {}, tools: { register: (tool) => registered.push(tool) } },
+    config: {},
+    botOf: () => ({ id: 'a' }),
+  }
+  install(rt)
+  const open = rt.browserOpen({
+    a: { url: `${'a'.repeat(1999)}\u{1F600}tail`, title: `${'b'.repeat(299)}\u{1F600}tail` },
+  })
+  assert.equal(open.a.url.isWellFormed(), true)
+  assert.equal(open.a.title.isWellFormed(), true)
+  assert.equal(open.a.url, 'a'.repeat(1999))
+  assert.equal(open.a.title, 'b'.repeat(299))
+  // A cut that lands on a whole character keeps it.
+  const roomy = rt.browserOpen({ a: { url: `${'a'.repeat(1998)}\u{1F600}`, title: `${'b'.repeat(298)}\u{1F600}` } })
+  assert.equal(roomy.a.url, `${'a'.repeat(1998)}\u{1F600}`)
+  assert.equal(roomy.a.title, `${'b'.repeat(298)}\u{1F600}`)
+  assert.equal(registered.length > 0, true)
+})

core/test/claim-cue.test.js

@@ -10,7 +10,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { test } from 'node:test'
 import { createRuntime } from '../index.js'
-import { CLAIM_CUE_REPEAT_MS } from '../src/host/groups.js'
+import { CLAIM_CUE_REPEAT_MS, claimCueText } from '../src/host/groups.js'
 import { fakeContext } from './fake-ctx.js'
 
 const team = () => ({
@@ -180,3 +180,20 @@ test('a claim outside a room posts no line (#1253)', async (t) => {
 test('the repeat window is a stated number of minutes, not forever (#1253)', () => {
   assert.equal(CLAIM_CUE_REPEAT_MS, 30 * 60 * 1000)
 })
+
+// #1916: the reference of a title with no issue number is a bounded prefix, and the bound is a
+// code-unit one: 60 must not land between the halves of an emoji, or the room notice and the
+// room log both keep an ill-formed string.
+const utf8Prefix = (whole, part) => Buffer.from(part, 'utf8').equals(Buffer.from(whole, 'utf8').subarray(0, Buffer.byteLength(part)))
+
+test('a claim cue reference cut at 60 never splits a surrogate pair (#1916)', () => {
+  const title = `${'a'.repeat(59)}\u{1F600}tail`
+  const ref = claimCueText({ title }).replace(/^接了 /, '')
+  assert.equal(ref.isWellFormed(), true)
+  assert.equal(ref.includes('\uFFFD'), false)
+  assert.ok(ref.length <= 60, `ref is ${ref.length} units`)
+  assert.equal(utf8Prefix(title, ref), true, 'the ref is a byte prefix of the title')
+  // A cut that lands on a whole character keeps the whole character.
+  const roomy = claimCueText({ title: `${'a'.repeat(58)}\u{1F600}tail` }).replace(/^接了 /, '')
+  assert.equal(roomy, `${'a'.repeat(58)}\u{1F600}`)
+})

core/test/diagnostics.test.js

@@ -92,6 +92,23 @@ test('warnings are kept, cleaned, written to the log, and read back after a rest
   quiet.mock.restore()
 })
 
+// #1916: the browser-log lines in the report are bounded, and those bounds are code-unit ones:
+// a line cut inside an emoji is stored and shown as U+FFFD in a report meant for a public issue.
+test('a browser-log line cut at its bound never splits a surrogate pair (#1916)', async (t) => {
+  const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-'))
+  t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
+  const rt = host(home, { config: { diagnosticsFile: false } })
+  const report = await rt.diagnostics.report({
+    clientErrors: [{ time: 1, text: `${'x'.repeat(499)}\u{1F600}tail`, code: `${'y'.repeat(39)}\u{1F600}tail` }],
+  })
+  const line = report.markdown.split('\n').find(item => item.includes('ERROR browser')) ?? ''
+  assert.notEqual(line, '')
+  assert.equal(line.isWellFormed(), true)
+  assert.equal(line.includes('\ufffd'), false)
+  assert.ok(line.includes('x'.repeat(499)), 'the text keeps its whole-character prefix')
+  assert.ok(line.includes(`[${'y'.repeat(39)}]`), `expected the code prefix, got ${line}`)
+})
+
 test('warnings send only cleaned text to the logger and stderr', (t) => {
   const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-'))
   t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))

core/test/group-bot-label.test.js

@@ -117,3 +117,11 @@ test('#526: real names stay as they are', () => {
   const view = roster([seatMine, { ...reviewer, label: undefined }], 'group')
   assert.equal(remoteSpeaker({ remote: { handle }, senderName: 'Reviewer' }, view, 'r').name, 'Reviewer (flybot682b8a43)')
 })
+
+// #1916: a friend's Bot or room name reaches a display label, and its 60-unit bound must not land
+// inside an emoji: the half pair would show as U+FFFD wherever the label is drawn.
+test('a peer name cut at 60 never splits a surrogate pair (#1916)', () => {
+  assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`), 'b'.repeat(59))
+  assert.equal(labels.cleanPeerName(`${'b'.repeat(58)}\u{1F600}tail`), `${'b'.repeat(58)}\u{1F600}`)
+  assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`, 'fallback').isWellFormed(), true)
+})