host: room turns cut card, stub and transcript text on a code point boundary

changelog.d/1960-room-context-code-point-bound.fixed.md

@@ -0,0 +1 @@
+A room turn and the stored `room.log` no longer split a surrogate pair at a character bound: the status stub's hundred units, the transcript clip, and the card, board and presence fields now cut on a code point boundary, so a title or a Bot essay that ends on an emoji keeps the whole emoji instead of an unpaired half the encoder turns into U+FFFD ([#1960](https://github.com/ThinkFlowLab/FlyBot/issues/1960)).

core/src/host/room-context.js

@@ -5,6 +5,7 @@
 // fill that slice. Progress for the open job is a compact board snapshot in the
 // same turn, not those essays.
 import { columnOf, compareForWork, progressOf } from '../shared/task-board.js'
+import { cut } from '../shared/text-boundary.js'
 import { fenceUntrusted } from './delivery.js'
 
 // Characters, not a tokenizer. The essays this was measured on are mostly CJK,
@@ -42,7 +43,7 @@ const oneLine = (value, max) => {
   }
   flat = flat.replace(/\s+/g, ' ').trim()
   if (flat.length <= max) return flat
-  return `${flat.slice(0, Math.max(0, max - 1))}…`
+  return `${cut(flat, Math.max(0, max - 1)).text}…`
 }
 
 export function statusShaped(text) {
@@ -92,7 +93,7 @@ export function boardCoversCue(tasks, { roomId, botId, text } = {}) {
 export function compactStatus(text) {
   const flat = oneLine(text, 100000)
   const mentions = [...new Set([...flat.matchAll(/@[^\s@,，。]{1,40}/g)].map(match => match[0]))].slice(0, 4)
-  const head = flat.slice(0, 100).trim()
+  const head = cut(flat, 100).text.trim()
   const more = flat.length > 100 ? '…' : ''
   const called = mentions.length > 0 ? ` ${mentions.join(' ')}` : ''
   return `[status on the board, ${flat.length} chars] ${head}${more}${called}`
@@ -139,7 +140,7 @@ export function fitRoomLines(lines, { budget = ROOM_CONTEXT_BUDGET, render, prep
     if (used + gap + text.length > budget) {
       const room = budget - used - gap
       if (room < 48) break
-      text = `${text.slice(0, room - 1)}…`
+      text = `${cut(text, room - 1).text}…`
       kept.push({ line, text })
       break
     }

core/test/room-context.test.js

@@ -213,3 +213,55 @@ test('presence: no members or no room renders nothing at all', () => {
   assert.equal(formatRoomPresence(presenceTasks(), { roomId: 'room', members: [], now: presenceAt }), '')
   assert.equal(formatRoomPresence(presenceTasks(), { members: presenceMembers, now: presenceAt }), '')
 })
+
+// P-305: a character bound must not land between the halves of a surrogate pair. `oneLine` ends a long
+// card, board or presence line with `flat.slice(0, max - 1) + '…'` and `fitRoomLines` clips a transcript
+// line with `text.slice(0, room - 1) + '…'`; both are bare `slice`, so a pair straddling the bound is
+// stored and sent whole-half — a lone high surrogate the encoder turns into U+FFFD.
+test('P-305: a card title cut at its bound keeps a whole emoji', () => {
+  // 78 units, then an emoji whose pair sits across the 80-unit bound.
+  const title = `${'a'.repeat(78)}\u{1F600}TAIL`
+  const board = formatRoomBoard({
+    card: { id: 'card', status: 'open', title, ownerBotId: 'dev2', origin: { roomId: 'room' }, createdAt: presenceAt, updatedAt: presenceAt },
+  }, { roomId: 'room', nameOf: () => 'Dev2', now: presenceAt })
+  assert.equal(board.isWellFormed(), true, `the board line is not well formed: ${JSON.stringify(board.slice(-60))}`)
+  const card = board.split('\n').find(item => item.startsWith('- '))
+  assert.equal(card, `- Dev2 [open] ${'a'.repeat(78)}…`, 'the pair on the bound is dropped whole, never cut in half')
+
+  // Control: a title that ends exactly on the bound keeps the whole pair …
+  const exact = formatRoomBoard({
+    card: { id: 'card', status: 'open', title: `${'b'.repeat(78)}\u{1F600}`, ownerBotId: 'dev2', origin: { roomId: 'room' } },
+  }, { roomId: 'room', nameOf: () => 'Dev2' })
+  assert.equal(exact.split('\n').find(item => item.startsWith('- ')), `- Dev2 [open] ${'b'.repeat(78)}\u{1F600}`)
+  // … and a family emoji well inside the bound is untouched.
+  const family = `Ship \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466} now`
+  const kept = formatRoomBoard({
+    card: { id: 'card', status: 'open', title: family, ownerBotId: 'dev2', origin: { roomId: 'room' } },
+  }, { roomId: 'room', nameOf: () => 'Dev2' })
+  assert.match(kept, new RegExp(`- Dev2 \\[open\\] Ship \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466} now$`, 'm'))
+  assert.equal(kept.isWellFormed(), true)
+})
+
+test('P-305: a clipped transcript line ends on a whole emoji', () => {
+  const budget = 1000
+  const clipped = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `${'a'.repeat(budget - 2)}\u{1F600}TAIL` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(clipped.text.isWellFormed(), true, `the clipped line is not well formed: ${JSON.stringify(clipped.text.slice(-8))}`)
+  assert.equal(clipped.text, `${'a'.repeat(budget - 2)}…`, 'the pair on the clip point is dropped whole')
+
+  // Control: a line that fits with a family emoji is untouched …
+  const fits = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `work \u{1F468}\u200D\u{1F469}\u200D\u{1F467}` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(fits.text, `work \u{1F468}\u200D\u{1F469}\u200D\u{1F467}`)
+  // … and a pair that ends just before the clip point survives whole.
+  const before = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `${'b'.repeat(budget - 3)}\u{1F600}TAIL` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(before.text.isWellFormed(), true)
+  assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`)
+})

core/test/room-membership.test.js

@@ -1,5 +1,5 @@
 import assert from 'node:assert/strict'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { test } from 'node:test'
@@ -10,7 +10,7 @@ import { install as installMemberAuth } from '../src/host/member-auth.js'
 import { install as installAuthz } from '../src/host/authz.js'
 import { install as installReach } from '../src/host/reach.js'
 import { install as installRooms, currentParticipants, isCrossOwnerRoom, linesVisibleTo, roomVisibleTo } from '../src/host/room-membership.js'
-import { compactStatus } from '../src/host/room-context.js'
+import { compactStatus, fitRoomLines, prepareRoomLine } from '../src/host/room-context.js'
 // Room membership (#50): humans and Chiefs share a room with distinct authorship.
 // Team mode stays disabled — these tests cover the membership surface only.
 
@@ -487,3 +487,28 @@ test('channel: a legacy single-owner create-room still works', async (t) => {
   const view = (await call('state', {}, alice.sessionToken)).value
   assert.ok(view.rooms.some(room => room.name === 'Solo'))
 })
+
+// P-305: `compactStatus` keeps the first hundred units of the flattened essay as the stub every writer of
+// `room.log` stores (#1251), and that cut was a bare `slice`. A Bot essay whose 100th unit sat inside an
+// emoji pair therefore stored a lone high surrogate, which the encoder turns into U+FFFD. The assertion is
+// on the value read back from `state.json` by a fresh runtime, not on the object the writer kept in memory.
+test('P-305: the stored status stub survives the store round trip well formed', async (t) => {
+  const { rt, statePath } = host(t, TEAM)
+  await rt.load()
+  // 99 units, then a whole emoji whose pair straddles the 100-unit stub bound, then a status essay.
+  const essay = `${'a'.repeat(99)}\u{1F600} CI on tip abcdef12 is green. tests-ok passed. ${'detail '.repeat(80)}TSAIL`
+  assert.equal(essay.isWellFormed(), true, 'the input the Bot sent is well formed')
+
+  const line = await rt.postRoomMessage('shared', { text: essay, authorId: 'chief-a', authorKind: 'bot' })
+  assert.equal(line.text, compactStatus(essay), 'the writer keeps the stub it stored')
+  await rt.save()
+
+  const reloaded = host(t, readFileSync(statePath, 'utf8'))
+  await reloaded.rt.load()
+  const stored = reloaded.rt.state.rooms.shared.log.at(-1).text
+  assert.equal(stored.isWellFormed(), true, `the reloaded stub is not well formed: ${JSON.stringify(stored)}`)
+  assert.equal(Buffer.from(stored, 'utf8').toString('utf8'), stored, 'the reloaded stub survives a UTF-8 round trip')
+  assert.match(stored, /^\[status on the board, \d+ chars\] a{99}…$/, 'the pair on the bound is dropped whole, never stored in half')
+  const turn = fitRoomLines(reloaded.rt.state.rooms.shared.log, { render: item => `${item.who}: ${item.text}`, prepare: prepareRoomLine })
+  assert.equal(turn.text.isWellFormed(), true, `the turn built from the reloaded log is not well formed: ${JSON.stringify(turn.text.slice(0, 160))}`)
+})

docs/subsystems/core-host.md

@@ -180,7 +180,7 @@ A member's group Session reads the group's fixed rules from a system-prompt sect
 
 ### Room context budget (#1203)
 
-A group turn does not hand the model the raw ring buffer. `src/host/room-context.js` `fitRoomLines` takes the lines the count caps already chose (`ROOM_FIRST_LOOK` 12, `ROOM_DELTA_MAX` 24, and `ROOM_EARLIER_MAX` 8 of those from before the round) and keeps a newer-first slice of at most `ROOM_CONTEXT_BUDGET` characters (4000). The budget is characters because the measured essays are mostly CJK, about one token per character. A long bot line that reads as CI, merge-queue, or review narration (`statusShaped`: at least 400 characters and two of the status hints) is collapsed to a one-line stub that keeps the opening and up to four `@` names. Since #1251 every writer of `room.log` stores that same stub rather than only the turn: `groups.js` `appendRoom` (a group round) and `room-membership.js` `append` (`postRoomMessage`, a seat's `line` op, a face-room event) both pass the stamped line through `prepareRoomLine`, so the stored line and the next turn cannot disagree. A person's line (`botId` absent), a `kind`-carrying event line **from this home** and a short handoff come back unchanged and are stored verbatim. A fenced peer line (#98) is compacted *inside* its fence and wrapped again with the same `from` name, because compacting the fenced string would strip the markers and store a guest's text as this home's own; the fence decides *before* the `kind`/`botId` exemptions, because both travel from the other home (`room-membership.js` copies the peer's `kind`), so a peer could otherwise mark its essay `notice` or type it as a person and store it whole. A long Bot line that is not status-shaped is still stored whole — the store gate reuses `#1204`'s rule, it does not widen it. When lines are left out, the turn says so and points at `read_group_chat` for an exact earlier line; `read_group_chat` returns the stored line, which for a Bot essay is the stub. Covered by `core/test/group-work.test.js`, `core/test/room-membership.test.js` and `core/test/remote-room.test.js` (the stores) and `core/test/room-context.test.js` (the rule).
+A group turn does not hand the model the raw ring buffer. `src/host/room-context.js` `fitRoomLines` takes the lines the count caps already chose (`ROOM_FIRST_LOOK` 12, `ROOM_DELTA_MAX` 24, and `ROOM_EARLIER_MAX` 8 of those from before the round) and keeps a newer-first slice of at most `ROOM_CONTEXT_BUDGET` characters (4000). The budget is characters because the measured essays are mostly CJK, about one token per character. A long bot line that reads as CI, merge-queue, or review narration (`statusShaped`: at least 400 characters and two of the status hints) is collapsed to a one-line stub that keeps the opening and up to four `@` names. Since #1251 every writer of `room.log` stores that same stub rather than only the turn: `groups.js` `appendRoom` (a group round) and `room-membership.js` `append` (`postRoomMessage`, a seat's `line` op, a face-room event) both pass the stamped line through `prepareRoomLine`, so the stored line and the next turn cannot disagree. A person's line (`botId` absent), a `kind`-carrying event line **from this home** and a short handoff come back unchanged and are stored verbatim. A fenced peer line (#98) is compacted *inside* its fence and wrapped again with the same `from` name, because compacting the fenced string would strip the markers and store a guest's text as this home's own; the fence decides *before* the `kind`/`botId` exemptions, because both travel from the other home (`room-membership.js` copies the peer's `kind`), so a peer could otherwise mark its essay `notice` or type it as a person and store it whole. A long Bot line that is not status-shaped is still stored whole — the store gate reuses `#1204`'s rule, it does not widen it. When lines are left out, the turn says so and points at `read_group_chat` for an exact earlier line; `read_group_chat` returns the stored line, which for a Bot essay is the stub. Every character bound in this module goes through `shared/text-boundary.js` `cut`, never a bare `slice`: the stub's hundred units (`compactStatus`), the transcript clip at `room - 1` in `fitRoomLines`, and the card, board and presence fields `oneLine` caps. A bound that lands between the halves of a surrogate pair drops the pair whole, so neither `room.log` nor a rendered turn carries a lone surrogate the encoder would turn into U+FFFD ([#1960](https://github.com/ThinkFlowLab/FlyBot/issues/1960)). Covered by `core/test/group-work.test.js`, `core/test/room-membership.test.js` and `core/test/remote-room.test.js` (the stores) and `core/test/room-context.test.js` (the rule).
 
 The open task board for that room is a compact snapshot in the same turn (`formatRoomBoard`): the newest overall task, then its subtasks and any room card that is not under a different overall task, up to eight cards. Done cards stay off. Titles and notes are one line, capped, with `<` and `>` removed. An unclaimed queue item reads `unassigned [open 3h Dev]` — its role and its wait on the card itself — and the snapshot ends with one line per role that has something waiting (`- Queue Dev: 2 waiting, oldest 3h`), so starvation is visible without reading every card (#1247). A role with nothing waiting is omitted. A face-to-face room has no snapshot and keeps `@Name` as how its lead gives a part. The standing section does not carry the snapshot or the transcript, so it still changes only when the group settings change. In a group that has a board, `ROOM_REPLY_BOARD` and the task-lead note say assigning, claiming, and progress are `track_tasks` writes (overall task, subtask, assignee, column, progress), `@Name` only wakes, and a status the board already shows is `[PASS]`. A task-drive wake in the group says to set that card's column and progress, and appends the same snapshot after its cue. Covered by `core/test/room-context.test.js`, `core/test/rounds.test.js`, and `core/test/task-drive.test.js`.
 

patches/ds-bot/files/src/host/room-context.js

@@ -5,6 +5,7 @@
 // fill that slice. Progress for the open job is a compact board snapshot in the
 // same turn, not those essays.
 import { columnOf, compareForWork, progressOf } from '../shared/task-board.js'
+import { cut } from '../shared/text-boundary.js'
 import { fenceUntrusted } from './delivery.js'
 
 // Characters, not a tokenizer. The essays this was measured on are mostly CJK,
@@ -42,7 +43,7 @@ const oneLine = (value, max) => {
   }
   flat = flat.replace(/\s+/g, ' ').trim()
   if (flat.length <= max) return flat
-  return `${flat.slice(0, Math.max(0, max - 1))}…`
+  return `${cut(flat, Math.max(0, max - 1)).text}…`
 }
 
 export function statusShaped(text) {
@@ -92,7 +93,7 @@ export function boardCoversCue(tasks, { roomId, botId, text } = {}) {
 export function compactStatus(text) {
   const flat = oneLine(text, 100000)
   const mentions = [...new Set([...flat.matchAll(/@[^\s@,，。]{1,40}/g)].map(match => match[0]))].slice(0, 4)
-  const head = flat.slice(0, 100).trim()
+  const head = cut(flat, 100).text.trim()
   const more = flat.length > 100 ? '…' : ''
   const called = mentions.length > 0 ? ` ${mentions.join(' ')}` : ''
   return `[status on the board, ${flat.length} chars] ${head}${more}${called}`
@@ -139,7 +140,7 @@ export function fitRoomLines(lines, { budget = ROOM_CONTEXT_BUDGET, render, prep
     if (used + gap + text.length > budget) {
       const room = budget - used - gap
       if (room < 48) break
-      text = `${text.slice(0, room - 1)}…`
+      text = `${cut(text, room - 1).text}…`
       kept.push({ line, text })
       break
     }

patches/ds-bot/files/test/room-context.test.js

@@ -213,3 +213,55 @@ test('presence: no members or no room renders nothing at all', () => {
   assert.equal(formatRoomPresence(presenceTasks(), { roomId: 'room', members: [], now: presenceAt }), '')
   assert.equal(formatRoomPresence(presenceTasks(), { members: presenceMembers, now: presenceAt }), '')
 })
+
+// P-305: a character bound must not land between the halves of a surrogate pair. `oneLine` ends a long
+// card, board or presence line with `flat.slice(0, max - 1) + '…'` and `fitRoomLines` clips a transcript
+// line with `text.slice(0, room - 1) + '…'`; both are bare `slice`, so a pair straddling the bound is
+// stored and sent whole-half — a lone high surrogate the encoder turns into U+FFFD.
+test('P-305: a card title cut at its bound keeps a whole emoji', () => {
+  // 78 units, then an emoji whose pair sits across the 80-unit bound.
+  const title = `${'a'.repeat(78)}\u{1F600}TAIL`
+  const board = formatRoomBoard({
+    card: { id: 'card', status: 'open', title, ownerBotId: 'dev2', origin: { roomId: 'room' }, createdAt: presenceAt, updatedAt: presenceAt },
+  }, { roomId: 'room', nameOf: () => 'Dev2', now: presenceAt })
+  assert.equal(board.isWellFormed(), true, `the board line is not well formed: ${JSON.stringify(board.slice(-60))}`)
+  const card = board.split('\n').find(item => item.startsWith('- '))
+  assert.equal(card, `- Dev2 [open] ${'a'.repeat(78)}…`, 'the pair on the bound is dropped whole, never cut in half')
+
+  // Control: a title that ends exactly on the bound keeps the whole pair …
+  const exact = formatRoomBoard({
+    card: { id: 'card', status: 'open', title: `${'b'.repeat(78)}\u{1F600}`, ownerBotId: 'dev2', origin: { roomId: 'room' } },
+  }, { roomId: 'room', nameOf: () => 'Dev2' })
+  assert.equal(exact.split('\n').find(item => item.startsWith('- ')), `- Dev2 [open] ${'b'.repeat(78)}\u{1F600}`)
+  // … and a family emoji well inside the bound is untouched.
+  const family = `Ship \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466} now`
+  const kept = formatRoomBoard({
+    card: { id: 'card', status: 'open', title: family, ownerBotId: 'dev2', origin: { roomId: 'room' } },
+  }, { roomId: 'room', nameOf: () => 'Dev2' })
+  assert.match(kept, new RegExp(`- Dev2 \\[open\\] Ship \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466} now$`, 'm'))
+  assert.equal(kept.isWellFormed(), true)
+})
+
+test('P-305: a clipped transcript line ends on a whole emoji', () => {
+  const budget = 1000
+  const clipped = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `${'a'.repeat(budget - 2)}\u{1F600}TAIL` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(clipped.text.isWellFormed(), true, `the clipped line is not well formed: ${JSON.stringify(clipped.text.slice(-8))}`)
+  assert.equal(clipped.text, `${'a'.repeat(budget - 2)}…`, 'the pair on the clip point is dropped whole')
+
+  // Control: a line that fits with a family emoji is untouched …
+  const fits = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `work \u{1F468}\u200D\u{1F469}\u200D\u{1F467}` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(fits.text, `work \u{1F468}\u200D\u{1F469}\u200D\u{1F467}`)
+  // … and a pair that ends just before the clip point survives whole.
+  const before = fitRoomLines(
+    [{ seq: 1, who: 'User', text: `${'b'.repeat(budget - 3)}\u{1F600}TAIL` }],
+    { budget, render: line => line.text },
+  )
+  assert.equal(before.text.isWellFormed(), true)
+  assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`)
+})