host: the memory-review and room-context one-line bounds judge in code points

changelog.d/2339-code-point-bounds.fixed.md

@@ -0,0 +1 @@
+The memory-review user line and the room/board one-line fields still judged their limit in UTF-16 units while they cut in code points, so a line of characters outside the BMP was cut while it was already under its own limit: a 1,497-character user line reached the review as 1,497 emoji under a 3,000-character bound and lost about 1,400 of them, and a 40-character holder name or an 80-character card title came back at 19 and 39. Both bounds now judge with `fits()` in `core/src/shared/text-boundary.js`, with the maxima and the cuts unchanged ([#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339)).

core/src/host/memory.js

@@ -28,7 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm'
 import { currentParticipants } from './room-membership.js'
 import { defineTool } from './tool-schema.js'
 import { clip, codePoints, splitRef, unquote } from './text.js'
-import { codePointStart, cut } from '../shared/text-boundary.js'
+import { codePointStart, cut, fits } from '../shared/text-boundary.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
 import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js'
@@ -1269,7 +1269,7 @@ export function install(rt) {
   const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve())
   // A long message often ends with what the user wants of it, so both ends stay.
   const ends = (text, max) => {
-    if (text.length <= max) return text
+    if (fits(text, max)) return text
     const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3)))
     // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back.
     const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length))

core/src/host/room-context.js

@@ -5,7 +5,7 @@
 // fill that slice. Progress for the open job is a compact board snapshot in the
 // same turn, not those essays.
 import { columnOf, compareForWork, progressOf } from '../shared/task-board.js'
-import { cut } from '../shared/text-boundary.js'
+import { cut, fits } from '../shared/text-boundary.js'
 import { fenceUntrusted } from './delivery.js'
 
 // Characters, not a tokenizer. The essays this was measured on are mostly CJK,
@@ -42,7 +42,7 @@ const oneLine = (value, max) => {
     flat += code < 0x20 || code === 0x7f || char === '<' || char === '>' ? ' ' : char
   }
   flat = flat.replace(/\s+/g, ' ').trim()
-  if (flat.length <= max) return flat
+  if (fits(flat, max)) return flat
   return `${cut(flat, Math.max(0, max - 1)).text}…`
 }
 

core/test/memory.test.js

@@ -1957,3 +1957,32 @@ test('#2159: a hard-linked journal file takes no record, and the memory write st
   const recorded = readFileSync(journalPath, 'utf8').trim().split('\n').map(line => JSON.parse(line))
   assert.deepEqual(recorded.map(record => record.op), ['add'], 'the record lands once the file carries one name')
 })
+
+// P-514: `ends` judged with `text.length` while it stepped by code point, so a user message under
+// REVIEW_USER_LINE_MAX (3000 characters) was cut in the middle. 2900 characters are 5800 UTF-16
+// units, so the unfixed bound keeps 1500 of them (a 2000-unit head and a 1000-unit tail) and drops
+// 1400. ASCII is the control: it is under the limit in both readings and must read the same.
+test('a user line under 3000 characters is kept whole for the review (P-514)', async () => {
+  const render = async (text) => {
+    const history = {
+      b2: [
+        { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } },
+        { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } },
+      ],
+    }
+    const replies = [JSON.stringify({ changes: [] })]
+    const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } })
+    await call('state', {})
+    ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+    await reviewed(call)
+    await stop()
+    const prompt = requests[0].messages[0].content[0].text
+    const line = prompt.split('\n').find(one => one.startsWith(`User: ${[...text][0]}`))
+    assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`)
+    return line
+  }
+  const wide = '😀'.repeat(2900)
+  assert.equal(await render(wide), `User: ${wide}`, '2900 characters are under the 3000-character limit, though they are 5800 UTF-16 units')
+  const ascii = 'a'.repeat(2900)
+  assert.equal(await render(ascii), `User: ${ascii}`, 'ASCII under the limit reads the same before and after')
+})

core/test/room-context.test.js

@@ -265,3 +265,20 @@ test('P-305: a clipped transcript line ends on a whole emoji', () => {
   assert.equal(before.text.isWellFormed(), true)
   assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`)
 })
+
+// P-514: `oneLine` judged with `flat.length` while its cut steps by code point, so a title under
+// its own limit was cut. 80 characters are under the 80-character title limit although they are
+// 160 UTF-16 units; the whole title must reach the board.
+test('a board line of characters outside the BMP is under its own limit and is not cut (P-514)', () => {
+  const title = '😀'.repeat(80)
+  const name = '😀'.repeat(40)
+  const tasks = { overall: { id: 'overall', kind: 'group', status: 'open', title, ownerBotId: 'chief', origin: { roomId: 'room' }, updatedAt: 2 } }
+  const board = formatRoomBoard(tasks, { roomId: 'room', nameOf: () => name })
+  const line = board.split('\n').find(one => one.includes('😀'))
+  assert.ok(line !== undefined, `the card is on the board:\n${board}`)
+  // The holder name goes through oneLine at 40, the title through oneLine at 80; each is under its
+  // limit in characters (40 and 80) though over it in UTF-16 units (80 and 160).
+  assert.ok(line.startsWith(`- ${name} [`), `all 40 characters of the holder name are kept: ${line.slice(0, 60)}`)
+  assert.ok(line.includes(title), `all 80 characters of the title are kept: ${line.slice(-60)}`)
+  assert.equal(line.includes('…'), false, line)
+})

core/test/text-bounds.test.js

@@ -70,3 +70,23 @@ test('#2321: the memory review clip is the shared one, not a third copy (core/sr
   assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`)
   assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js')
 })
+
+// P-514: two more bounds, declared inside a factory and a module rather than exported, are named
+// here the same way the table names its rows: a revert to `text.length <= max` fails at the
+// consumer's own `file:line`, not somewhere in a rendered prompt.
+const LIMITS_514 = [
+  { consumer: 'ends', path: 'host/memory.js', declaration: 'const ends = (text, max) =>' },
+  { consumer: 'oneLine', path: 'host/room-context.js', declaration: 'const oneLine = (value, max) =>' },
+]
+
+for (const row of LIMITS_514) {
+  test(`P-514: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => {
+    const where = locate(row.path, row.declaration)
+    const source = read(row.path)
+    const from = source.indexOf(row.declaration)
+    assert.notEqual(from, -1, `${where} declaration moved: ${row.declaration}`)
+    const block = source.slice(from)
+    const body = block.slice(0, block.indexOf('\n\n') === -1 ? block.length : block.indexOf('\n\n'))
+    assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from the shared boundary`)
+  })
+}

patches/ds-bot/0001-core-fixes.patch

@@ -5239,7 +5239,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js
 +import { currentParticipants } from './room-membership.js'
 +import { defineTool } from './tool-schema.js'
 +import { clip, codePoints, splitRef, unquote } from './text.js'
-+import { codePointStart, cut } from '../shared/text-boundary.js'
++import { codePointStart, cut, fits } from '../shared/text-boundary.js'
  import { output } from './tools.js'
 +import { retiredBot } from './store.js'
 +import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js'
@@ -5765,7 +5765,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js
    // A long message often ends with what the user wants of it, so both ends stay.
 -  const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text)
 +  const ends = (text, max) => {
-+    if (text.length <= max) return text
++    if (fits(text, max)) return text
 +    const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3)))
 +    // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back.
 +    const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length))
@@ -11852,7 +11852,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js
    const view = (await call('memory-view', { botId: 'b2' })).value
    assert.equal(view.error, null)
    assert.equal(view.summary?.text, 'You like tea.')
-@@ -1076,3 +1076,801 @@ test('a memory call names the current part of the Bot chat', async (t) => {
+@@ -1076,3 +1076,830 @@ test('a memory call names the current part of the Bot chat', async (t) => {
    assert.equal(answer.ok, true)
    assert.equal(requests[0].sessionId, 'b2-part2')
  })
@@ -12654,6 +12654,35 @@ diff --git a/test/memory.test.js b/test/memory.test.js
 +  const recorded = readFileSync(journalPath, 'utf8').trim().split('\n').map(line => JSON.parse(line))
 +  assert.deepEqual(recorded.map(record => record.op), ['add'], 'the record lands once the file carries one name')
 +})
++
++// P-514: `ends` judged with `text.length` while it stepped by code point, so a user message under
++// REVIEW_USER_LINE_MAX (3000 characters) was cut in the middle. 2900 characters are 5800 UTF-16
++// units, so the unfixed bound keeps 1500 of them (a 2000-unit head and a 1000-unit tail) and drops
++// 1400. ASCII is the control: it is under the limit in both readings and must read the same.
++test('a user line under 3000 characters is kept whole for the review (P-514)', async () => {
++  const render = async (text) => {
++    const history = {
++      b2: [
++        { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } },
++        { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } },
++      ],
++    }
++    const replies = [JSON.stringify({ changes: [] })]
++    const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } })
++    await call('state', {})
++    ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
++    await reviewed(call)
++    await stop()
++    const prompt = requests[0].messages[0].content[0].text
++    const line = prompt.split('\n').find(one => one.startsWith(`User: ${[...text][0]}`))
++    assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`)
++    return line
++  }
++  const wide = '😀'.repeat(2900)
++  assert.equal(await render(wide), `User: ${wide}`, '2900 characters are under the 3000-character limit, though they are 5800 UTF-16 units')
++  const ascii = 'a'.repeat(2900)
++  assert.equal(await render(ascii), `User: ${ascii}`, 'ASCII under the limit reads the same before and after')
++})
 diff --git a/test/model-menu.test.js b/test/model-menu.test.js
 --- a/test/model-menu.test.js
 +++ b/test/model-menu.test.js

patches/ds-bot/files/src/host/room-context.js

@@ -5,7 +5,7 @@
 // fill that slice. Progress for the open job is a compact board snapshot in the
 // same turn, not those essays.
 import { columnOf, compareForWork, progressOf } from '../shared/task-board.js'
-import { cut } from '../shared/text-boundary.js'
+import { cut, fits } from '../shared/text-boundary.js'
 import { fenceUntrusted } from './delivery.js'
 
 // Characters, not a tokenizer. The essays this was measured on are mostly CJK,
@@ -42,7 +42,7 @@ const oneLine = (value, max) => {
     flat += code < 0x20 || code === 0x7f || char === '<' || char === '>' ? ' ' : char
   }
   flat = flat.replace(/\s+/g, ' ').trim()
-  if (flat.length <= max) return flat
+  if (fits(flat, max)) return flat
   return `${cut(flat, Math.max(0, max - 1)).text}…`
 }
 

patches/ds-bot/files/test/room-context.test.js

@@ -265,3 +265,20 @@ test('P-305: a clipped transcript line ends on a whole emoji', () => {
   assert.equal(before.text.isWellFormed(), true)
   assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`)
 })
+
+// P-514: `oneLine` judged with `flat.length` while its cut steps by code point, so a title under
+// its own limit was cut. 80 characters are under the 80-character title limit although they are
+// 160 UTF-16 units; the whole title must reach the board.
+test('a board line of characters outside the BMP is under its own limit and is not cut (P-514)', () => {
+  const title = '😀'.repeat(80)
+  const name = '😀'.repeat(40)
+  const tasks = { overall: { id: 'overall', kind: 'group', status: 'open', title, ownerBotId: 'chief', origin: { roomId: 'room' }, updatedAt: 2 } }
+  const board = formatRoomBoard(tasks, { roomId: 'room', nameOf: () => name })
+  const line = board.split('\n').find(one => one.includes('😀'))
+  assert.ok(line !== undefined, `the card is on the board:\n${board}`)
+  // The holder name goes through oneLine at 40, the title through oneLine at 80; each is under its
+  // limit in characters (40 and 80) though over it in UTF-16 units (80 and 160).
+  assert.ok(line.startsWith(`- ${name} [`), `all 40 characters of the holder name are kept: ${line.slice(0, 60)}`)
+  assert.ok(line.includes(title), `all 80 characters of the title are kept: ${line.slice(-60)}`)
+  assert.equal(line.includes('…'), false, line)
+})

patches/ds-bot/files/test/text-bounds.test.js

@@ -70,3 +70,23 @@ test('#2321: the memory review clip is the shared one, not a third copy (core/sr
   assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`)
   assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js')
 })
+
+// P-514: two more bounds, declared inside a factory and a module rather than exported, are named
+// here the same way the table names its rows: a revert to `text.length <= max` fails at the
+// consumer's own `file:line`, not somewhere in a rendered prompt.
+const LIMITS_514 = [
+  { consumer: 'ends', path: 'host/memory.js', declaration: 'const ends = (text, max) =>' },
+  { consumer: 'oneLine', path: 'host/room-context.js', declaration: 'const oneLine = (value, max) =>' },
+]
+
+for (const row of LIMITS_514) {
+  test(`P-514: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => {
+    const where = locate(row.path, row.declaration)
+    const source = read(row.path)
+    const from = source.indexOf(row.declaration)
+    assert.notEqual(from, -1, `${where} declaration moved: ${row.declaration}`)
+    const block = source.slice(from)
+    const body = block.slice(0, block.indexOf('\n\n') === -1 ? block.length : block.indexOf('\n\n'))
+    assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from the shared boundary`)
+  })
+}