host: judge a text bound in code points, not in UTF-16 units (Fixes #2321)

changelog.d/2321-fits-text-bound.fixed.md

@@ -0,0 +1 @@
+The host judged a text bound in UTF-16 units while it cut in code points, so a character outside the BMP spent two of a limit's budget: a 300-character message was cut by a 400-character limit and came back short through `read_own_chat` and search rows. Bounds now judge with `fits()` in `core/src/shared/text-boundary.js`, `clip` and `excerpt` use it, and the third copy of `clip` in the memory review is the shared one ([#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321)).

core/client.js

@@ -1740,6 +1740,12 @@ var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
 var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
 var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at;
 var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end;
+var fits = (text2, max) => {
+  const value = String(text2);
+  let n = 0;
+  for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 65535 ? 2 : 1;
+  return n <= max;
+};
 var cut = (text2, max) => {
   const flat = String(text2).toWellFormed();
   return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max };

core/src/host/memory.js

@@ -27,7 +27,7 @@ import { dirname, join, relative, sep } from 'node:path'
 import { BlockAssembler } from '@deepseek-ai/dsh-llm'
 import { currentParticipants } from './room-membership.js'
 import { defineTool } from './tool-schema.js'
-import { codePoints, splitRef, unquote } from './text.js'
+import { clip, codePoints, splitRef, unquote } from './text.js'
 import { codePointStart, cut } from '../shared/text-boundary.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
@@ -1267,7 +1267,6 @@ export function install(rt) {
     return cursors
   }
   const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve())
-  const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text)
   // A long message often ends with what the user wants of it, so both ends stay.
   const ends = (text, max) => {
     if (text.length <= max) return text

core/src/host/text.js

@@ -1,7 +1,7 @@
 // Pure text helpers shared by the host modules: message text, names, model refs,
 // cutting and estimating text, and the clock format of chat anchors.
 import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js'
-import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js'
+import { codePointEnd, codePointStart, cut, fits } from '../shared/text-boundary.js'
 
 export { cut }
 
@@ -44,7 +44,7 @@ export const codePoints = (text) => {
 
 // The part of a long message around the first match, so a search hit stays visible.
 export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => {
-  if (text.length <= max) return text
+  if (fits(text, max)) return text
   const found = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120)
   const at = codePointStart(text, found)
   return `${at > 0 ? '…' : ''}${text.slice(at, codePointEnd(text, at + max))}…`
@@ -58,7 +58,7 @@ export const estimateTokens = (text) => {
 }
 // #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never
 // "… (cut; undefined)"; a caller that gives one keeps "… (cut; <reason>)" byte for byte.
-export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text)
+export const clip = (text, max, rest) => (fits(text, max) ? text : `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})`)
 
 export const createClock = (timeZone) => {
   const clockFormat = new Intl.DateTimeFormat('en-CA', {

core/src/shared/text-boundary.js

@@ -12,6 +12,16 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC
 // The end of a bounded prefix, moved back onto a code point boundary.
 export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end)
 
+// Whether a text is inside a bound (#2321). Judged in code points, not in UTF-16 units: a character
+// outside the BMP costs one of the budget, so a 300-character text is under a 400-character limit
+// even though it is 600 units long, and a bound never cuts text that is already under its own limit.
+export const fits = (text, max) => {
+  const value = String(text)
+  let n = 0
+  for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1
+  return n <= max
+}
+
 // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped.
 // The input is flattened first (#1963): a string that already carried an unpaired surrogate went
 // straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit

core/test/memory.test.js

@@ -558,13 +558,13 @@ test('a long Bot line cut for the review context keeps whole characters (#1918)'
   assert.equal(prompt.includes('\ufffd'), false)
   const shown = prompt.split('\n').find(one => one.startsWith('Writer: x'))
   assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`)
-  assert.ok(shown.endsWith('…(cut)'), shown.slice(-20))
-  assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length))
+  assert.ok(shown.endsWith('… (cut)'), shown.slice(-20))
+  assert.ok(shown.length <= 'Writer: '.length + 800 + ' … (cut)'.length, String(shown.length))
   // #1978: pin what survives, not only how much may survive. At 800 the cut steps back off the
   // emoji's pair, so the whole kept line is `Writer: ` plus 791 x's; a smaller limit keeps a
   // shorter head — the direction that drops context silently — and must go red here.
   assert.ok(shown.startsWith(`Writer: ${'x'.repeat(791)}`), shown.slice(0, 40))
-  assert.ok(shown.replace(' …(cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length))
+  assert.ok(shown.replace('… (cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length))
 })
 
 test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => {

core/test/text-bounds.test.js

@@ -0,0 +1,72 @@
+// #2321: a host bound judges in code points, not in UTF-16 units. A character outside the BMP
+// costs one of the budget, so a text that is under its own limit is never cut. Every row names the
+// consumer that judges: a later change back to `text.length > max` fails here at that `file:line`.
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { clip, codePoints, excerpt } from '../src/host/text.js'
+import * as boundary from '../src/shared/text-boundary.js'
+
+const read = (path) => readFileSync(fileURLToPath(new URL(`../src/${path}`, import.meta.url)), 'utf8')
+const lineOf = (source, needle) => {
+  const at = source.indexOf(needle)
+  return at === -1 ? undefined : source.slice(0, at).split('\n').length
+}
+const locate = (path, needle) => {
+  const line = lineOf(read(path), needle)
+  return line === undefined ? `core/src/${path}` : `core/src/${path}:${line}`
+}
+
+// The table: a limit, the text it judges, and the consumer that owns the judgment. Each text is
+// under `limit` characters and over it in UTF-16 units, which is the shape #2321 is about.
+const LIMITS = [
+  { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(300) },
+  { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(201) },
+  { consumer: 'excerpt', path: 'host/text.js', declaration: 'export const excerpt =', limit: 400, text: '😀'.repeat(300) },
+]
+
+const run = (row) => (row.consumer === 'clip' ? clip(row.text, row.limit, 'r') : excerpt(row.text, '', row.limit))
+
+for (const row of LIMITS) {
+  test(`#2321: ${row.consumer} keeps a text under its own limit (${locate(row.path, row.declaration)})`, () => {
+    const kept = `${codePoints(row.text)} characters / ${row.text.length} UTF-16 units, limit ${row.limit}`
+    assert.equal(run(row), row.text, `a text of ${kept} is under its limit, so ${row.consumer} must not cut it`)
+  })
+}
+
+// The judgment itself: each consumer must ask `fits`, so reverting one to `text.length > max` names
+// only that consumer's own file:line and the table cannot rot silently.
+for (const row of LIMITS) {
+  test(`#2321: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => {
+    const where = locate(row.path, row.declaration)
+    const source = read(row.path)
+    const from = source.indexOf(row.declaration)
+    const block = source.slice(from)
+    const body = row.consumer === 'clip' ? block.slice(0, block.indexOf('\n')) : block.slice(0, block.indexOf('\n}') + 2)
+    assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from shared/text-boundary.js`)
+  })
+}
+
+test('#2321: an over-limit clip still cuts and says so', () => {
+  const out = clip('😀'.repeat(201), 200, 'r')
+  assert.equal(out.endsWith('… (cut; r)'), true, out.slice(-24))
+  assert.equal(codePoints(out.slice(0, -'… (cut; r)'.length)) <= 200, true, `it keeps at most 200 characters: ${codePoints(out)}`)
+})
+
+test('#2321: fits counts code points, not UTF-16 units (core/src/shared/text-boundary.js)', () => {
+  assert.equal(typeof boundary.fits, 'function', 'shared/text-boundary.js must export fits(text, max)')
+  assert.equal(boundary.fits('', 0), true)
+  assert.equal(boundary.fits('abc', 3), true)
+  assert.equal(boundary.fits('abcd', 3), false)
+  assert.equal(boundary.fits('😀'.repeat(201), 400), true, '201 characters are under a 400-character limit, though 402 UTF-16 units are not')
+  assert.equal(boundary.fits('😀'.repeat(401), 400), false)
+  assert.equal(boundary.fits('a😀b', 3), true)
+})
+
+test('#2321: the memory review clip is the shared one, not a third copy (core/src/host/memory.js)', () => {
+  const source = read('host/memory.js')
+  const duplicate = lineOf(source, 'const clip = (text, max) =>')
+  assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`)
+  assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js')
+})

patches/ds-bot/0001-core-fixes.patch

@@ -5219,7 +5219,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js
 -import { splitRef, unquote } from './text.js'
 +import { currentParticipants } from './room-membership.js'
 +import { defineTool } from './tool-schema.js'
-+import { codePoints, splitRef, unquote } from './text.js'
++import { clip, codePoints, splitRef, unquote } from './text.js'
 +import { codePointStart, cut } from '../shared/text-boundary.js'
  import { output } from './tools.js'
 +import { retiredBot } from './store.js'
@@ -5720,7 +5720,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js
      if (text === '') throw new Error('Type a question or a change first')
      const numbered = [
        ...allEntries(loadScope(ownScope(bot.id))).map(entry => ({ ...entry, scope: 'own' })),
-@@ -1074,18 +1074,28 @@ export function install(rt) {
+@@ -1074,18 +1074,27 @@ export function install(rt) {
    let cursors
    const cursorsNow = () => {
      if (cursors === undefined) {
@@ -5743,7 +5743,6 @@ diff --git a/src/host/memory.js b/src/host/memory.js
    }
 -  const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} …(cut)` : text)
 +  const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve())
-+  const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text)
    // A long message often ends with what the user wants of it, so both ends stay.
 -  const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text)
 +  const ends = (text, max) => {
@@ -7199,7 +7198,7 @@ diff --git a/src/host/text.js b/src/host/text.js
  // Pure text helpers shared by the host modules: message text, names, model refs,
  // cutting and estimating text, and the clock format of chat anchors.
  import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js'
-+import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js'
++import { codePointEnd, codePointStart, cut, fits } from '../shared/text-boundary.js'
 +
 +export { cut }
 
@@ -7227,9 +7226,10 @@ diff --git a/src/host/text.js b/src/host/text.js
 +
  // The part of a long message around the first match, so a search hit stays visible.
  export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => {
-   if (text.length <= max) return text
+-  if (text.length <= max) return text
 -  const at = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120)
 -  return `${at > 0 ? '…' : ''}${text.slice(at, at + max)}…`
++  if (fits(text, max)) return text
 +  const found = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120)
 +  const at = codePointStart(text, found)
 +  return `${at > 0 ? '…' : ''}${text.slice(at, codePointEnd(text, at + max))}…`
@@ -7243,7 +7243,7 @@ diff --git a/src/host/text.js b/src/host/text.js
 -export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, max)}… (cut; ${rest})` : text)
 +// #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never
 +// "… (cut; undefined)"; a caller that gives one keeps "… (cut; <reason>)" byte for byte.
-+export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text)
++export const clip = (text, max, rest) => (fits(text, max) ? text : `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})`)
 
  export const createClock = (timeZone) => {
    const clockFormat = new Intl.DateTimeFormat('en-CA', {
@@ -11753,13 +11753,13 @@ diff --git a/test/memory.test.js b/test/memory.test.js
 +  assert.equal(prompt.includes('\ufffd'), false)
 +  const shown = prompt.split('\n').find(one => one.startsWith('Writer: x'))
 +  assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`)
-+  assert.ok(shown.endsWith('…(cut)'), shown.slice(-20))
-+  assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length))
++  assert.ok(shown.endsWith('… (cut)'), shown.slice(-20))
++  assert.ok(shown.length <= 'Writer: '.length + 800 + ' … (cut)'.length, String(shown.length))
 +  // #1978: pin what survives, not only how much may survive. At 800 the cut steps back off the
 +  // emoji's pair, so the whole kept line is `Writer: ` plus 791 x's; a smaller limit keeps a
 +  // shorter head — the direction that drops context silently — and must go red here.
 +  assert.ok(shown.startsWith(`Writer: ${'x'.repeat(791)}`), shown.slice(0, 40))
-+  assert.ok(shown.replace(' …(cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length))
++  assert.ok(shown.replace('… (cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length))
 +})
 +
 +test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => {