From 40b871f8f874c70d5264d813ed78b8b8f322469b Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:54:36 +0800 Subject: [PATCH] host: the memory-review and room-context one-line bounds judge in code points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2339 Owner: Dev-4 Written-by: Dev-4 Claim-Id: 3a11368f0c774f6391e17a68f5af2c08 Branch: dev/4-p514-code-point-bounds Base: d36903dd94bad0bd00de75d07bf7134799b8d0c4 Diff: 11 files, +143/−10 Tier: T2 why: path core/src/host/memory.js (and core/src/host/room-context.js); tests, docs/**, changelog.d/** and patches/ds-bot/** are exempt or T1 ## What was wrong Two bounds in core/src/host/ still asked `text.length <= max` before they cut in code points, so a character outside the BMP spent two of a limit's budget and text that was already under its own limit was cut anyway: - memory.js `ends()` (REVIEW_USER_LINE_MAX = 3,000): a user line of 1,497 emoji (2,994 UTF-16 units) was replaced by 1,497 emoji plus …(middle cut)…, losing about 1,400 characters — the user's own line in the memory review. - room-context.js `oneLine()` (40/80/24/60): a 40-character holder name came back as 19 emoji + …, an 80-character card title as 39 + …, so a board row named a different Bot than the one on the card. Both judged with `text.length` while they cut with a code-point boundary, which is the mismatch #2321 fixed for `clip` and `excerpt`. ## What changed - core/src/host/memory.js: `ends()` judges with `fits(text, max)` from ../shared/text-boundary.js; the head/tail cut and the 3,000-unit budget are unchanged. - core/src/host/room-context.js: `oneLine()` judges with `fits(flat, max)`; the flattening, the maxima and the `cut(flat, max - 1)` + … are unchanged. - Tests: text-bounds.test.js gains the static pair (#2339), room-context.test.js asserts the 40/80 board fields whole, memory.test.js asserts the review user line whole plus the ASCII control (ends('a'.repeat(2900), 3000) unchanged). - docs/subsystems/core-host.md records the two new consumers of `fits`; changelog.d/2339-code-point-bounds.fixed.md is the entry. ASCII text judges exactly as `length <= max` did: `fits` answers `length <= max` for text with no character outside the BMP. --- changelog.d/2339-code-point-bounds.fixed.md | 1 + core/src/host/memory.js | 4 +-- core/src/host/room-context.js | 4 +-- core/test/memory.test.js | 29 +++++++++++++++ core/test/room-context.test.js | 17 +++++++++ core/test/text-bounds.test.js | 20 +++++++++++ docs/subsystems/core-host.md | 2 +- patches/ds-bot/0001-core-fixes.patch | 35 +++++++++++++++++-- patches/ds-bot/files/src/host/room-context.js | 4 +-- .../ds-bot/files/test/room-context.test.js | 17 +++++++++ patches/ds-bot/files/test/text-bounds.test.js | 20 +++++++++++ 11 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 changelog.d/2339-code-point-bounds.fixed.md diff --git a/changelog.d/2339-code-point-bounds.fixed.md b/changelog.d/2339-code-point-bounds.fixed.md new file mode 100644 index 000000000..1529ef16e --- /dev/null +++ b/changelog.d/2339-code-point-bounds.fixed.md @@ -0,0 +1 @@ +The memory-review user line and the room/board one-line fields still judged their limit in UTF-16 units while they cut in code points, so a line of characters outside the BMP was cut while it was already under its own limit: a 1,497-character user line reached the review as 1,497 emoji under a 3,000-character bound and lost about 1,400 of them, and a 40-character holder name or an 80-character card title came back at 19 and 39. Both bounds now judge with `fits()` in `core/src/shared/text-boundary.js`, with the maxima and the cuts unchanged ([#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339)). diff --git a/core/src/host/memory.js b/core/src/host/memory.js index 68d50dbf3..76c13c976 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -28,7 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm' import { currentParticipants } from './room-membership.js' import { defineTool } from './tool-schema.js' import { clip, codePoints, splitRef, unquote } from './text.js' -import { codePointStart, cut } from '../shared/text-boundary.js' +import { codePointStart, cut, fits } from '../shared/text-boundary.js' import { output } from './tools.js' import { retiredBot } from './store.js' import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -1269,7 +1269,7 @@ export function install(rt) { const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve()) // A long message often ends with what the user wants of it, so both ends stay. const ends = (text, max) => { - if (text.length <= max) return text + if (fits(text, max)) return text const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) diff --git a/core/src/host/room-context.js b/core/src/host/room-context.js index 9b639b7e3..f37fc5f6f 100644 --- a/core/src/host/room-context.js +++ b/core/src/host/room-context.js @@ -5,7 +5,7 @@ // fill that slice. Progress for the open job is a compact board snapshot in the // same turn, not those essays. import { columnOf, compareForWork, progressOf } from '../shared/task-board.js' -import { cut } from '../shared/text-boundary.js' +import { cut, fits } from '../shared/text-boundary.js' import { fenceUntrusted } from './delivery.js' // Characters, not a tokenizer. The essays this was measured on are mostly CJK, @@ -42,7 +42,7 @@ const oneLine = (value, max) => { flat += code < 0x20 || code === 0x7f || char === '<' || char === '>' ? ' ' : char } flat = flat.replace(/\s+/g, ' ').trim() - if (flat.length <= max) return flat + if (fits(flat, max)) return flat return `${cut(flat, Math.max(0, max - 1)).text}…` } diff --git a/core/test/memory.test.js b/core/test/memory.test.js index 53fc9986f..03a8ba8b2 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -1957,3 +1957,32 @@ test('#2159: a hard-linked journal file takes no record, and the memory write st const recorded = readFileSync(journalPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)) assert.deepEqual(recorded.map(record => record.op), ['add'], 'the record lands once the file carries one name') }) + +// P-514: `ends` judged with `text.length` while it stepped by code point, so a user message under +// REVIEW_USER_LINE_MAX (3000 characters) was cut in the middle. 2900 characters are 5800 UTF-16 +// units, so the unfixed bound keeps 1500 of them (a 2000-unit head and a 1000-unit tail) and drops +// 1400. ASCII is the control: it is under the limit in both readings and must read the same. +test('a user line under 3000 characters is kept whole for the review (P-514)', async () => { + const render = async (text) => { + const history = { + b2: [ + { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, + { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } }, + ], + } + const replies = [JSON.stringify({ changes: [] })] + const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) + await call('state', {}) + ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) + await reviewed(call) + await stop() + const prompt = requests[0].messages[0].content[0].text + const line = prompt.split('\n').find(one => one.startsWith(`User: ${[...text][0]}`)) + assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) + return line + } + const wide = '😀'.repeat(2900) + assert.equal(await render(wide), `User: ${wide}`, '2900 characters are under the 3000-character limit, though they are 5800 UTF-16 units') + const ascii = 'a'.repeat(2900) + assert.equal(await render(ascii), `User: ${ascii}`, 'ASCII under the limit reads the same before and after') +}) diff --git a/core/test/room-context.test.js b/core/test/room-context.test.js index ac5856729..c850c851c 100644 --- a/core/test/room-context.test.js +++ b/core/test/room-context.test.js @@ -265,3 +265,20 @@ test('P-305: a clipped transcript line ends on a whole emoji', () => { assert.equal(before.text.isWellFormed(), true) assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`) }) + +// P-514: `oneLine` judged with `flat.length` while its cut steps by code point, so a title under +// its own limit was cut. 80 characters are under the 80-character title limit although they are +// 160 UTF-16 units; the whole title must reach the board. +test('a board line of characters outside the BMP is under its own limit and is not cut (P-514)', () => { + const title = '😀'.repeat(80) + const name = '😀'.repeat(40) + const tasks = { overall: { id: 'overall', kind: 'group', status: 'open', title, ownerBotId: 'chief', origin: { roomId: 'room' }, updatedAt: 2 } } + const board = formatRoomBoard(tasks, { roomId: 'room', nameOf: () => name }) + const line = board.split('\n').find(one => one.includes('😀')) + assert.ok(line !== undefined, `the card is on the board:\n${board}`) + // The holder name goes through oneLine at 40, the title through oneLine at 80; each is under its + // limit in characters (40 and 80) though over it in UTF-16 units (80 and 160). + assert.ok(line.startsWith(`- ${name} [`), `all 40 characters of the holder name are kept: ${line.slice(0, 60)}`) + assert.ok(line.includes(title), `all 80 characters of the title are kept: ${line.slice(-60)}`) + assert.equal(line.includes('…'), false, line) +}) diff --git a/core/test/text-bounds.test.js b/core/test/text-bounds.test.js index ba8a4c897..971a9a466 100644 --- a/core/test/text-bounds.test.js +++ b/core/test/text-bounds.test.js @@ -70,3 +70,23 @@ test('#2321: the memory review clip is the shared one, not a third copy (core/sr assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`) assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js') }) + +// P-514: two more bounds, declared inside a factory and a module rather than exported, are named +// here the same way the table names its rows: a revert to `text.length <= max` fails at the +// consumer's own `file:line`, not somewhere in a rendered prompt. +const LIMITS_514 = [ + { consumer: 'ends', path: 'host/memory.js', declaration: 'const ends = (text, max) =>' }, + { consumer: 'oneLine', path: 'host/room-context.js', declaration: 'const oneLine = (value, max) =>' }, +] + +for (const row of LIMITS_514) { + test(`P-514: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => { + const where = locate(row.path, row.declaration) + const source = read(row.path) + const from = source.indexOf(row.declaration) + assert.notEqual(from, -1, `${where} declaration moved: ${row.declaration}`) + const block = source.slice(from) + const body = block.slice(0, block.indexOf('\n\n') === -1 ? block.length : block.indexOf('\n\n')) + assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from the shared boundary`) + }) +} diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index ee2297f9e..e8e1f1574 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `…` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `…` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds — `String(text).toWellFormed()` first, `codePointEnd` on the flat string — so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too — a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `…` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file — the name its chat is created with (40) and the name a refresh writes (40) — go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too — the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` — the text an MCP server's failure leaves on the owner's connector card — keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `…` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well — the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole — so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate — a character outside the BMP spends one of the budget and not two — and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. The memory review's own third `clip` is the exported one now, so its cut reads `… (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `…` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `…` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds — `String(text).toWellFormed()` first, `codePointEnd` on the flat string — so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too — a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `…` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file — the name its chat is created with (40) and the name a refresh writes (40) — go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too — the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` — the text an MCP server's failure leaves on the owner's connector card — keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `…` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well — the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole — so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate — a character outside the BMP spends one of the budget and not two — and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. Since [#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339) the two bounds in `core/src/host/` that still judged with `text.length` use it as well — the memory review's 3,000-unit two-ended `ends` in `memory.js` and `room-context.js`'s `oneLine` (40/80/24/60) — so a user line or a board line of characters outside the BMP stays whole inside its own limit instead of being cut at half of it; both maxima and every `cut` call are unchanged, so an ASCII text judges exactly as `length <= max` did. The memory review's own third `clip` is the exported one now, so its cut reads `… (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` · 's Bot`, ` · `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index ee6a3ed46..86e5aa6cf 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -5239,7 +5239,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js +import { currentParticipants } from './room-membership.js' +import { defineTool } from './tool-schema.js' +import { clip, codePoints, splitRef, unquote } from './text.js' -+import { codePointStart, cut } from '../shared/text-boundary.js' ++import { codePointStart, cut, fits } from '../shared/text-boundary.js' import { output } from './tools.js' +import { retiredBot } from './store.js' +import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -5765,7 +5765,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js // A long message often ends with what the user wants of it, so both ends stay. - const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text) + const ends = (text, max) => { -+ if (text.length <= max) return text ++ if (fits(text, max)) return text + const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) + // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. + const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) @@ -11852,7 +11852,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js const view = (await call('memory-view', { botId: 'b2' })).value assert.equal(view.error, null) assert.equal(view.summary?.text, 'You like tea.') -@@ -1076,3 +1076,801 @@ test('a memory call names the current part of the Bot chat', async (t) => { +@@ -1076,3 +1076,830 @@ test('a memory call names the current part of the Bot chat', async (t) => { assert.equal(answer.ok, true) assert.equal(requests[0].sessionId, 'b2-part2') }) @@ -12654,6 +12654,35 @@ diff --git a/test/memory.test.js b/test/memory.test.js + const recorded = readFileSync(journalPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)) + assert.deepEqual(recorded.map(record => record.op), ['add'], 'the record lands once the file carries one name') +}) ++ ++// P-514: `ends` judged with `text.length` while it stepped by code point, so a user message under ++// REVIEW_USER_LINE_MAX (3000 characters) was cut in the middle. 2900 characters are 5800 UTF-16 ++// units, so the unfixed bound keeps 1500 of them (a 2000-unit head and a 1000-unit tail) and drops ++// 1400. ASCII is the control: it is under the limit in both readings and must read the same. ++test('a user line under 3000 characters is kept whole for the review (P-514)', async () => { ++ const render = async (text) => { ++ const history = { ++ b2: [ ++ { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, ++ { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } }, ++ ], ++ } ++ const replies = [JSON.stringify({ changes: [] })] ++ const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) ++ await call('state', {}) ++ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) ++ await reviewed(call) ++ await stop() ++ const prompt = requests[0].messages[0].content[0].text ++ const line = prompt.split('\n').find(one => one.startsWith(`User: ${[...text][0]}`)) ++ assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) ++ return line ++ } ++ const wide = '😀'.repeat(2900) ++ assert.equal(await render(wide), `User: ${wide}`, '2900 characters are under the 3000-character limit, though they are 5800 UTF-16 units') ++ const ascii = 'a'.repeat(2900) ++ assert.equal(await render(ascii), `User: ${ascii}`, 'ASCII under the limit reads the same before and after') ++}) diff --git a/test/model-menu.test.js b/test/model-menu.test.js --- a/test/model-menu.test.js +++ b/test/model-menu.test.js diff --git a/patches/ds-bot/files/src/host/room-context.js b/patches/ds-bot/files/src/host/room-context.js index 9b639b7e3..f37fc5f6f 100644 --- a/patches/ds-bot/files/src/host/room-context.js +++ b/patches/ds-bot/files/src/host/room-context.js @@ -5,7 +5,7 @@ // fill that slice. Progress for the open job is a compact board snapshot in the // same turn, not those essays. import { columnOf, compareForWork, progressOf } from '../shared/task-board.js' -import { cut } from '../shared/text-boundary.js' +import { cut, fits } from '../shared/text-boundary.js' import { fenceUntrusted } from './delivery.js' // Characters, not a tokenizer. The essays this was measured on are mostly CJK, @@ -42,7 +42,7 @@ const oneLine = (value, max) => { flat += code < 0x20 || code === 0x7f || char === '<' || char === '>' ? ' ' : char } flat = flat.replace(/\s+/g, ' ').trim() - if (flat.length <= max) return flat + if (fits(flat, max)) return flat return `${cut(flat, Math.max(0, max - 1)).text}…` } diff --git a/patches/ds-bot/files/test/room-context.test.js b/patches/ds-bot/files/test/room-context.test.js index ac5856729..c850c851c 100644 --- a/patches/ds-bot/files/test/room-context.test.js +++ b/patches/ds-bot/files/test/room-context.test.js @@ -265,3 +265,20 @@ test('P-305: a clipped transcript line ends on a whole emoji', () => { assert.equal(before.text.isWellFormed(), true) assert.equal(before.text, `${'b'.repeat(budget - 3)}\u{1F600}…`) }) + +// P-514: `oneLine` judged with `flat.length` while its cut steps by code point, so a title under +// its own limit was cut. 80 characters are under the 80-character title limit although they are +// 160 UTF-16 units; the whole title must reach the board. +test('a board line of characters outside the BMP is under its own limit and is not cut (P-514)', () => { + const title = '😀'.repeat(80) + const name = '😀'.repeat(40) + const tasks = { overall: { id: 'overall', kind: 'group', status: 'open', title, ownerBotId: 'chief', origin: { roomId: 'room' }, updatedAt: 2 } } + const board = formatRoomBoard(tasks, { roomId: 'room', nameOf: () => name }) + const line = board.split('\n').find(one => one.includes('😀')) + assert.ok(line !== undefined, `the card is on the board:\n${board}`) + // The holder name goes through oneLine at 40, the title through oneLine at 80; each is under its + // limit in characters (40 and 80) though over it in UTF-16 units (80 and 160). + assert.ok(line.startsWith(`- ${name} [`), `all 40 characters of the holder name are kept: ${line.slice(0, 60)}`) + assert.ok(line.includes(title), `all 80 characters of the title are kept: ${line.slice(-60)}`) + assert.equal(line.includes('…'), false, line) +}) diff --git a/patches/ds-bot/files/test/text-bounds.test.js b/patches/ds-bot/files/test/text-bounds.test.js index ba8a4c897..971a9a466 100644 --- a/patches/ds-bot/files/test/text-bounds.test.js +++ b/patches/ds-bot/files/test/text-bounds.test.js @@ -70,3 +70,23 @@ test('#2321: the memory review clip is the shared one, not a third copy (core/sr assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`) assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js') }) + +// P-514: two more bounds, declared inside a factory and a module rather than exported, are named +// here the same way the table names its rows: a revert to `text.length <= max` fails at the +// consumer's own `file:line`, not somewhere in a rendered prompt. +const LIMITS_514 = [ + { consumer: 'ends', path: 'host/memory.js', declaration: 'const ends = (text, max) =>' }, + { consumer: 'oneLine', path: 'host/room-context.js', declaration: 'const oneLine = (value, max) =>' }, +] + +for (const row of LIMITS_514) { + test(`P-514: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => { + const where = locate(row.path, row.declaration) + const source = read(row.path) + const from = source.indexOf(row.declaration) + assert.notEqual(from, -1, `${where} declaration moved: ${row.declaration}`) + const block = source.slice(from) + const body = block.slice(0, block.indexOf('\n\n') === -1 ? block.length : block.indexOf('\n\n')) + assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from the shared boundary`) + }) +}