diff --git a/changelog.d/2321-fits-text-bound.fixed.md b/changelog.d/2321-fits-text-bound.fixed.md new file mode 100644 index 000000000..8fbbf33b2 --- /dev/null +++ b/changelog.d/2321-fits-text-bound.fixed.md @@ -0,0 +1 @@ +The host judged a text bound in UTF-16 units while it cut in code points, so a character outside the BMP spent two of a limit's budget: a 300-character message was cut by a 400-character limit and came back short through `read_own_chat` and search rows. Bounds now judge with `fits()` in `core/src/shared/text-boundary.js`, `clip` and `excerpt` use it, and the third copy of `clip` in the memory review is the shared one ([#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321)). diff --git a/core/client.js b/core/client.js index 564017a1f..b571d9146 100644 --- a/core/client.js +++ b/core/client.js @@ -1740,6 +1740,12 @@ var isHighSurrogate = (code) => code >= 55296 && code <= 56319; var isLowSurrogate = (code) => code >= 56320 && code <= 57343; var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; +var fits = (text2, max) => { + const value = String(text2); + let n = 0; + for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 65535 ? 2 : 1; + return n <= max; +}; var cut = (text2, max) => { const flat = String(text2).toWellFormed(); return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max }; diff --git a/core/src/host/memory.js b/core/src/host/memory.js index 5acc4a326..68d50dbf3 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -27,7 +27,7 @@ import { dirname, join, relative, sep } from 'node:path' import { BlockAssembler } from '@deepseek-ai/dsh-llm' import { currentParticipants } from './room-membership.js' import { defineTool } from './tool-schema.js' -import { codePoints, splitRef, unquote } from './text.js' +import { clip, codePoints, splitRef, unquote } from './text.js' import { codePointStart, cut } from '../shared/text-boundary.js' import { output } from './tools.js' import { retiredBot } from './store.js' @@ -1267,7 +1267,6 @@ export function install(rt) { return cursors } const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve()) - const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text) // A long message often ends with what the user wants of it, so both ends stay. const ends = (text, max) => { if (text.length <= max) return text diff --git a/core/src/host/text.js b/core/src/host/text.js index ad6e627db..5e59e49a7 100644 --- a/core/src/host/text.js +++ b/core/src/host/text.js @@ -1,7 +1,7 @@ // Pure text helpers shared by the host modules: message text, names, model refs, // cutting and estimating text, and the clock format of chat anchors. import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js' -import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js' +import { codePointEnd, codePointStart, cut, fits } from '../shared/text-boundary.js' export { cut } @@ -44,7 +44,7 @@ export const codePoints = (text) => { // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { - if (text.length <= max) return text + if (fits(text, max)) return text const found = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120) const at = codePointStart(text, found) return `${at > 0 ? '…' : ''}${text.slice(at, codePointEnd(text, at + max))}…` @@ -58,7 +58,7 @@ export const estimateTokens = (text) => { } // #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never // "… (cut; undefined)"; a caller that gives one keeps "… (cut; )" byte for byte. -export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text) +export const clip = (text, max, rest) => (fits(text, max) ? text : `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})`) export const createClock = (timeZone) => { const clockFormat = new Intl.DateTimeFormat('en-CA', { diff --git a/core/src/shared/text-boundary.js b/core/src/shared/text-boundary.js index dd411bc85..903c5595e 100644 --- a/core/src/shared/text-boundary.js +++ b/core/src/shared/text-boundary.js @@ -12,6 +12,16 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC // The end of a bounded prefix, moved back onto a code point boundary. export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) +// Whether a text is inside a bound (#2321). Judged in code points, not in UTF-16 units: a character +// outside the BMP costs one of the budget, so a 300-character text is under a 400-character limit +// even though it is 600 units long, and a bound never cuts text that is already under its own limit. +export const fits = (text, max) => { + const value = String(text) + let n = 0 + for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1 + return n <= max +} + // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. // The input is flattened first (#1963): a string that already carried an unpaired surrogate went // straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit diff --git a/core/test/memory.test.js b/core/test/memory.test.js index f80b7bef8..53fc9986f 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -558,13 +558,13 @@ test('a long Bot line cut for the review context keeps whole characters (#1918)' assert.equal(prompt.includes('\ufffd'), false) const shown = prompt.split('\n').find(one => one.startsWith('Writer: x')) assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`) - assert.ok(shown.endsWith('…(cut)'), shown.slice(-20)) - assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length)) + assert.ok(shown.endsWith('… (cut)'), shown.slice(-20)) + assert.ok(shown.length <= 'Writer: '.length + 800 + ' … (cut)'.length, String(shown.length)) // #1978: pin what survives, not only how much may survive. At 800 the cut steps back off the // emoji's pair, so the whole kept line is `Writer: ` plus 791 x's; a smaller limit keeps a // shorter head — the direction that drops context silently — and must go red here. assert.ok(shown.startsWith(`Writer: ${'x'.repeat(791)}`), shown.slice(0, 40)) - assert.ok(shown.replace(' …(cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length)) + assert.ok(shown.replace('… (cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length)) }) test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => { diff --git a/core/test/text-bounds.test.js b/core/test/text-bounds.test.js new file mode 100644 index 000000000..ba8a4c897 --- /dev/null +++ b/core/test/text-bounds.test.js @@ -0,0 +1,72 @@ +// #2321: a host bound judges in code points, not in UTF-16 units. A character outside the BMP +// costs one of the budget, so a text that is under its own limit is never cut. Every row names the +// consumer that judges: a later change back to `text.length > max` fails here at that `file:line`. +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { clip, codePoints, excerpt } from '../src/host/text.js' +import * as boundary from '../src/shared/text-boundary.js' + +const read = (path) => readFileSync(fileURLToPath(new URL(`../src/${path}`, import.meta.url)), 'utf8') +const lineOf = (source, needle) => { + const at = source.indexOf(needle) + return at === -1 ? undefined : source.slice(0, at).split('\n').length +} +const locate = (path, needle) => { + const line = lineOf(read(path), needle) + return line === undefined ? `core/src/${path}` : `core/src/${path}:${line}` +} + +// The table: a limit, the text it judges, and the consumer that owns the judgment. Each text is +// under `limit` characters and over it in UTF-16 units, which is the shape #2321 is about. +const LIMITS = [ + { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(300) }, + { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(201) }, + { consumer: 'excerpt', path: 'host/text.js', declaration: 'export const excerpt =', limit: 400, text: '😀'.repeat(300) }, +] + +const run = (row) => (row.consumer === 'clip' ? clip(row.text, row.limit, 'r') : excerpt(row.text, '', row.limit)) + +for (const row of LIMITS) { + test(`#2321: ${row.consumer} keeps a text under its own limit (${locate(row.path, row.declaration)})`, () => { + const kept = `${codePoints(row.text)} characters / ${row.text.length} UTF-16 units, limit ${row.limit}` + assert.equal(run(row), row.text, `a text of ${kept} is under its limit, so ${row.consumer} must not cut it`) + }) +} + +// The judgment itself: each consumer must ask `fits`, so reverting one to `text.length > max` names +// only that consumer's own file:line and the table cannot rot silently. +for (const row of LIMITS) { + test(`#2321: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => { + const where = locate(row.path, row.declaration) + const source = read(row.path) + const from = source.indexOf(row.declaration) + const block = source.slice(from) + const body = row.consumer === 'clip' ? block.slice(0, block.indexOf('\n')) : block.slice(0, block.indexOf('\n}') + 2) + assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from shared/text-boundary.js`) + }) +} + +test('#2321: an over-limit clip still cuts and says so', () => { + const out = clip('😀'.repeat(201), 200, 'r') + assert.equal(out.endsWith('… (cut; r)'), true, out.slice(-24)) + assert.equal(codePoints(out.slice(0, -'… (cut; r)'.length)) <= 200, true, `it keeps at most 200 characters: ${codePoints(out)}`) +}) + +test('#2321: fits counts code points, not UTF-16 units (core/src/shared/text-boundary.js)', () => { + assert.equal(typeof boundary.fits, 'function', 'shared/text-boundary.js must export fits(text, max)') + assert.equal(boundary.fits('', 0), true) + assert.equal(boundary.fits('abc', 3), true) + assert.equal(boundary.fits('abcd', 3), false) + assert.equal(boundary.fits('😀'.repeat(201), 400), true, '201 characters are under a 400-character limit, though 402 UTF-16 units are not') + assert.equal(boundary.fits('😀'.repeat(401), 400), false) + assert.equal(boundary.fits('a😀b', 3), true) +}) + +test('#2321: the memory review clip is the shared one, not a third copy (core/src/host/memory.js)', () => { + const source = read('host/memory.js') + const duplicate = lineOf(source, 'const clip = (text, max) =>') + assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`) + assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js') +}) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 9dc15dd86..f978f38a4 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `…` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `…` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds — `String(text).toWellFormed()` first, `codePointEnd` on the flat string — so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too — a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `…` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file — the name its chat is created with (40) and the name a refresh writes (40) — go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too — the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` — the text an MCP server's failure leaves on the owner's connector card — keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `…` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well — the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole — so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `…` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `…` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds — `String(text).toWellFormed()` first, `codePointEnd` on the flat string — so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too — a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `…` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file — the name its chat is created with (40) and the name a refresh writes (40) — go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too — the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) — so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` — the text an MCP server's failure leaves on the owner's connector card — keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `…` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well — the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole — so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate — a character outside the BMP spends one of the budget and not two — and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. The memory review's own third `clip` is the exported one now, so its cut reads `… (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` · 's Bot`, ` · `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index a4a70edfc..b4d3c403d 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -5219,7 +5219,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js -import { splitRef, unquote } from './text.js' +import { currentParticipants } from './room-membership.js' +import { defineTool } from './tool-schema.js' -+import { codePoints, splitRef, unquote } from './text.js' ++import { clip, codePoints, splitRef, unquote } from './text.js' +import { codePointStart, cut } from '../shared/text-boundary.js' import { output } from './tools.js' +import { retiredBot } from './store.js' @@ -5720,7 +5720,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js if (text === '') throw new Error('Type a question or a change first') const numbered = [ ...allEntries(loadScope(ownScope(bot.id))).map(entry => ({ ...entry, scope: 'own' })), -@@ -1074,18 +1074,28 @@ export function install(rt) { +@@ -1074,18 +1074,27 @@ export function install(rt) { let cursors const cursorsNow = () => { if (cursors === undefined) { @@ -5743,7 +5743,6 @@ diff --git a/src/host/memory.js b/src/host/memory.js } - const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} …(cut)` : text) + const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve()) -+ const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text) // A long message often ends with what the user wants of it, so both ends stay. - const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text) + const ends = (text, max) => { @@ -7199,7 +7198,7 @@ diff --git a/src/host/text.js b/src/host/text.js // Pure text helpers shared by the host modules: message text, names, model refs, // cutting and estimating text, and the clock format of chat anchors. import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js' -+import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js' ++import { codePointEnd, codePointStart, cut, fits } from '../shared/text-boundary.js' + +export { cut } @@ -7227,9 +7226,10 @@ diff --git a/src/host/text.js b/src/host/text.js + // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { - if (text.length <= max) return text +- if (text.length <= max) return text - const at = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120) - return `${at > 0 ? '…' : ''}${text.slice(at, at + max)}…` ++ if (fits(text, max)) return text + const found = wanted === '' ? 0 : Math.max(0, text.toLowerCase().indexOf(wanted) - 120) + const at = codePointStart(text, found) + return `${at > 0 ? '…' : ''}${text.slice(at, codePointEnd(text, at + max))}…` @@ -7243,7 +7243,7 @@ diff --git a/src/host/text.js b/src/host/text.js -export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, max)}… (cut; ${rest})` : text) +// #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never +// "… (cut; undefined)"; a caller that gives one keeps "… (cut; )" byte for byte. -+export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text) ++export const clip = (text, max, rest) => (fits(text, max) ? text : `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})`) export const createClock = (timeZone) => { const clockFormat = new Intl.DateTimeFormat('en-CA', { @@ -11753,13 +11753,13 @@ diff --git a/test/memory.test.js b/test/memory.test.js + assert.equal(prompt.includes('\ufffd'), false) + const shown = prompt.split('\n').find(one => one.startsWith('Writer: x')) + assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`) -+ assert.ok(shown.endsWith('…(cut)'), shown.slice(-20)) -+ assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length)) ++ assert.ok(shown.endsWith('… (cut)'), shown.slice(-20)) ++ assert.ok(shown.length <= 'Writer: '.length + 800 + ' … (cut)'.length, String(shown.length)) + // #1978: pin what survives, not only how much may survive. At 800 the cut steps back off the + // emoji's pair, so the whole kept line is `Writer: ` plus 791 x's; a smaller limit keeps a + // shorter head — the direction that drops context silently — and must go red here. + assert.ok(shown.startsWith(`Writer: ${'x'.repeat(791)}`), shown.slice(0, 40)) -+ assert.ok(shown.replace(' …(cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length)) ++ assert.ok(shown.replace('… (cut)', '').length > 'Writer: '.length + 800 - 10, String(shown.length)) +}) + +test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => { diff --git a/patches/ds-bot/0008-client-bundle.patch b/patches/ds-bot/0008-client-bundle.patch index e30494886..467f870e9 100644 --- a/patches/ds-bot/0008-client-bundle.patch +++ b/patches/ds-bot/0008-client-bundle.patch @@ -1136,7 +1136,7 @@ diff --git a/client.js b/client.js if (!REFUSALS.has(code)) { noteClientError(endpoint, Object.assign(new Error(result.error?.message ?? "Request failed"), { code })); markNoted(error); -@@ -1097,6 +1097,400 @@ var call = async (endpoint, payload, signal) => { +@@ -1097,6 +1097,406 @@ var call = async (endpoint, payload, signal) => { return result.value; }; @@ -1145,6 +1145,12 @@ diff --git a/client.js b/client.js +var isLowSurrogate = (code) => code >= 56320 && code <= 57343; +var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; +var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; ++var fits = (text2, max) => { ++ const value = String(text2); ++ let n = 0; ++ for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 65535 ? 2 : 1; ++ return n <= max; ++}; +var cut = (text2, max) => { + const flat = String(text2).toWellFormed(); + return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max }; @@ -1876,7 +1882,6 @@ diff --git a/client.js b/client.js } return spans; } -- +function unmatchedMentions(text2, bots, { references = [] } = {}) { + const value = String(text2 ?? ""); + const skip = [ @@ -1924,7 +1929,7 @@ diff --git a/client.js b/client.js + } + return stripFence(out).replace(/\s+/g, " ").trim(); +} -+ + // src/client/text.js var MENTION_ORIGIN = "https://bots.local/"; -function linkMentions(text, bots, selfId) { @@ -2170,7 +2175,7 @@ diff --git a/client.js b/client.js var formatTokens = (count) => count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count); function usageOf(list, ids) { let input = 0; -@@ -1624,970 +1624,2058 @@ function usageOf(list, ids) { +@@ -1624,969 +1624,2057 @@ function usageOf(list, ids) { const hit = input > 0 ? Math.round(cacheRead / input * 100) : 0; return t("{tokens} tok · Cache hit {hit}%", { tokens: formatTokens(input + output), hit }); } @@ -2439,11 +2444,15 @@ diff --git a/client.js b/client.js - }; - animation.oncancel = () => { - if (!node.getAnimations().some((other) => other.id === "bt-morph" && other !== animation)) delete node.dataset.morph; +- }; +- }); +function isAgentSession(roster, cache, sessionId) { + if (sessionId === void 0 || sessionId === null || sessionId === "") return false; + if (roster.ready) return roster.agentOf[sessionId] !== void 0; + return cache.has(sessionId); -+} + } +-function useLinger(value, ms) { +- const [kept, setKept] = (0, import_react.useState)(value); +function countCards(state, sessionId, callIds, delta) { + const own = { ...state[sessionId] }; + for (const callId of callIds) { @@ -2460,7 +2469,9 @@ diff --git a/client.js b/client.js +var botState = (running, activity, alert) => running ? STATES.includes(activity) ? activity : "working" : alert ? "alert" : "idle"; +var toolState = (name) => /^(message_bot|stop_bot|create_bot|update_bot|create_group|update_group|delete_group|post_to_group)$/.test(name) ? "sending" : /search|fetch|browse|read|grep|glob|list|recall/i.test(name) ? "searching" : "working"; +function useGaze(ref, enabled) { -+ (0, import_react.useEffect)(() => { + (0, import_react.useEffect)(() => { +- if (value) { +- setKept(value); + if (!enabled) return void 0; + let frame = 0; + let point = null; @@ -2472,8 +2483,7 @@ diff --git a/client.js b/client.js + const clamp = (value) => Math.max(-1, Math.min(1, value)); + node.style.setProperty("--bt-gx", clamp((point.x - box.left - box.width / 2) / 160).toFixed(2)); + node.style.setProperty("--bt-gy", clamp((point.y - box.top - box.height / 2) / 160).toFixed(2)); - }; -- }); ++ }; + const onMove = (event) => { + point = { x: event.clientX, y: event.clientY }; + if (!frame) frame = requestAnimationFrame(place); @@ -2491,9 +2501,7 @@ diff --git a/client.js b/client.js + document.documentElement.removeEventListener("pointerleave", onLeave); + }; + }, [enabled]); - } --function useLinger(value, ms) { -- const [kept, setKept] = (0, import_react.useState)(value); ++} +var FIDGETS = ["hop", "sway", "stretch", "look"]; +function BotMark({ bot, look: given, size = 36, state = "idle", live = false, gaze = false, pokeable = false, label }) { + const look = given ?? (bot?.remote ? lookAvoidingLocals(bot) : lookOf(bot)); @@ -2573,9 +2581,7 @@ diff --git a/client.js b/client.js +function useSettled(state) { + const prev = (0, import_react.useRef)(state); + const [settled, setSettled] = (0, import_react.useState)(false); - (0, import_react.useEffect)(() => { -- if (value) { -- setKept(value); ++ (0, import_react.useEffect)(() => { + const was = prev.current; + prev.current = state; + if (state !== "idle") { @@ -4291,7 +4297,6 @@ diff --git a/client.js b/client.js -.bt-panel-view>[data-move=out]{animation:bt-page-out ${SPRING_SHAPE.duration}ms ${SPRING_SHAPE.easing} backwards,bt-veil-in .2s ease-out backwards} -.bt-panel-view>:is([data-move=in],[data-move=out]):nth-child(2){animation-delay:25ms} -.bt-panel-view>:is([data-move=in],[data-move=out]):nth-child(n+3){animation-delay:50ms} --.bt-panel-view>.bt-drawer-body[data-move=tab]{animation:bt-veil-in .2s ease-out} +// src/client/styles.js +var SHELL_CSS = ` +body{--bt-ink:#141414;--bt-ink-2:rgba(20,20,20,.6);--bt-ink-3:rgba(20,20,20,.36);--bt-bubble:#eeeeee;--bt-sidebar:#f7f7f7;--bt-main:#fcfcfc;--bt-line:rgba(20,20,20,.08);--bt-line-2:rgba(20,20,20,.15);--bt-line-weak:rgba(20,20,20,.1);--bt-hover:rgba(119,119,119,.09);--bt-active:rgba(119,119,119,.17);--bt-card:#ffffff;--bt-accent:#4d6bfe;--bt-green:#00c972;--bt-star:#ff9800;--bt-user:#070707;--bt-user-ink:#fcfcfc;--bt-tag-bg:rgba(20,20,20,.06);--bt-tag-ink:rgba(20,20,20,.56)} @@ -5160,10 +5165,9 @@ diff --git a/client.js b/client.js +.bt-panel-view>[data-move=out]{animation:bt-page-out ${SPRING_SHAPE.duration}ms ${SPRING_SHAPE.easing} backwards,bt-veil-in .2s ease-out backwards} +.bt-panel-view>:is([data-move=in],[data-move=out]):nth-child(2){animation-delay:25ms} +.bt-panel-view>:is([data-move=in],[data-move=out]):nth-child(n+3){animation-delay:50ms} -+.bt-panel-view>.bt-drawer-body[data-move=tab]{animation:bt-veil-in .2s ease-out} + .bt-panel-view>.bt-drawer-body[data-move=tab]{animation:bt-veil-in .2s ease-out} .bt-drawer-name-input{animation:bt-veil-in .16s ease-out} .bt-pop-in{display:inline-flex;animation:bt-check-pop ${SPRING_POP.duration}ms ${SPRING_POP.easing}} - :is(.bt-soft,.bt-send,.bt-icon-btn,.bt-settings-close,.bt-mini,.bt-tab,.bt-nc-send,.bt-avedit-tab,.bt-shape,.bt-pick,.bt-swatch,.bt-palette-clear){transition:background-color .12s ease,color .12s ease,border-color .12s ease,opacity .12s ease,${spring(SPRING_TAP, "scale")}} @@ -2597,9 +2597,20 @@ body[data-ds-dark-theme] .bt-theme-chip i{background:var(--da)} `; var CONVERSATION_CSS = ` diff --git a/patches/ds-bot/files/src/shared/text-boundary.js b/patches/ds-bot/files/src/shared/text-boundary.js index dd411bc85..903c5595e 100644 --- a/patches/ds-bot/files/src/shared/text-boundary.js +++ b/patches/ds-bot/files/src/shared/text-boundary.js @@ -12,6 +12,16 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC // The end of a bounded prefix, moved back onto a code point boundary. export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) +// Whether a text is inside a bound (#2321). Judged in code points, not in UTF-16 units: a character +// outside the BMP costs one of the budget, so a 300-character text is under a 400-character limit +// even though it is 600 units long, and a bound never cuts text that is already under its own limit. +export const fits = (text, max) => { + const value = String(text) + let n = 0 + for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1 + return n <= max +} + // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. // The input is flattened first (#1963): a string that already carried an unpaired surrogate went // straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit diff --git a/patches/ds-bot/files/test/text-bounds.test.js b/patches/ds-bot/files/test/text-bounds.test.js new file mode 100644 index 000000000..ba8a4c897 --- /dev/null +++ b/patches/ds-bot/files/test/text-bounds.test.js @@ -0,0 +1,72 @@ +// #2321: a host bound judges in code points, not in UTF-16 units. A character outside the BMP +// costs one of the budget, so a text that is under its own limit is never cut. Every row names the +// consumer that judges: a later change back to `text.length > max` fails here at that `file:line`. +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { clip, codePoints, excerpt } from '../src/host/text.js' +import * as boundary from '../src/shared/text-boundary.js' + +const read = (path) => readFileSync(fileURLToPath(new URL(`../src/${path}`, import.meta.url)), 'utf8') +const lineOf = (source, needle) => { + const at = source.indexOf(needle) + return at === -1 ? undefined : source.slice(0, at).split('\n').length +} +const locate = (path, needle) => { + const line = lineOf(read(path), needle) + return line === undefined ? `core/src/${path}` : `core/src/${path}:${line}` +} + +// The table: a limit, the text it judges, and the consumer that owns the judgment. Each text is +// under `limit` characters and over it in UTF-16 units, which is the shape #2321 is about. +const LIMITS = [ + { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(300) }, + { consumer: 'clip', path: 'host/text.js', declaration: 'export const clip =', limit: 400, text: '😀'.repeat(201) }, + { consumer: 'excerpt', path: 'host/text.js', declaration: 'export const excerpt =', limit: 400, text: '😀'.repeat(300) }, +] + +const run = (row) => (row.consumer === 'clip' ? clip(row.text, row.limit, 'r') : excerpt(row.text, '', row.limit)) + +for (const row of LIMITS) { + test(`#2321: ${row.consumer} keeps a text under its own limit (${locate(row.path, row.declaration)})`, () => { + const kept = `${codePoints(row.text)} characters / ${row.text.length} UTF-16 units, limit ${row.limit}` + assert.equal(run(row), row.text, `a text of ${kept} is under its limit, so ${row.consumer} must not cut it`) + }) +} + +// The judgment itself: each consumer must ask `fits`, so reverting one to `text.length > max` names +// only that consumer's own file:line and the table cannot rot silently. +for (const row of LIMITS) { + test(`#2321: ${row.consumer} judges with fits, not with UTF-16 units (${locate(row.path, row.declaration)})`, () => { + const where = locate(row.path, row.declaration) + const source = read(row.path) + const from = source.indexOf(row.declaration) + const block = source.slice(from) + const body = row.consumer === 'clip' ? block.slice(0, block.indexOf('\n')) : block.slice(0, block.indexOf('\n}') + 2) + assert.equal(body.includes('fits('), true, `${where} judges in UTF-16 units: it must judge with fits(...) from shared/text-boundary.js`) + }) +} + +test('#2321: an over-limit clip still cuts and says so', () => { + const out = clip('😀'.repeat(201), 200, 'r') + assert.equal(out.endsWith('… (cut; r)'), true, out.slice(-24)) + assert.equal(codePoints(out.slice(0, -'… (cut; r)'.length)) <= 200, true, `it keeps at most 200 characters: ${codePoints(out)}`) +}) + +test('#2321: fits counts code points, not UTF-16 units (core/src/shared/text-boundary.js)', () => { + assert.equal(typeof boundary.fits, 'function', 'shared/text-boundary.js must export fits(text, max)') + assert.equal(boundary.fits('', 0), true) + assert.equal(boundary.fits('abc', 3), true) + assert.equal(boundary.fits('abcd', 3), false) + assert.equal(boundary.fits('😀'.repeat(201), 400), true, '201 characters are under a 400-character limit, though 402 UTF-16 units are not') + assert.equal(boundary.fits('😀'.repeat(401), 400), false) + assert.equal(boundary.fits('a😀b', 3), true) +}) + +test('#2321: the memory review clip is the shared one, not a third copy (core/src/host/memory.js)', () => { + const source = read('host/memory.js') + const duplicate = lineOf(source, 'const clip = (text, max) =>') + assert.equal(duplicate, undefined, `core/src/host/memory.js:${duplicate} keeps its own clip: import the one from ./text.js`) + assert.match(source, /import \{[^}]*\bclip\b[^}]*\} from '\.\/text\.js'/, 'core/src/host/memory.js must import clip from ./text.js') +})