diff --git a/changelog.d/1912-archive-cut-boundary.fixed.md b/changelog.d/1912-archive-cut-boundary.fixed.md new file mode 100644 index 000000000..d99f9a3da --- /dev/null +++ b/changelog.d/1912-archive-cut-boundary.fixed.md @@ -0,0 +1 @@ +A decision or a meeting archive no longer stores half of an emoji ([#1912](https://github.com/ThinkFlowLab/FlyBot/issues/1912)): the host's one `cut` helper stops a bounded prefix before the surrogate pair it would split, and every archive writer now uses it instead of `slice(0, max)` β€” `decisions.jsonl`, `decisions/-.md`, and the claim line, diagnostics report, activity-feed label and browser text beside them β€” so the stored record round-trips through UTF-8 as the whole character instead of U+FFFD. diff --git a/core/src/host/browser.js b/core/src/host/browser.js index 46f8babe7..c12a9afa4 100644 --- a/core/src/host/browser.js +++ b/core/src/host/browser.js @@ -3,6 +3,7 @@ // the app window, not here, so `read_browser` reaches it through the window: the client // holds a long poll open (`browser-wait`) that reports which Bots have a page open and // carries read requests back; the client answers with `browser-result`. +import { cut } from './text.js' import { defineTool } from './tool-schema.js' import { output } from './tools.js' @@ -131,7 +132,7 @@ export function createBrowserChannel({ now = Date.now, waitMs = WAIT_MS, staleMs return { wait, result, read, openPages, dispose } } -const clean = (value, max) => String(value ?? '').replace(/\s+/g, ' ').trim().slice(0, max) +const clean = (value, max) => cut(String(value ?? '').replace(/\s+/g, ' ').trim(), max).text /** * Where a part of the text that starts at `start` ends: at a blank line, else a line break, diff --git a/core/src/host/decisions.js b/core/src/host/decisions.js index d5bb479b8..5c3ff2fc1 100644 --- a/core/src/host/decisions.js +++ b/core/src/host/decisions.js @@ -10,6 +10,7 @@ // can show it without reading the file. import { appendFile, mkdir, open, readFile, rename, stat, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' +import { cut } from './text.js' import { defineTool } from './tool-schema.js' import { looksSecret } from './memory.js' import { output } from './tools.js' @@ -23,7 +24,7 @@ const KEPT = 2_000 const MAX_BYTES = 1_000_000 const NAME_SAFE = /^[\w-]{1,128}$/ -const oneLine = (text, max) => String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) +const oneLine = (text, max) => cut(String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim(), max).text export function install(rt) { if (rt.config?.decisions !== true) { diff --git a/core/src/host/diagnostics.js b/core/src/host/diagnostics.js index 56a9c4a97..c2daf6dff 100644 --- a/core/src/host/diagnostics.js +++ b/core/src/host/diagnostics.js @@ -10,6 +10,7 @@ import { arch, homedir, platform, release } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { format } from 'node:util' +import { cut } from './text.js' export const KEEP = 200 const LOG_MAX_BYTES = 1 << 20 @@ -47,7 +48,8 @@ export function cleanText(text, { redact = value => value, home = homedir() } = const doubled = home.replace(/\\/g, '\\\\') if (doubled !== home) out = out.replace(new RegExp(quote(doubled), 'g'), '~') } - return out.length > TEXT_MAX ? `${out.slice(0, TEXT_MAX)}…` : out + const kept = cut(out, TEXT_MAX) + return kept.cut ? `${kept.text}…` : out } const errorText = error => `${error.name && error.name !== 'Error' ? `${error.name}: ` : ''}${error.message}` diff --git a/core/src/host/flybot-feed.js b/core/src/host/flybot-feed.js index 0e38a6b2d..1b6af294f 100644 --- a/core/src/host/flybot-feed.js +++ b/core/src/host/flybot-feed.js @@ -8,6 +8,7 @@ // user pasted or a tool printed. Strings that came from outside the team (a tool name // from an MCP server, a remote handle, an error code) are cleaned here. import { SOURCE_KIND } from './constants.js' +import { cut } from './text.js' import { normRemoteHandle, remoteBotTitle } from '../shared/remote-label.js' export const FEED_EVENTS = Object.freeze(['turn.start', 'turn.end', 'tool.start', 'tool.end', 'exchange.delivered', 'decision']) @@ -22,7 +23,7 @@ const CONTROLS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g export const cleanLabel = (value, max = 80) => { let out = String(value ?? '').replace(CONTROLS, ' ') for (let previous; previous !== out;) { previous = out; out = out.replace(/<\s*\/?\s*untrusted-?/gi, '') } - return out.replace(/\s+/g, ' ').trim().slice(0, max) + return cut(out.replace(/\s+/g, ' ').trim(), max).text } // An error code is an enumerated value (RATE_LIMIT, AUTH): upper-case words only, so free text or a key put in the field is not kept. const cleanCode = code => (typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,39}$/.test(code) ? code : 'UNKNOWN') diff --git a/core/src/host/groups.js b/core/src/host/groups.js index 01098752f..76eefe12c 100644 --- a/core/src/host/groups.js +++ b/core/src/host/groups.js @@ -21,7 +21,7 @@ import { createUserMessage } from '@deepseek-ai/dsh-llm' import { defineTool } from './tool-schema.js' import { ROOM_LATE_ROUNDS, SOURCE_KIND } from './constants.js' import { OWNER, OTHER, SCHEDULED, combine } from './turn-origin.js' -import { escapeRegExp, textOf } from './text.js' +import { cut, escapeRegExp, textOf } from './text.js' import { mentionSpans } from '../shared/mentions.js' import { GROUP_REF, output } from './tools.js' import { failureLine } from './turns.js' @@ -43,7 +43,7 @@ export const CLAIM_CUE_REPEAT_MS = 30 * 60 * 1000 // that requires the line is (#1253), not because the room is assumed to speak Chinese. const CLAIM_CUE_REF_MAX = 60 const CLAIM_CUE_STEP_MAX = 100 -const claimLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) +const claimLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text export function claimCueText(task) { const title = claimLine(task?.title, 300) if (title === '') return '' diff --git a/core/src/host/meetings.js b/core/src/host/meetings.js index 38b8efb55..a6eed6405 100644 --- a/core/src/host/meetings.js +++ b/core/src/host/meetings.js @@ -8,6 +8,7 @@ import { createHash, randomBytes } from 'node:crypto' import { link, mkdir, open, readFile, rm } from 'node:fs/promises' import { join } from 'node:path' +import { cut } from './text.js' import { defineTool } from './tool-schema.js' import { output } from './tools.js' import { workDirFor } from './project-state.js' @@ -23,7 +24,7 @@ const NAME_MAX = 80 const PARTICIPANTS_MAX = 32 const OPEN_STATES = new Set(['open', 'deliberating']) -const oneLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) +const oneLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text // A slug keeps letters and digits of every script, so two different Chinese questions do not both // fall back to `meeting` and collide on one day's file. const slugOf = value => oneLine(value, 80).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '').slice(0, 80) diff --git a/core/src/host/text.js b/core/src/host/text.js index 6525bb29d..5b109c849 100644 --- a/core/src/host/text.js +++ b/core/src/host/text.js @@ -43,6 +43,10 @@ const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) +// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own +// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped. +export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) + // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { if (text.length <= max) return text diff --git a/core/test/decisions.test.js b/core/test/decisions.test.js index dac8effd7..aefdbd2f3 100644 --- a/core/test/decisions.test.js +++ b/core/test/decisions.test.js @@ -84,3 +84,16 @@ test('#603: a byte cap bounds an oversized log left by an older build', async (t assert.equal(log.at(-1).option, 'after the byte cap') assert.equal(log.length, 2) }) + +test('#1912: an option cut at an emoji boundary stays well formed in the log', async (t) => { + const { home, tool, stop } = await boot({ decisions: true }) + t.after(stop) + // The 200-unit cut lands inside the emoji, so a code-unit slice stores half a surrogate pair. + const option = `${'a'.repeat(199)}πŸ˜€tail` + await tool('record_decision', { option, choice: 'used' }) + const text = readFileSync(join(home, 'decisions.jsonl'), 'utf8') + const [entry] = text.trim().split('\n').map(line => JSON.parse(line)) + assert.equal(entry.option.isWellFormed(), true, `the stored option must stay well formed, got: ${JSON.stringify(entry.option.slice(-4))}`) + assert.equal(entry.option.length <= 200, true, `the stored option must stay within 200 units, got ${entry.option.length}`) + assert.equal(option.startsWith(entry.option), true, 'the stored option must be a prefix of the recorded one') +}) diff --git a/core/test/meetings-persist.test.js b/core/test/meetings-persist.test.js index aae4d3fed..be294e6ca 100644 --- a/core/test/meetings-persist.test.js +++ b/core/test/meetings-persist.test.js @@ -3,7 +3,7 @@ // second decide is refused, with no second decision file and no second card), and an open meeting // keeps its recorded positions. import assert from 'node:assert/strict' -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' @@ -85,3 +85,22 @@ test('P-209: an open meeting and a recorded position survive a restart', async ( assert.match(read, /\[deliberating\]/, `an open meeting must survive a restart, got: ${read.split('\n')[0]}`) assert.match(read, /- Chief: agree β€” the existing store/, `the position must survive, got: ${read}`) }) + +test('#1912: a decision cut at an emoji boundary writes no replacement character into the archive', async (t) => { + const { dir, meeting } = await setup(t) + const text = await meeting({ action: 'open', question: 'Which storage shape should Stage 4 use?', participants: ['Chief', 'Dev', 'Reviewer'] }) + const id = text.split(' ')[0] + await positionAll(meeting, id) + // The 600-unit cut lands inside the emoji, so a code-unit slice keeps half of a surrogate pair. + const decision = `${'a'.repeat(599)}πŸ˜€tail` + await meeting({ action: 'decide', meeting: id, owner: 'Chief', decision, evidence: [{ pr: 1912 }] }) + const [file] = readdirSync(join(dir, 'decisions')).filter(name => name.endsWith('.md')) + const bytes = readFileSync(join(dir, 'decisions', file)) + const written = bytes.toString('utf8') + assert.equal(written.includes('\ufffd'), false, 'the archive must not hold U+FFFD') + assert.equal(Buffer.from(written, 'utf8').equals(bytes), true, 'the archive must round-trip through UTF-8 unchanged') + const lines = written.split('\n') + const kept = lines[lines.indexOf('## Decision') + 2] + assert.equal(decision.startsWith(kept), true, `the kept body must be a prefix of the decision, got: ${JSON.stringify(kept)}`) + assert.equal(kept.length <= 600, true, `the kept body must stay within 600 units, got ${kept.length}`) +}) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 539db1c57..7602d4ca4 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit β€” one character costs one whatever its UTF-16 width β€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit β€” one character costs one whatever its UTF-16 width β€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it β€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` β€” so a decision or a meeting record holds the whole character instead of U+FFFD. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative β€” an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` Β· 's Bot`, ` Β· `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/docs/subsystems/plugin-runtime.md b/docs/subsystems/plugin-runtime.md index cbbc2ab85..64afeee7b 100644 --- a/docs/subsystems/plugin-runtime.md +++ b/docs/subsystems/plugin-runtime.md @@ -35,7 +35,7 @@ FlyBot's plugin API surface on top of the patched core: host hooks, client slots | File | Role | | --- | --- | | `core/src/host/hooks.js` | Host hook points: send middleware, transports, grants, the owner-bound held-send approvals (#52), task hold/decide/delete seams (#54), and `bot.update({ name?, hidden? })` for plugin rename/hide (#452). A hold's waiting status is the shared rule `src/shared/approval-waiting.js` `isApprovalWaiting()`: `decide()`, `heldOf()` (the chat card), `PENDING_MAX`, the in-flight filter and the `APPROVALS_KEPT` cleanup all ask it instead of comparing to `pending`, so a `pending_approval` hold is decidable, drawn as waiting, counted and never cleaned up mid-wait (#754). It also exposes `flybot.approvals.stats()` β€” per-Bot, per-status counts of held sends, read-only (#396/#57 direction). A dispatch that fails keeps its error on the record, and an error the sender marks `noRetry: true` β€” a refusal nothing can change, such as a friend’s Bot no longer shared β€” reaches the chat view as `retry: false` so the card draws no Retry (#954); every other failure stays retryable. Whether a failure can still be retried is one rule in `src/shared/held-retry.js` (#968): `canRetry(record)` reads the flags or, for an older record, the stored sentence, and `approvals.list` marks a failed terminal approval `retry: false` so a dock card and the chat card answer the same question the same way. `rooms.list()` copies `sharedFrom.remoteRoomId`, `ticket` and `inviterHandle` when a room has them, so a mention of an inbound seat can name the host's room id inside the share line (#1126). | -| `core/src/host/flybot-feed.js` | Activity feed: folds a Bot Session's `session/event`s into `flybot.on('turn.start' \| 'turn.end' \| 'tool.start' \| 'tool.end' \| 'exchange.delivered' \| 'decision')` (#241). `turn.end` also carries the turn's token counts (summed from its `assistant/message` usage, floored and capped, counts only) under `tokens` (#357). `decision` is fired by `decisions.js` (#396), not folded from a Session event. | +| `core/src/host/flybot-feed.js` | Activity feed: folds a Bot Session's `session/event`s into `flybot.on('turn.start' \| 'turn.end' \| 'tool.start' \| 'tool.end' \| 'exchange.delivered' \| 'decision')` (#241). `turn.end` also carries the turn's token counts (summed from its `assistant/message` usage, floored and capped, counts only) under `tokens` (#357). `decision` is fired by `decisions.js` (#396), not folded from a Session event. Since #1912 `cleanLabel` bounds its text through `cut` in `text.js`, so a cleaned tool name or remote handle never ends on half of a surrogate pair. | | `core/src/client/flybot-slots.js` | Client slots, including the approval card queue (`flybot.approvals`, `flybot/decide`) and `flybot.sidebar.structure` (`parentOf`, optional `include`, `menuItems`, `nodeActions`). Every `flybot.sidebar.section` entry is wrapped in `div.bt-flybot-section`, and a wrapper a section leaves empty (its component renders null) draws no separator: the `:empty` rule zeroes that margin, border and padding, keyed on emptiness rather than on a section id, so the next null-rendering section is covered too (#818; `core/test/flybot-section-empty.test.js` renders the host entry and pins both rules). Its Approve / Reject and the held-reply card both ask `approval-waiting.js` `isApprovalWaiting()` (`pending` and `pending_approval` wait), so a hold carrying the spelling Bot Social's own outbound hold uses stays answerable (#734). Its dock card draws **Retry** only while `src/shared/held-retry.js` `canRetry(approval)` allows it β€” a terminal failure asks one shared predicate instead of `status === 'failed'` (#968) β€” and it publishes `globalThis.__flybot__.canRetry` so a classic-script plugin's own dock asks the same rule rather than keeping a second copy. | | `core/src/client/flybot-registry.js` | Registry. A friend's Bot row in the @ menu carries `remoteMentionLabel` β€” the one name its bubble and the sidebar preview wear β€” so the menu inserts the name the user reads, while the row keeps the handle as its value and the send still routes by handle (#967). | | `core/types/flybot.d.ts` | Types (`BotView.hidden`, `bot.update`, …). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index e60ca789f..b344ae143 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4467,7 +4467,14 @@ diff --git a/src/host/connectors.js b/src/host/connectors.js diff --git a/src/host/diagnostics.js b/src/host/diagnostics.js --- a/src/host/diagnostics.js +++ b/src/host/diagnostics.js -@@ -16,7 +16,7 @@ const LOG_MAX_BYTES = 1 << 20 +@@ -10,13 +10,14 @@ import { arch, homedir, platform, release } from 'node:os' + import { dirname, join } from 'node:path' + import { fileURLToPath } from 'node:url' + import { format } from 'node:util' ++import { cut } from './text.js' + + export const KEEP = 200 + const LOG_MAX_BYTES = 1 << 20 const STACK_LINES = 8 const TEXT_MAX = 2000 export const REPORT_ENTRIES = 20 @@ -4476,6 +4483,16 @@ diff --git a/src/host/diagnostics.js b/src/host/diagnostics.js const quote = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +@@ -47,7 +47,8 @@ export function cleanText(text, { redact = value => value, home = homedir() } = + const doubled = home.replace(/\\/g, '\\\\') + if (doubled !== home) out = out.replace(new RegExp(quote(doubled), 'g'), '~') + } +- return out.length > TEXT_MAX ? `${out.slice(0, TEXT_MAX)}…` : out ++ const kept = cut(out, TEXT_MAX) ++ return kept.cut ? `${kept.text}…` : out + } + + const errorText = error => `${error.name && error.name !== 'Error' ? `${error.name}: ` : ''}${error.message}` @@ -66,7 +66,7 @@ export function reportMarkdown({ facts, client = {}, entries = [], clientEntries const lines = [ '| | |', @@ -6275,7 +6292,7 @@ diff --git a/src/host/steward.js b/src/host/steward.js diff --git a/src/host/text.js b/src/host/text.js --- a/src/host/text.js +++ b/src/host/text.js -@@ -23,11 +23,32 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'β€œβ€γ€Œ +@@ -23,11 +23,36 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'β€œβ€γ€Œ export const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -6298,6 +6315,10 @@ diff --git a/src/host/text.js b/src/host/text.js +// client side, here for the host). +const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) +const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) ++ ++// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own ++// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped. ++export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) + // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { diff --git a/patches/ds-bot/0012-room-membership.patch b/patches/ds-bot/0012-room-membership.patch index 2fe35ee7a..e2bf5473a 100644 --- a/patches/ds-bot/0012-room-membership.patch +++ b/patches/ds-bot/0012-room-membership.patch @@ -857,7 +857,7 @@ diff --git a/src/host/groups.js b/src/host/groups.js +import { defineTool } from './tool-schema.js' +import { ROOM_LATE_ROUNDS, SOURCE_KIND } from './constants.js' +import { OWNER, OTHER, SCHEDULED, combine } from './turn-origin.js' -+import { escapeRegExp, textOf } from './text.js' ++import { cut, escapeRegExp, textOf } from './text.js' import { mentionSpans } from '../shared/mentions.js' import { GROUP_REF, output } from './tools.js' import { failureLine } from './turns.js' @@ -879,7 +879,7 @@ diff --git a/src/host/groups.js b/src/host/groups.js +// that requires the line is (#1253), not because the room is assumed to speak Chinese. +const CLAIM_CUE_REF_MAX = 60 +const CLAIM_CUE_STEP_MAX = 100 -+const claimLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) ++const claimLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text +export function claimCueText(task) { + const title = claimLine(task?.title, 300) + if (title === '') return '' diff --git a/patches/ds-bot/0014-flybot-branding.patch b/patches/ds-bot/0014-flybot-branding.patch index de5c72156..06a38abcc 100644 --- a/patches/ds-bot/0014-flybot-branding.patch +++ b/patches/ds-bot/0014-flybot-branding.patch @@ -524,16 +524,17 @@ diff --git a/src/client/settings.js b/src/client/settings.js diff --git a/src/host/browser.js b/src/host/browser.js --- a/src/host/browser.js +++ b/src/host/browser.js -@@ -3,7 +3,7 @@ +@@ -3,7 +3,8 @@ // the app window, not here, so `read_browser` reaches it through the window: the client // holds a long poll open (`browser-wait`) that reports which Bots have a page open and // carries read requests back; the client answers with `browser-result`. -import { defineTool } from '@deepseek-ai/dsh-tools' ++import { cut } from './text.js' +import { defineTool } from './tool-schema.js' import { output } from './tools.js' // A waiting client counts as present; one that has not polled for this long is gone. -@@ -124,7 +124,7 @@ export function createBrowserChannel({ now = Date.now, waitMs = WAIT_MS, staleMs +@@ -124,14 +124,14 @@ export function createBrowserChannel({ now = Date.now, waitMs = WAIT_MS, staleMs function dispose() { for (const client of clients.values()) client.end?.() @@ -542,6 +543,14 @@ diff --git a/src/host/browser.js b/src/host/browser.js clients.clear() } + return { wait, result, read, openPages, dispose } + } + +-const clean = (value, max) => String(value ?? '').replace(/\s+/g, ' ').trim().slice(0, max) ++const clean = (value, max) => cut(String(value ?? '').replace(/\s+/g, ' ').trim(), max).text + + /** + * Where a part of the text that starts at `start` ends: at a blank line, else a line break, diff --git a/src/host/dsh-update.js b/src/host/dsh-update.js --- a/src/host/dsh-update.js +++ b/src/host/dsh-update.js diff --git a/patches/ds-bot/files/src/host/decisions.js b/patches/ds-bot/files/src/host/decisions.js index d5bb479b8..5c3ff2fc1 100644 --- a/patches/ds-bot/files/src/host/decisions.js +++ b/patches/ds-bot/files/src/host/decisions.js @@ -10,6 +10,7 @@ // can show it without reading the file. import { appendFile, mkdir, open, readFile, rename, stat, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' +import { cut } from './text.js' import { defineTool } from './tool-schema.js' import { looksSecret } from './memory.js' import { output } from './tools.js' @@ -23,7 +24,7 @@ const KEPT = 2_000 const MAX_BYTES = 1_000_000 const NAME_SAFE = /^[\w-]{1,128}$/ -const oneLine = (text, max) => String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) +const oneLine = (text, max) => cut(String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim(), max).text export function install(rt) { if (rt.config?.decisions !== true) { diff --git a/patches/ds-bot/files/src/host/flybot-feed.js b/patches/ds-bot/files/src/host/flybot-feed.js index 0e38a6b2d..1b6af294f 100644 --- a/patches/ds-bot/files/src/host/flybot-feed.js +++ b/patches/ds-bot/files/src/host/flybot-feed.js @@ -8,6 +8,7 @@ // user pasted or a tool printed. Strings that came from outside the team (a tool name // from an MCP server, a remote handle, an error code) are cleaned here. import { SOURCE_KIND } from './constants.js' +import { cut } from './text.js' import { normRemoteHandle, remoteBotTitle } from '../shared/remote-label.js' export const FEED_EVENTS = Object.freeze(['turn.start', 'turn.end', 'tool.start', 'tool.end', 'exchange.delivered', 'decision']) @@ -22,7 +23,7 @@ const CONTROLS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g export const cleanLabel = (value, max = 80) => { let out = String(value ?? '').replace(CONTROLS, ' ') for (let previous; previous !== out;) { previous = out; out = out.replace(/<\s*\/?\s*untrusted-?/gi, '') } - return out.replace(/\s+/g, ' ').trim().slice(0, max) + return cut(out.replace(/\s+/g, ' ').trim(), max).text } // An error code is an enumerated value (RATE_LIMIT, AUTH): upper-case words only, so free text or a key put in the field is not kept. const cleanCode = code => (typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,39}$/.test(code) ? code : 'UNKNOWN') diff --git a/patches/ds-bot/files/src/host/meetings.js b/patches/ds-bot/files/src/host/meetings.js index 38b8efb55..a6eed6405 100644 --- a/patches/ds-bot/files/src/host/meetings.js +++ b/patches/ds-bot/files/src/host/meetings.js @@ -8,6 +8,7 @@ import { createHash, randomBytes } from 'node:crypto' import { link, mkdir, open, readFile, rm } from 'node:fs/promises' import { join } from 'node:path' +import { cut } from './text.js' import { defineTool } from './tool-schema.js' import { output } from './tools.js' import { workDirFor } from './project-state.js' @@ -23,7 +24,7 @@ const NAME_MAX = 80 const PARTICIPANTS_MAX = 32 const OPEN_STATES = new Set(['open', 'deliberating']) -const oneLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max) +const oneLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text // A slug keeps letters and digits of every script, so two different Chinese questions do not both // fall back to `meeting` and collide on one day's file. const slugOf = value => oneLine(value, 80).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '').slice(0, 80) diff --git a/patches/ds-bot/files/test/decisions.test.js b/patches/ds-bot/files/test/decisions.test.js index dac8effd7..aefdbd2f3 100644 --- a/patches/ds-bot/files/test/decisions.test.js +++ b/patches/ds-bot/files/test/decisions.test.js @@ -84,3 +84,16 @@ test('#603: a byte cap bounds an oversized log left by an older build', async (t assert.equal(log.at(-1).option, 'after the byte cap') assert.equal(log.length, 2) }) + +test('#1912: an option cut at an emoji boundary stays well formed in the log', async (t) => { + const { home, tool, stop } = await boot({ decisions: true }) + t.after(stop) + // The 200-unit cut lands inside the emoji, so a code-unit slice stores half a surrogate pair. + const option = `${'a'.repeat(199)}πŸ˜€tail` + await tool('record_decision', { option, choice: 'used' }) + const text = readFileSync(join(home, 'decisions.jsonl'), 'utf8') + const [entry] = text.trim().split('\n').map(line => JSON.parse(line)) + assert.equal(entry.option.isWellFormed(), true, `the stored option must stay well formed, got: ${JSON.stringify(entry.option.slice(-4))}`) + assert.equal(entry.option.length <= 200, true, `the stored option must stay within 200 units, got ${entry.option.length}`) + assert.equal(option.startsWith(entry.option), true, 'the stored option must be a prefix of the recorded one') +}) diff --git a/patches/ds-bot/files/test/meetings-persist.test.js b/patches/ds-bot/files/test/meetings-persist.test.js index aae4d3fed..be294e6ca 100644 --- a/patches/ds-bot/files/test/meetings-persist.test.js +++ b/patches/ds-bot/files/test/meetings-persist.test.js @@ -3,7 +3,7 @@ // second decide is refused, with no second decision file and no second card), and an open meeting // keeps its recorded positions. import assert from 'node:assert/strict' -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' @@ -85,3 +85,22 @@ test('P-209: an open meeting and a recorded position survive a restart', async ( assert.match(read, /\[deliberating\]/, `an open meeting must survive a restart, got: ${read.split('\n')[0]}`) assert.match(read, /- Chief: agree β€” the existing store/, `the position must survive, got: ${read}`) }) + +test('#1912: a decision cut at an emoji boundary writes no replacement character into the archive', async (t) => { + const { dir, meeting } = await setup(t) + const text = await meeting({ action: 'open', question: 'Which storage shape should Stage 4 use?', participants: ['Chief', 'Dev', 'Reviewer'] }) + const id = text.split(' ')[0] + await positionAll(meeting, id) + // The 600-unit cut lands inside the emoji, so a code-unit slice keeps half of a surrogate pair. + const decision = `${'a'.repeat(599)}πŸ˜€tail` + await meeting({ action: 'decide', meeting: id, owner: 'Chief', decision, evidence: [{ pr: 1912 }] }) + const [file] = readdirSync(join(dir, 'decisions')).filter(name => name.endsWith('.md')) + const bytes = readFileSync(join(dir, 'decisions', file)) + const written = bytes.toString('utf8') + assert.equal(written.includes('\ufffd'), false, 'the archive must not hold U+FFFD') + assert.equal(Buffer.from(written, 'utf8').equals(bytes), true, 'the archive must round-trip through UTF-8 unchanged') + const lines = written.split('\n') + const kept = lines[lines.indexOf('## Decision') + 2] + assert.equal(decision.startsWith(kept), true, `the kept body must be a prefix of the decision, got: ${JSON.stringify(kept)}`) + assert.equal(kept.length <= 600, true, `the kept body must stay within 600 units, got ${kept.length}`) +})