From 7bef17d7d8efcd30f71290f36dd36cc6a6feaf0b Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:34:38 +0800 Subject: [PATCH] host: cut the last code-unit bounds on a code point boundary (#1916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight rows (nine cuts) still used slice(0, max) on text that can carry an emoji: the claim cue's fallback reference, the task lead's name in a member prompt, a failed group Session's error text, read_own_chat's query and anchor, an open page's address and title, a diagnostics browser-log line, and cleanPeerName. Each now goes through cut(), so the prefix stops before a split surrogate pair. shared/remote-label.js cannot import host/text.js, so the boundary rule moved to shared/text-boundary.js and host/text.js builds cut on it: one implementation for both layers. The browser test joins patch 0014 beside browser.js; 0001/0007/0008/0012/0014 are regenerated and the new shared file and client bundle are in files/. Docs-not-needed: core/client.js is regenerated only — the change reaches it from shared/remote-label.js (a bundle input), and no core/src/client file changed. --- changelog.d/1916-code-unit-cuts.fixed.md | 1 + core/client.js | 12 +- core/src/host/browser.js | 2 +- core/src/host/diagnostics.js | 4 +- core/src/host/groups.js | 10 +- core/src/host/text.js | 19 +-- core/src/shared/remote-label.js | 6 +- core/src/shared/text-boundary.js | 16 +++ core/test/browser-channel.test.js | 26 +++- core/test/claim-cue.test.js | 19 ++- core/test/diagnostics.test.js | 17 +++ core/test/group-bot-label.test.js | 8 ++ core/test/group-work.test.js | 56 +++++++- docs/subsystems/core-host.md | 4 +- patches/ds-bot/0001-core-fixes.patch | 123 +++++++++++++++--- patches/ds-bot/0007-version-and-tooling.patch | 24 ++++ patches/ds-bot/0008-client-bundle.patch | 49 ++++--- patches/ds-bot/0012-room-membership.patch | 24 +++- patches/ds-bot/0014-flybot-branding.patch | 48 +++++++ .../ds-bot/files/src/shared/remote-label.js | 6 +- .../ds-bot/files/src/shared/text-boundary.js | 16 +++ patches/ds-bot/files/test/claim-cue.test.js | 19 ++- .../ds-bot/files/test/group-bot-label.test.js | 8 ++ patches/ds-bot/topics.json | 3 +- 24 files changed, 444 insertions(+), 76 deletions(-) create mode 100644 changelog.d/1916-code-unit-cuts.fixed.md create mode 100644 core/src/shared/text-boundary.js create mode 100644 patches/ds-bot/files/src/shared/text-boundary.js diff --git a/changelog.d/1916-code-unit-cuts.fixed.md b/changelog.d/1916-code-unit-cuts.fixed.md new file mode 100644 index 000000000..217fc9f74 --- /dev/null +++ b/changelog.d/1916-code-unit-cuts.fixed.md @@ -0,0 +1 @@ +Eight more places stopped storing or showing half of an emoji ([#1916](https://github.com/ThinkFlowLab/FlyBot/issues/1916)): the claim cue's fallback reference, the task lead's name in a member's prompt, a failed group Session's error text, `read_own_chat`'s query and anchor, an open page's address and title, a diagnostics browser-log line, and a friend's Bot name now cut on a code point boundary. The rule itself moved to `core/src/shared/text-boundary.js`, so the shared label code and the host's `cut` use one implementation instead of two. diff --git a/core/client.js b/core/client.js index aaa92232d..279ed1ec5 100644 --- a/core/client.js +++ b/core/client.js @@ -1718,10 +1718,18 @@ var call = async (endpoint, payload, signal) => { return result.value; }; +// src/shared/text-boundary.js +var isHighSurrogate = (code) => code >= 55296 && code <= 56319; +var isLowSurrogate = (code) => code >= 56320 && code <= 57343; +var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; +var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; +var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max }); + // src/shared/remote-label.js function cleanPeerName(name, fallback = "") { - const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim().slice(0, 60); - return text2 || fallback; + const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim(); + const label = cut(text2, 60).text; + return label || fallback; } function normRemoteHandle(handle) { return `@${String(handle ?? "").trim().replace(/^@+/, "").toLowerCase()}`; diff --git a/core/src/host/browser.js b/core/src/host/browser.js index c12a9afa4..1347f2fc0 100644 --- a/core/src/host/browser.js +++ b/core/src/host/browser.js @@ -251,7 +251,7 @@ export function install(rt) { if (value === null || typeof value !== 'object') return open for (const [botId, page] of Object.entries(value).slice(0, OPEN_MAX)) { if (rt.botOf(botId) === undefined || typeof page?.url !== 'string') continue - open[botId] = { url: page.url.slice(0, 2000), title: String(page.title ?? '').slice(0, 300) } + open[botId] = { url: cut(page.url, 2000).text, title: cut(String(page.title ?? ''), 300).text } } return open } diff --git a/core/src/host/diagnostics.js b/core/src/host/diagnostics.js index c2daf6dff..f8a9215c8 100644 --- a/core/src/host/diagnostics.js +++ b/core/src/host/diagnostics.js @@ -208,7 +208,7 @@ export function install(rt) { } const sanitizeClient = (client) => { - const pick = (value, max) => (typeof value === 'string' ? clean(value.slice(0, max)) : undefined) + const pick = (value, max) => (typeof value === 'string' ? clean(cut(value, max).text) : undefined) return { surface: pick(client?.surface, 40), language: pick(client?.language, 20), @@ -217,7 +217,7 @@ export function install(rt) { } const sanitizeClientEntries = list => (Array.isArray(list) ? list : []).slice(-20).flatMap((item) => { if (typeof item?.text !== 'string' || typeof item.time !== 'number') return [] - return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(item.text.slice(0, 500)), ...(typeof item.code === 'string' ? { code: clean(item.code.slice(0, 40)) } : {}) }] + return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(cut(item.text, 500).text), ...(typeof item.code === 'string' ? { code: clean(cut(item.code, 40).text) } : {}) }] }) async function report(body = {}) { diff --git a/core/src/host/groups.js b/core/src/host/groups.js index 76eefe12c..f31897ee7 100644 --- a/core/src/host/groups.js +++ b/core/src/host/groups.js @@ -48,7 +48,7 @@ export function claimCueText(task) { const title = claimLine(task?.title, 300) if (title === '') return '' const found = title.match(/^#(\d{1,7})\b/) ?? title.match(/#(\d{1,7})\b/) - const ref = found ? `#${found[1]}` : title.slice(0, CLAIM_CUE_REF_MAX) + const ref = found ? `#${found[1]}` : cut(title, CLAIM_CUE_REF_MAX).text const rest = found ? title.slice(title.indexOf(found[0]) + found[0].length).trim() : '' const step = claimLine(task?.note, CLAIM_CUE_STEP_MAX) || claimLine(rest, CLAIM_CUE_STEP_MAX) return step === '' ? `接了 ${ref}` : `接了 ${ref},第一步是 ${step}` @@ -327,7 +327,7 @@ export function install(rt) { if (leadId === member.id) { return 'You are this room\'s task lead (总负责人) for the board. When the group is given a job, assign it on the board with track_tasks: one overall task, and a subtask for each part (assignee is that Bot\'s name). Nobody else plans. A chat line does not assign. A card you assign is claimed: that Bot starts the work in its own chat and does not wait for a confirmation. Progress is that card\'s column and progress. A question a person has to decide is one short post in this group; do not @ a Bot to hand the work on.' } - const name = String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' ').slice(0, 80) + const name = cut(String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' '), 80).text return `${name} is this room's task lead (总负责人) for the board: it assigns the parts on the board. Claim the card it assigns you by setting its column to in-progress with track_tasks, and update that card's column and progress as the work moves. Do not record a new plan, assign work, or post a status in the group. @Name only wakes someone; it does not assign. Start the work in your own chat; do not wait for a confirmation. A question a person has to decide is one short post in this group; do not @ a Bot to hand the work on.` } @@ -719,7 +719,7 @@ export function install(rt) { round.speaking = null slot.status = 'failed' round.failed.push(member.name) - round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: String(error?.message ?? error).slice(0, 300), time: Date.now() }) + round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: cut(String(error?.message ?? error), 300).text, time: Date.now() }) continue } const prompt = roomPrompt(room, member, round) @@ -961,8 +961,8 @@ export function install(rt) { if (botId === undefined) return 'Only Bots on the team can use read_own_chat.' const member = rt.botOf(botId) const room = rt.roomOf(owner?.roomId) - const query = String(args.query ?? '').trim().slice(0, 80) - const around = String(args.around ?? '').trim().slice(0, 40) + const query = cut(String(args.query ?? '').trim(), 80).text + const around = cut(String(args.around ?? '').trim(), 40).text const parts = rt.partsOf(botId) const number = Number.isInteger(args.part) ? args.part : undefined if (room !== undefined && member !== undefined) { diff --git a/core/src/host/text.js b/core/src/host/text.js index 5b109c849..c2f5e7878 100644 --- a/core/src/host/text.js +++ b/core/src/host/text.js @@ -1,6 +1,9 @@ // Pure text helpers shared by the host modules: message text, names, model refs, // cutting and estimating text, and the clock format of chat anchors. import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js' +import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js' + +export { cut } // A color id saved under its legacy name reads as the current one. export const currentColor = color => (typeof color === 'string' && Object.hasOwn(LEGACY_COLORS, color) ? LEGACY_COLORS[color] : color) @@ -34,18 +37,10 @@ export const codePoints = (text) => { return n } -const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff -const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff - -// A cut moves back onto a code point boundary: half of a surrogate pair reaches a UTF-8 encoder as -// U+FFFD, so a clipped emoji would show up as a replacement character at every exit (#1642 for the -// client side, here for the host). -const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) -const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) - -// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own -// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped. -export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) +// A cut moves back onto a code point boundary (#1642 client side, #1912 here, #1916 shared): half of +// a surrogate pair reaches a UTF-8 encoder as U+FFFD, so a clipped emoji shows up as a replacement +// character at every exit. The boundary rule itself lives in `shared/text-boundary.js`, because a +// shared label (`shared/remote-label.js`) bounds a peer name too and must not import this module. // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { diff --git a/core/src/shared/remote-label.js b/core/src/shared/remote-label.js index 7312732e0..f229208c2 100644 --- a/core/src/shared/remote-label.js +++ b/core/src/shared/remote-label.js @@ -1,10 +1,12 @@ // Display labels for a friend's published Bot and for a room the owner did not start. // Peer-controlled strings (names, homes, handles) are untrusted: strip controls and // markup so a name cannot look like instructions when it later reaches a model. +import { cut } from './text-boundary.js' export function cleanPeerName(name, fallback = '') { - const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim().slice(0, 60) - return text || fallback + const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim() + const label = cut(text, 60).text + return label || fallback } export function normRemoteHandle(handle) { diff --git a/core/src/shared/text-boundary.js b/core/src/shared/text-boundary.js new file mode 100644 index 000000000..af6579e24 --- /dev/null +++ b/core/src/shared/text-boundary.js @@ -0,0 +1,16 @@ +// Where a bounded prefix may end. Both sides bound text (#1908/#1912/#1916): the host cuts what a +// Bot reads back and what it stores, and a shared label cuts a peer's name. A cut that lands between +// the halves of a surrogate pair reaches a UTF-8 encoder as U+FFFD, so every bound goes through here +// instead of `slice`, and `host/text.js` builds its `cut` on this one implementation. + +const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff +const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff + +// The start of a bounded window, moved back onto a code point boundary. +export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) + +// The end of a bounded prefix, moved back onto a code point boundary. +export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) + +// A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. +export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) diff --git a/core/test/browser-channel.test.js b/core/test/browser-channel.test.js index e49ae0ef1..0ee67cbe0 100644 --- a/core/test/browser-channel.test.js +++ b/core/test/browser-channel.test.js @@ -1,6 +1,6 @@ import { test } from 'node:test' import assert from 'node:assert/strict' -import { createBrowserChannel, pageText, partEnd } from '../src/host/browser.js' +import { createBrowserChannel, install, pageText, partEnd } from '../src/host/browser.js' const ids = items => items.map(item => item.id) @@ -141,3 +141,27 @@ test('partEnd prefers a blank line, then a line break, and cuts hard only withou assert.equal(partEnd('a\n' + 'b'.repeat(20), 0, 10), 10) assert.equal(partEnd('short', 0, 10), 5) }) + +// #1916: the open-page snapshot is what a Bot reads back from outside the team, and both bounds +// are code-unit ones: 2000 for an address and 300 for a title must not land inside an emoji. +test('the open-page snapshot never cuts a title or an address in half (#1916)', () => { + const registered = [] + const rt = { + ctx: { get: () => ({ name: 'desktop' }), effect: () => {}, tools: { register: (tool) => registered.push(tool) } }, + config: {}, + botOf: () => ({ id: 'a' }), + } + install(rt) + const open = rt.browserOpen({ + a: { url: `${'a'.repeat(1999)}\u{1F600}tail`, title: `${'b'.repeat(299)}\u{1F600}tail` }, + }) + assert.equal(open.a.url.isWellFormed(), true) + assert.equal(open.a.title.isWellFormed(), true) + assert.equal(open.a.url, 'a'.repeat(1999)) + assert.equal(open.a.title, 'b'.repeat(299)) + // A cut that lands on a whole character keeps it. + const roomy = rt.browserOpen({ a: { url: `${'a'.repeat(1998)}\u{1F600}`, title: `${'b'.repeat(298)}\u{1F600}` } }) + assert.equal(roomy.a.url, `${'a'.repeat(1998)}\u{1F600}`) + assert.equal(roomy.a.title, `${'b'.repeat(298)}\u{1F600}`) + assert.equal(registered.length > 0, true) +}) diff --git a/core/test/claim-cue.test.js b/core/test/claim-cue.test.js index 0aa31cf61..feabe4a11 100644 --- a/core/test/claim-cue.test.js +++ b/core/test/claim-cue.test.js @@ -10,7 +10,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' import { createRuntime } from '../index.js' -import { CLAIM_CUE_REPEAT_MS } from '../src/host/groups.js' +import { CLAIM_CUE_REPEAT_MS, claimCueText } from '../src/host/groups.js' import { fakeContext } from './fake-ctx.js' const team = () => ({ @@ -180,3 +180,20 @@ test('a claim outside a room posts no line (#1253)', async (t) => { test('the repeat window is a stated number of minutes, not forever (#1253)', () => { assert.equal(CLAIM_CUE_REPEAT_MS, 30 * 60 * 1000) }) + +// #1916: the reference of a title with no issue number is a bounded prefix, and the bound is a +// code-unit one: 60 must not land between the halves of an emoji, or the room notice and the +// room log both keep an ill-formed string. +const utf8Prefix = (whole, part) => Buffer.from(part, 'utf8').equals(Buffer.from(whole, 'utf8').subarray(0, Buffer.byteLength(part))) + +test('a claim cue reference cut at 60 never splits a surrogate pair (#1916)', () => { + const title = `${'a'.repeat(59)}\u{1F600}tail` + const ref = claimCueText({ title }).replace(/^接了 /, '') + assert.equal(ref.isWellFormed(), true) + assert.equal(ref.includes('\uFFFD'), false) + assert.ok(ref.length <= 60, `ref is ${ref.length} units`) + assert.equal(utf8Prefix(title, ref), true, 'the ref is a byte prefix of the title') + // A cut that lands on a whole character keeps the whole character. + const roomy = claimCueText({ title: `${'a'.repeat(58)}\u{1F600}tail` }).replace(/^接了 /, '') + assert.equal(roomy, `${'a'.repeat(58)}\u{1F600}`) +}) diff --git a/core/test/diagnostics.test.js b/core/test/diagnostics.test.js index ed54d6ebe..231934cdb 100644 --- a/core/test/diagnostics.test.js +++ b/core/test/diagnostics.test.js @@ -92,6 +92,23 @@ test('warnings are kept, cleaned, written to the log, and read back after a rest quiet.mock.restore() }) +// #1916: the browser-log lines in the report are bounded, and those bounds are code-unit ones: +// a line cut inside an emoji is stored and shown as U+FFFD in a report meant for a public issue. +test('a browser-log line cut at its bound never splits a surrogate pair (#1916)', async (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 })) + const rt = host(home, { config: { diagnosticsFile: false } }) + const report = await rt.diagnostics.report({ + clientErrors: [{ time: 1, text: `${'x'.repeat(499)}\u{1F600}tail`, code: `${'y'.repeat(39)}\u{1F600}tail` }], + }) + const line = report.markdown.split('\n').find(item => item.includes('ERROR browser')) ?? '' + assert.notEqual(line, '') + assert.equal(line.isWellFormed(), true) + assert.equal(line.includes('\ufffd'), false) + assert.ok(line.includes('x'.repeat(499)), 'the text keeps its whole-character prefix') + assert.ok(line.includes(`[${'y'.repeat(39)}]`), `expected the code prefix, got ${line}`) +}) + test('warnings send only cleaned text to the logger and stderr', (t) => { const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-')) t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 })) diff --git a/core/test/group-bot-label.test.js b/core/test/group-bot-label.test.js index 08ae245fd..f9c37ff94 100644 --- a/core/test/group-bot-label.test.js +++ b/core/test/group-bot-label.test.js @@ -117,3 +117,11 @@ test('#526: real names stay as they are', () => { const view = roster([seatMine, { ...reviewer, label: undefined }], 'group') assert.equal(remoteSpeaker({ remote: { handle }, senderName: 'Reviewer' }, view, 'r').name, 'Reviewer (flybot682b8a43)') }) + +// #1916: a friend's Bot or room name reaches a display label, and its 60-unit bound must not land +// inside an emoji: the half pair would show as U+FFFD wherever the label is drawn. +test('a peer name cut at 60 never splits a surrogate pair (#1916)', () => { + assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`), 'b'.repeat(59)) + assert.equal(labels.cleanPeerName(`${'b'.repeat(58)}\u{1F600}tail`), `${'b'.repeat(58)}\u{1F600}`) + assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`, 'fallback').isWellFormed(), true) +}) diff --git a/core/test/group-work.test.js b/core/test/group-work.test.js index 0be1eef2e..7bfaec18d 100644 --- a/core/test/group-work.test.js +++ b/core/test/group-work.test.js @@ -3,7 +3,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' -import { apply } from '../index.js' +import { createRuntime, provideFlyBot } from '../index.js' import { fakeContext } from './fake-ctx.js' import { ROOM_LATE_ROUNDS, ROOM_WORK_GRACE_MS } from '../src/host/constants.js' import { compactStatus, prepareRoomLine } from '../src/host/room-context.js' @@ -71,7 +71,8 @@ async function start(t, room, history = {}) { let route const register = ctx.connection.fetch.register ctx.connection.fetch.register = (options) => { route = options; return register(options) } - apply(ctx, { home }) + const rt = createRuntime(ctx, { home }) + provideFlyBot(ctx, rt.flybot) t.after(async () => { await ctx.dispose() rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) @@ -116,7 +117,7 @@ async function start(t, room, history = {}) { return { content: post.content, source: post.source } } await tool('list_bots', {}, 'a') - return { home, agentOf, emit, tool, call, begin, finish, round, say, takePost } + return { home, ctx, rt, agentOf, emit, tool, call, begin, finish, round, say, takePost } } const settle = async (check) => { @@ -466,6 +467,55 @@ test('a kind-carrying room line stays verbatim in the store (#1251)', async (t) assert.equal(prepareRoomLine(long), long) }) +// #1916: the lookup line goes into the room log and is also what the search matches, so a bound +// that lands inside an emoji would store an ill-formed string and make the character unsearchable. +const utf8Prefix = (whole, part) => Buffer.from(part, 'utf8').equals(Buffer.from(whole, 'utf8').subarray(0, Buffer.byteLength(part))) + +test('a read_own_chat query or anchor cut at its bound never splits a surrogate pair (#1916)', async (t) => { + const host = await start(t) + const log = () => JSON.parse(readFileSync(join(host.home, 'state.json'), 'utf8')).rooms.r1.log + const query = `${'a'.repeat(79)}\u{1F600}permalink` + await host.tool('read_own_chat', { query }, 'gb') + await settle(() => log().length === 1) + assert.equal(log()[0].text.isWellFormed(), true) + assert.ok(log()[0].text.length <= 80, `query is ${log()[0].text.length} units`) + assert.equal(utf8Prefix(query, log()[0].text), true, 'the stored query is a byte prefix of the input') + const around = `${'b'.repeat(39)}\u{1F600}anchor` + await host.tool('read_own_chat', { around }, 'gb') + await settle(() => log().length === 2) + assert.equal(log()[1].text.isWellFormed(), true) + assert.ok(log()[1].text.length <= `around ${'b'.repeat(39)}`.length, `anchor is ${log()[1].text.length} units`) + assert.equal(utf8Prefix(`around ${around}`, log()[1].text), true, 'the stored anchor is a byte prefix of the input') +}) + +// #1916: the task lead's name goes into every other member's system prompt, and a Bot name is the +// owner's to set, emoji included (#1182). +test('the task lead name in a member prompt never splits a surrogate pair (#1916)', async (t) => { + const host = await start(t, { taskLead: 'a' }) + const name = `${'l'.repeat(79)}\u{1F600}tail` + host.rt.state.bots.a.name = name + const prompt = host.ctx.sections.get('bot-room').text({ agent: { id: 'gb' } }) + assert.equal(prompt.isWellFormed(), true, 'the room prompt is well formed') + assert.equal(prompt.includes('\uFFFD'), false) + assert.ok(prompt.includes('l'.repeat(79)), 'the whole-character prefix is kept') +}) + +// #1916: a group Session that cannot be opened reports its error into the room (#1253 notice path), +// so the 300-unit bound on that text must not land inside an emoji. +test('a failed group Session reports its error without splitting a surrogate pair (#1916)', async (t) => { + const host = await start(t) + // No Session for Writer yet: the next round has to open one, and opening it fails. + delete host.rt.state.rooms.r1.sessions.b + host.ctx.sessionController.create = async () => { throw new Error(`${'e'.repeat(299)}\u{1F600}tail`) } + const result = host.round('c1', host.say('@Writer 汇报')) + await settle(() => result() !== undefined) + const failed = result().replies.find(reply => reply.kind === 'failed') + assert.ok(failed, `no failed reply: ${JSON.stringify(result())}`) + assert.equal(failed.text.isWellFormed(), true) + assert.equal(failed.text.includes('\uFFFD'), false) + assert.ok(failed.text.length <= 300, `text is ${failed.text.length} units`) +}) + // #1906: a search shows its best 20 hits and a bare read the latest 12, and neither said // how much it left out, so a user asking "didn't we decide X?" could think the deciding // message was gone. The caps stay; the answer says what it did not show. diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 7602d4ca4..8e6f6b69b 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` · 's Bot`, ` · `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | @@ -67,7 +67,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | `core/src/shared/workflow-run.js` | The running layer's pure half (#1316, #1272 step 4). `planJob({ definition, job, cards, facts, hold })` answers which node cards a job may create now: every decision is `enterNode`'s, so a node whose input is missing, stale or unreadable opens nothing and its refusal is that function's own sentence byte for byte, and a node that already has a card is in neither list — a card parented to another job is not this job's card, so `cardsOfJob` is what makes `(job, node)` one card across a replayed tick or a restarted host, even when a caller hands in the whole board. An open entry is the card the board already places (`parentId` the job card, `workflow`, `node`) plus the fields the pull queue reads (`role`, `queueRole`, `assignee: null`), so the hand-off is the claim and never an `@`. A held job opens nothing on any node, in the definition's own sentence: the planner renders that definition's `no-hold` guard (`pr` equal to `none`, refused for `hold`) through `renderRefusal`, which is the merge gate's wording byte for byte, and a definition guarding no held pull request gets `refusal: null` and the holder's name instead of a sentence this layer wrote. `jobDone` answers pin 6 — a job is done when its terminal node's outputs are present and current, never because a card moved — and `completions` is step (c)'s pure half: a done card whose declared outputs are current, append-only, an entry already in `history` never returned again. No fs, no clock, no timer, no network, no host import: `task-drive.js` `driveOnce` stays the only beat. Tests: `core/test/workflow-run.test.js`. | | `core/src/host/workflow-store.js` | Where a definition lives (#1272 step 3): the presets read from their modules plus the owner's `workflows/.json` in the workspace, and the only writer. `save` validates with the engine's own `validateWorkflow` (the roster's roles are the team; an empty roster is no team to check), refuses a preset id (`workflow/preset-read-only` — *Customize* is the only way to start from one), writes `workflow-doc.js`'s bytes only when they changed, and answers `changed: false` for an unedited definition (acceptance 7). `customize`, `setDefault` (the project's `workflows/project.json`), `list`, `open` (definition + bytes + layout + preview) and `definitions()` (the presets plus the workspace files) round it out; `install` wires `rt.workflowEndpoint` for `workflows` (read) and `workflow-save` (write, in `authz.js`'s `TASK_ENDPOINTS`), so the client writes a file through the host and never to GitHub. A file that does not parse is skipped with one warning instead of breaking the list. Tests: `core/test/workflow-store.test.js`. | | `core/src/host/workflow-facts.js` | The fact providers (#1272 step 2, acceptance 10): one function per fact a guard asks about — `github.labels`, `github.checkAtSha`, `github.verdictAtTip`, `github.mergeable`, `github.prBodyTip`, `task.progress`, `manual.tick` — each taking the answer a reader already got and returning one artifact (spec, revision, value), never fetching. No `GITHUB_TOKEN`, no answer, or an answer missing the field is `{ ok: false, reason: 'unreadable', why }`, never a guess: a fact nobody could read refuses the node it feeds instead of passing it. `workflowFact(name, args)` is the one spelling for a caller that has the name as data, and an unknown name is unreadable with the name in the reason. Tests: `core/test/workflow-repo-dev.test.js`. | -| `core/src/shared/remote-label.js` | Owner-badge labels for a friend's Bot and “Shared room from …” titles. `friendBotTitle(bot)` / `friendBotOwner(bot)` name a friend's chat from the roster (`Chief (SelfB1233)`) and its owner (`SelfB1233`, the person an approval card names) with one formatter for every client surface (#498); `friendRecord` reads a published roster record (`roster.remoteBots`, name/owner/home at the top level) as well as a chat's `remote`, so the name does not change when the chat appears (#967). `remoteMentionLabel(record, others)` is the one name a mention wears — that title, and on a title two different handles share the home too (`Chief (Bob) · bob2.home`), so the menu, the bubble, the preview and the host's match cannot drift apart (#967). `remoteBotName` / `isAutoOwnerName` / `seatOwnerName` keep group and face-room labels in that format too (#526): Bot Social's ` · ` member label is not a Bot name, and an owner name made up from an install id (`FlyBot 682b8a43`) gives way to the username in the home handle (`Reviewer (flybot682b8a43)`). `isListedRemoteChat` / `listedChatBots` keep gone, hidden, and home-level (`@something.home`) remotes out of the **发给** picker, mentions, and `list_bots` friends lines: a share that ended is not a place to send to. The sidebar asks a different question through `isShownRemoteChat` (#956): a chat whose Bot left the published list keeps its row, marked *no longer shared*, while only a hide the user set (`bot.hidden`) takes a row out. History stays on disk either way. | +| `core/src/shared/remote-label.js` | Owner-badge labels for a friend's Bot and “Shared room from …” titles. `friendBotTitle(bot)` / `friendBotOwner(bot)` name a friend's chat from the roster (`Chief (SelfB1233)`) and its owner (`SelfB1233`, the person an approval card names) with one formatter for every client surface (#498); `friendRecord` reads a published roster record (`roster.remoteBots`, name/owner/home at the top level) as well as a chat's `remote`, so the name does not change when the chat appears (#967). `remoteMentionLabel(record, others)` is the one name a mention wears — that title, and on a title two different handles share the home too (`Chief (Bob) · bob2.home`), so the menu, the bubble, the preview and the host's match cannot drift apart (#967). `remoteBotName` / `isAutoOwnerName` / `seatOwnerName` keep group and face-room labels in that format too (#526): Bot Social's ` · ` member label is not a Bot name, and an owner name made up from an install id (`FlyBot 682b8a43`) gives way to the username in the home handle (`Reviewer (flybot682b8a43)`). `isListedRemoteChat` / `listedChatBots` keep gone, hidden, and home-level (`@something.home`) remotes out of the **发给** picker, mentions, and `list_bots` friends lines: a share that ended is not a place to send to. The sidebar asks a different question through `isShownRemoteChat` (#956): a chat whose Bot left the published list keeps its row, marked *no longer shared*, while only a hide the user set (`bot.hidden`) takes a row out. History stays on disk either way. Since #1916 `cleanPeerName` bounds a peer name through the shared `cut` (`shared/text-boundary.js`), so a 60-unit label never ends in half of an emoji. | ## Invariants diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index b344ae143..432ff6e67 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4525,7 +4525,21 @@ diff --git a/src/host/diagnostics.js b/src/host/diagnostics.js console.error(`[ds-bot] ${line}${frames.length > 0 ? `\n${clean(frames.join('\n'))}` : ''}`) note({ level: 'warn', -@@ -217,7 +217,7 @@ export function install(rt) { +@@ -204,7 +204,7 @@ export function install(rt) { + } + + const sanitizeClient = (client) => { +- const pick = (value, max) => (typeof value === 'string' ? clean(value.slice(0, max)) : undefined) ++ const pick = (value, max) => (typeof value === 'string' ? clean(cut(value, max).text) : undefined) + return { + surface: pick(client?.surface, 40), + language: pick(client?.language, 20), +@@ -213,11 +213,11 @@ export function install(rt) { + } + const sanitizeClientEntries = list => (Array.isArray(list) ? list : []).slice(-20).flatMap((item) => { + if (typeof item?.text !== 'string' || typeof item.time !== 'number') return [] +- return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(item.text.slice(0, 500)), ...(typeof item.code === 'string' ? { code: clean(item.code.slice(0, 40)) } : {}) }] ++ return [{ time: item.time, level: item.level === 'warn' ? 'warn' : 'error', source: 'browser', text: clean(cut(item.text, 500).text), ...(typeof item.code === 'string' ? { code: clean(cut(item.code, 40).text) } : {}) }] }) async function report(body = {}) { @@ -6292,7 +6306,17 @@ diff --git a/src/host/steward.js b/src/host/steward.js diff --git a/src/host/text.js b/src/host/text.js --- a/src/host/text.js +++ b/src/host/text.js -@@ -23,11 +23,36 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'“”「 +@@ -1,6 +1,9 @@ + // Pure text helpers shared by the host modules: message text, names, model refs, + // cutting and estimating text, and the clock format of chat anchors. + import { COLORS, EXTRA_COLORS, LEGACY_COLORS, OWN_CHAT_LINE_MAX } from './constants.js' ++import { codePointEnd, codePointStart, cut } from '../shared/text-boundary.js' ++ ++export { cut } + + // A color id saved under its legacy name reads as the current one. + export const currentColor = color => (typeof color === 'string' && Object.hasOwn(LEGACY_COLORS, color) ? LEGACY_COLORS[color] : color) +@@ -23,11 +23,28 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'“”「 export const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -6307,18 +6331,10 @@ diff --git a/src/host/text.js b/src/host/text.js + return n +} + -+const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff -+const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff -+ -+// A cut moves back onto a code point boundary: half of a surrogate pair reaches a UTF-8 encoder as -+// U+FFFD, so a clipped emoji would show up as a replacement character at every exit (#1642 for the -+// client side, here for the host). -+const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) -+const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) -+ -+// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own -+// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped. -+export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) ++// A cut moves back onto a code point boundary (#1642 client side, #1912 here, #1916 shared): half of ++// a surrogate pair reaches a UTF-8 encoder as U+FFFD, so a clipped emoji shows up as a replacement ++// character at every exit. The boundary rule itself lives in `shared/text-boundary.js`, because a ++// shared label (`shared/remote-label.js`) bounds a peer name too and must not import this module. + // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { @@ -9343,8 +9359,12 @@ diff --git a/test/group-status.test.js b/test/group-status.test.js diff --git a/test/group-work.test.js b/test/group-work.test.js --- a/test/group-work.test.js +++ b/test/group-work.test.js -@@ -6,6 +6,7 @@ import { test } from 'node:test' - import { apply } from '../index.js' +@@ -3,9 +3,10 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' + import { tmpdir } from 'node:os' + import { join } from 'node:path' + import { test } from 'node:test' +-import { apply } from '../index.js' ++import { createRuntime, provideFlyBot } from '../index.js' import { fakeContext } from './fake-ctx.js' import { ROOM_LATE_ROUNDS, ROOM_WORK_GRACE_MS } from '../src/host/constants.js' +import { compactStatus, prepareRoomLine } from '../src/host/room-context.js' @@ -9360,7 +9380,7 @@ diff --git a/test/group-work.test.js b/test/group-work.test.js const home = mkdtempSync(join(tmpdir(), 'ds-bot-work-')) writeFileSync(join(home, 'state.json'), JSON.stringify(seed(room))) const ctx = fakeContext() -@@ -46,14 +46,24 @@ async function start(t, room) { +@@ -46,21 +46,32 @@ async function start(t, room) { }, } const keep = (message) => { calls.push(message); held.push(message) } @@ -9387,6 +9407,24 @@ diff --git a/test/group-work.test.js b/test/group-work.test.js sessionTitle: { rename: async () => {} }, sessions: { flush: async () => {} }, }) + let route + const register = ctx.connection.fetch.register + ctx.connection.fetch.register = (options) => { route = options; return register(options) } +- apply(ctx, { home }) ++ const rt = createRuntime(ctx, { home }) ++ provideFlyBot(ctx, rt.flybot) + t.after(async () => { + await ctx.dispose() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) +@@ -105,7 +105,7 @@ async function start(t, room) { + return { content: post.content, source: post.source } + } + await tool('list_bots', {}, 'a') +- return { home, agentOf, emit, tool, call, begin, finish, round, say, takePost } ++ return { home, ctx, rt, agentOf, emit, tool, call, begin, finish, round, say, takePost } + } + + const settle = async (check) => { @@ -113,6 +113,7 @@ const settle = async (check) => { while (!check() && Date.now() < until) await new Promise(resolve => setImmediate(resolve)) assert.ok(check(), 'condition not reached') @@ -9395,7 +9433,7 @@ diff --git a/test/group-work.test.js b/test/group-work.test.js const text = message => message.content.map(block => block.text ?? '').join('\n') const kinds = result => result.replies.map(reply => [reply.kind ?? 'message', reply.name]) -@@ -287,3 +287,205 @@ test('members still at work show in the team view and in the prompts of the othe +@@ -287,3 +287,254 @@ test('members still at work show in the team view and in the prompts of the othe host.finish('gc', '没问题。') await settle(() => result() !== undefined) }) @@ -9566,6 +9604,55 @@ diff --git a/test/group-work.test.js b/test/group-work.test.js + assert.equal(prepareRoomLine(long), long) +}) + ++// #1916: the lookup line goes into the room log and is also what the search matches, so a bound ++// that lands inside an emoji would store an ill-formed string and make the character unsearchable. ++const utf8Prefix = (whole, part) => Buffer.from(part, 'utf8').equals(Buffer.from(whole, 'utf8').subarray(0, Buffer.byteLength(part))) ++ ++test('a read_own_chat query or anchor cut at its bound never splits a surrogate pair (#1916)', async (t) => { ++ const host = await start(t) ++ const log = () => JSON.parse(readFileSync(join(host.home, 'state.json'), 'utf8')).rooms.r1.log ++ const query = `${'a'.repeat(79)}\u{1F600}permalink` ++ await host.tool('read_own_chat', { query }, 'gb') ++ await settle(() => log().length === 1) ++ assert.equal(log()[0].text.isWellFormed(), true) ++ assert.ok(log()[0].text.length <= 80, `query is ${log()[0].text.length} units`) ++ assert.equal(utf8Prefix(query, log()[0].text), true, 'the stored query is a byte prefix of the input') ++ const around = `${'b'.repeat(39)}\u{1F600}anchor` ++ await host.tool('read_own_chat', { around }, 'gb') ++ await settle(() => log().length === 2) ++ assert.equal(log()[1].text.isWellFormed(), true) ++ assert.ok(log()[1].text.length <= `around ${'b'.repeat(39)}`.length, `anchor is ${log()[1].text.length} units`) ++ assert.equal(utf8Prefix(`around ${around}`, log()[1].text), true, 'the stored anchor is a byte prefix of the input') ++}) ++ ++// #1916: the task lead's name goes into every other member's system prompt, and a Bot name is the ++// owner's to set, emoji included (#1182). ++test('the task lead name in a member prompt never splits a surrogate pair (#1916)', async (t) => { ++ const host = await start(t, { taskLead: 'a' }) ++ const name = `${'l'.repeat(79)}\u{1F600}tail` ++ host.rt.state.bots.a.name = name ++ const prompt = host.ctx.sections.get('bot-room').text({ agent: { id: 'gb' } }) ++ assert.equal(prompt.isWellFormed(), true, 'the room prompt is well formed') ++ assert.equal(prompt.includes('\uFFFD'), false) ++ assert.ok(prompt.includes('l'.repeat(79)), 'the whole-character prefix is kept') ++}) ++ ++// #1916: a group Session that cannot be opened reports its error into the room (#1253 notice path), ++// so the 300-unit bound on that text must not land inside an emoji. ++test('a failed group Session reports its error without splitting a surrogate pair (#1916)', async (t) => { ++ const host = await start(t) ++ // No Session for Writer yet: the next round has to open one, and opening it fails. ++ delete host.rt.state.rooms.r1.sessions.b ++ host.ctx.sessionController.create = async () => { throw new Error(`${'e'.repeat(299)}\u{1F600}tail`) } ++ const result = host.round('c1', host.say('@Writer 汇报')) ++ await settle(() => result() !== undefined) ++ const failed = result().replies.find(reply => reply.kind === 'failed') ++ assert.ok(failed, `no failed reply: ${JSON.stringify(result())}`) ++ assert.equal(failed.text.isWellFormed(), true) ++ assert.equal(failed.text.includes('\uFFFD'), false) ++ assert.ok(failed.text.length <= 300, `text is ${failed.text.length} units`) ++}) ++ +// #1906: a search shows its best 20 hits and a bare read the latest 12, and neither said +// how much it left out, so a user asking "didn't we decide X?" could think the deciding +// message was gone. The caps stay; the answer says what it did not show. diff --git a/patches/ds-bot/0007-version-and-tooling.patch b/patches/ds-bot/0007-version-and-tooling.patch index c75e35449..2a97fa96b 100644 --- a/patches/ds-bot/0007-version-and-tooling.patch +++ b/patches/ds-bot/0007-version-and-tooling.patch @@ -834,6 +834,30 @@ diff --git a/test/diagnostics.test.js b/test/diagnostics.test.js const again = host(home) assert.deepEqual((await again.diagnostics.entries()).map(entry => entry.code), ['EACCES', 'RATE_LIMIT']) assert.match(await again.diagnostics.logText(), /\n {4}at TestContext/) +@@ -84,6 +84,23 @@ test('warnings are kept, cleaned, written to the log, and read back after a rest + quiet.mock.restore() + }) + ++// #1916: the browser-log lines in the report are bounded, and those bounds are code-unit ones: ++// a line cut inside an emoji is stored and shown as U+FFFD in a report meant for a public issue. ++test('a browser-log line cut at its bound never splits a surrogate pair (#1916)', async (t) => { ++ const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-')) ++ t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 })) ++ const rt = host(home, { config: { diagnosticsFile: false } }) ++ const report = await rt.diagnostics.report({ ++ clientErrors: [{ time: 1, text: `${'x'.repeat(499)}\u{1F600}tail`, code: `${'y'.repeat(39)}\u{1F600}tail` }], ++ }) ++ const line = report.markdown.split('\n').find(item => item.includes('ERROR browser')) ?? '' ++ assert.notEqual(line, '') ++ assert.equal(line.isWellFormed(), true) ++ assert.equal(line.includes('\ufffd'), false) ++ assert.ok(line.includes('x'.repeat(499)), 'the text keeps its whole-character prefix') ++ assert.ok(line.includes(`[${'y'.repeat(39)}]`), `expected the code prefix, got ${line}`) ++}) ++ + test('warnings send only cleaned text to the logger and stderr', (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 })) @@ -108,6 +108,23 @@ test('warnings send only cleaned text to the logger and stderr', (t) => { } }) diff --git a/patches/ds-bot/0008-client-bundle.patch b/patches/ds-bot/0008-client-bundle.patch index d15f8c9d2..65836960a 100644 --- a/patches/ds-bot/0008-client-bundle.patch +++ b/patches/ds-bot/0008-client-bundle.patch @@ -1104,14 +1104,22 @@ diff --git a/client.js b/client.js if (!REFUSALS.has(code)) { noteClientError(endpoint, Object.assign(new Error(result.error?.message ?? "Request failed"), { code })); markNoted(error); -@@ -1097,6 +1097,388 @@ var call = async (endpoint, payload, signal) => { +@@ -1097,6 +1097,396 @@ var call = async (endpoint, payload, signal) => { return result.value; }; ++// src/shared/text-boundary.js ++var isHighSurrogate = (code) => code >= 55296 && code <= 56319; ++var isLowSurrogate = (code) => code >= 56320 && code <= 57343; ++var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; ++var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; ++var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max }); ++ +// src/shared/remote-label.js +function cleanPeerName(name, fallback = "") { -+ const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim().slice(0, 60); -+ return text2 || fallback; ++ const text2 = String(name ?? "").replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ").replace(/[[\]"`<>]/g, "").replace(/\s+/g, " ").trim(); ++ const label = cut(text2, 60).text; ++ return label || fallback; +} +function normRemoteHandle(handle) { + return `@${String(handle ?? "").trim().replace(/^@+/, "").toLowerCase()}`; @@ -1760,16 +1768,7 @@ diff --git a/client.js b/client.js - else delete SHAPES[id]; - refresh(); - }; -+// src/client/cluster-order.js -+function clusterOrder(people2) { -+ const list = Array.isArray(people2) ? people2 : []; -+ const remote = (bot) => Boolean(bot?.remote); -+ if (!list.some(remote)) return list; -+ const locals = list.filter((bot) => !remote(bot)); -+ const remotes = list.filter(remote); -+ if (list.length <= 4) return [...remotes, ...locals]; -+ return [...locals, ...remotes]; - } +-} -function registerTheme(id, theme) { - if (typeof id !== "string" || !/^[a-z][\w-]{0,31}$/.test(id)) throw new Error("Theme ids are lowercase words"); - for (const mode of ["light", "dark"]) { @@ -1783,7 +1782,16 @@ diff --git a/client.js b/client.js - else delete customThemes[id]; - registry.set((value) => value + 1); - }; --} ++// src/client/cluster-order.js ++function clusterOrder(people2) { ++ const list = Array.isArray(people2) ? people2 : []; ++ const remote = (bot) => Boolean(bot?.remote); ++ if (!list.some(remote)) return list; ++ const locals = list.filter((bot) => !remote(bot)); ++ const remotes = list.filter(remote); ++ if (list.length <= 4) return [...remotes, ...locals]; ++ return [...locals, ...remotes]; + } -var allThemes = () => ({ ...THEMES, ...customThemes }); // src/shared/mentions.js @@ -1834,9 +1842,6 @@ diff --git a/client.js b/client.js } - -// src/client/text.js --var MENTION_ORIGIN = "https://bots.local/"; --function linkMentions(text, bots, selfId) { -- if (!text || !text.includes("@")) return text; +function unmatchedMentions(text2, bots, { references = [] } = {}) { + const value = String(text2 ?? ""); + const skip = [ @@ -1886,7 +1891,9 @@ diff --git a/client.js b/client.js +} + +// src/client/text.js -+var MENTION_ORIGIN = "https://bots.local/"; + var MENTION_ORIGIN = "https://bots.local/"; +-function linkMentions(text, bots, selfId) { +- if (!text || !text.includes("@")) return text; +function linkMentions(text2, bots, selfId) { + if (!text2 || !text2.includes("@")) return text2; + const ordered = [...bots ?? []].sort((a, b) => Number(Boolean(a?.remote)) - Number(Boolean(b?.remote))); @@ -2124,7 +2131,7 @@ diff --git a/client.js b/client.js var formatTokens = (count) => count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count); function usageOf(list, ids) { let input = 0; -@@ -1624,901 +1624,1981 @@ function usageOf(list, ids) { +@@ -1624,902 +1624,1982 @@ function usageOf(list, ids) { const hit = input > 0 ? Math.round(cacheRead / input * 100) : 0; return t("{tokens} tok · Cache hit {hit}%", { tokens: formatTokens(input + output), hit }); } @@ -3969,6 +3976,7 @@ diff --git a/client.js b/client.js -.bt-mem-summary>:nth-child(5){animation-delay:.12s} -.bt-mem-summary>:nth-child(6){animation-delay:.15s} -.bt-mem-summary>:nth-child(n+7){animation-delay:.18s} +-@keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} +// src/client/motion.js +var import_react4 = require("react"); +var reducedMotion = () => window.matchMedia?.("(prefers-reduced-motion: reduce)").matches === true; @@ -4970,9 +4978,10 @@ diff --git a/client.js b/client.js +.bt-mem-summary>:nth-child(5){animation-delay:.12s} +.bt-mem-summary>:nth-child(6){animation-delay:.15s} +.bt-mem-summary>:nth-child(n+7){animation-delay:.18s} - @keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} ++@keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} .bt-mem-skel{display:flex;flex-direction:column;gap:24px} .bt-mem-skel-block{display:flex;flex-direction:column;gap:10px} + .bt-mem-skel-block span{display:block;height:12px;border-radius:6px;background:linear-gradient(90deg,var(--bt-hover) 25%,var(--bt-active) 50%,var(--bt-hover) 75%);background-size:200% 100%;animation:bt-mem-skel 1.4s ease-in-out infinite} @@ -2574,2206 +2574,4679 @@ body[data-ds-dark-theme] .bt-theme-chip i{background:var(--da)} @media (max-width:640px){.bt-mem{width:100vw;height:100%;border-radius:0;border:0}.bt-mem-body{padding:18px 18px 20px}.bt-mem-foot{padding:8px 12px 12px}} @media (prefers-reduced-motion:reduce){.bt-mem,.bt-mem-scrim,.bt-mem-summary>*,.bt-mem-empty,.bt-mem-reply,.bt-mem-skel-block span,.bt-mem-entries,.bt-mem-send>*,.bt-mem-changes li svg{animation:none}} diff --git a/patches/ds-bot/0012-room-membership.patch b/patches/ds-bot/0012-room-membership.patch index e2bf5473a..07fd125d7 100644 --- a/patches/ds-bot/0012-room-membership.patch +++ b/patches/ds-bot/0012-room-membership.patch @@ -884,7 +884,7 @@ diff --git a/src/host/groups.js b/src/host/groups.js + const title = claimLine(task?.title, 300) + if (title === '') return '' + const found = title.match(/^#(\d{1,7})\b/) ?? title.match(/#(\d{1,7})\b/) -+ const ref = found ? `#${found[1]}` : title.slice(0, CLAIM_CUE_REF_MAX) ++ const ref = found ? `#${found[1]}` : cut(title, CLAIM_CUE_REF_MAX).text + const rest = found ? title.slice(title.indexOf(found[0]) + found[0].length).trim() : '' + const step = claimLine(task?.note, CLAIM_CUE_STEP_MAX) || claimLine(rest, CLAIM_CUE_STEP_MAX) + return step === '' ? `接了 ${ref}` : `接了 ${ref},第一步是 ${step}` @@ -1153,7 +1153,7 @@ diff --git a/src/host/groups.js b/src/host/groups.js - await rt.save() - await rt.ensureCapabilities(id) - return { id, brief: await rt.ownChatBrief(member) } -+ const name = String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' ').slice(0, 80) ++ const name = cut(String(rt.botOf(leadId)?.name ?? 'The task lead').replace(/[\r\n]+/g, ' '), 80).text + return `${name} is this room's task lead (总负责人) for the board: it assigns the parts on the board. Claim the card it assigns you by setting its column to in-progress with track_tasks, and update that card's column and progress as the work moves. Do not record a new plan, assign work, or post a status in the group. @Name only wakes someone; it does not assign. Start the work in your own chat; do not wait for a confirmation. A question a person has to decide is one short post in this group; do not @ a Bot to hand the work on.` } @@ -1603,7 +1603,14 @@ diff --git a/src/host/groups.js b/src/host/groups.js slot.status = 'speaking' round.speaking = member.id round.turns += 1 -@@ -339,8 +339,23 @@ export function install(rt) { +@@ -333,14 +333,29 @@ export function install(rt) { + round.speaking = null + slot.status = 'failed' + round.failed.push(member.name) +- round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: String(error?.message ?? error).slice(0, 300), time: Date.now() }) ++ round.replies.push({ kind: 'failed', botId: member.id, name: member.name, color: member.color, code: 'SESSION', text: cut(String(error?.message ?? error), 300).text, time: Date.now() }) + continue + } const prompt = roomPrompt(room, member, round) round.shown?.push(member.id) const text = session.brief === undefined ? prompt : `${session.brief}\n\n${prompt}` @@ -1814,6 +1821,17 @@ diff --git a/src/host/groups.js b/src/host/groups.js // Private context enters a group only through this tool, and the group sees each use. ctx.tools.register(defineTool({ name: 'read_own_chat', +@@ -489,8 +489,8 @@ export function install(rt) { + if (botId === undefined) return 'Only Bots on the team can use read_own_chat.' + const member = rt.botOf(botId) + const room = rt.roomOf(owner?.roomId) +- const query = String(args.query ?? '').trim().slice(0, 80) +- const around = String(args.around ?? '').trim().slice(0, 40) ++ const query = cut(String(args.query ?? '').trim(), 80).text ++ const around = cut(String(args.around ?? '').trim(), 40).text + const parts = rt.partsOf(botId) + const number = Number.isInteger(args.part) ? args.part : undefined + if (room !== undefined && member !== undefined) { @@ -517,13 +517,17 @@ export function install(rt) { return `Part ${number} of your own chat (${rt.clock(part.startedAt)} – ${rt.clock(part.endedAt)}). The handoff note that closed it:\n${rt.noteText(part) ?? '(none: the part ended on a context overflow)'}${partsNote}` } diff --git a/patches/ds-bot/0014-flybot-branding.patch b/patches/ds-bot/0014-flybot-branding.patch index 06a38abcc..269102a91 100644 --- a/patches/ds-bot/0014-flybot-branding.patch +++ b/patches/ds-bot/0014-flybot-branding.patch @@ -551,6 +551,15 @@ diff --git a/src/host/browser.js b/src/host/browser.js /** * Where a part of the text that starts at `start` ends: at a blank line, else a line break, +@@ -250,7 +250,7 @@ export function install(rt) { + if (value === null || typeof value !== 'object') return open + for (const [botId, page] of Object.entries(value).slice(0, OPEN_MAX)) { + if (rt.botOf(botId) === undefined || typeof page?.url !== 'string') continue +- open[botId] = { url: page.url.slice(0, 2000), title: String(page.title ?? '').slice(0, 300) } ++ open[botId] = { url: cut(page.url, 2000).text, title: cut(String(page.title ?? ''), 300).text } + } + return open + } diff --git a/src/host/dsh-update.js b/src/host/dsh-update.js --- a/src/host/dsh-update.js +++ b/src/host/dsh-update.js @@ -1647,6 +1656,45 @@ diff --git a/src/host/update.js b/src/host/update.js const service = manager() if (typeof service?.installBundle !== 'function') throw new Error('The DSH plugin manager is not available in this profile') const chosen = target +diff --git a/test/browser-channel.test.js b/test/browser-channel.test.js +--- a/test/browser-channel.test.js ++++ b/test/browser-channel.test.js +@@ -1,6 +1,6 @@ + import { test } from 'node:test' + import assert from 'node:assert/strict' +-import { createBrowserChannel, pageText, partEnd } from '../src/host/browser.js' ++import { createBrowserChannel, install, pageText, partEnd } from '../src/host/browser.js' + + const ids = items => items.map(item => item.id) + +@@ -141,3 +141,27 @@ test('partEnd prefers a blank line, then a line break, and cuts hard only withou + assert.equal(partEnd('a\n' + 'b'.repeat(20), 0, 10), 10) + assert.equal(partEnd('short', 0, 10), 5) + }) ++ ++// #1916: the open-page snapshot is what a Bot reads back from outside the team, and both bounds ++// are code-unit ones: 2000 for an address and 300 for a title must not land inside an emoji. ++test('the open-page snapshot never cuts a title or an address in half (#1916)', () => { ++ const registered = [] ++ const rt = { ++ ctx: { get: () => ({ name: 'desktop' }), effect: () => {}, tools: { register: (tool) => registered.push(tool) } }, ++ config: {}, ++ botOf: () => ({ id: 'a' }), ++ } ++ install(rt) ++ const open = rt.browserOpen({ ++ a: { url: `${'a'.repeat(1999)}\u{1F600}tail`, title: `${'b'.repeat(299)}\u{1F600}tail` }, ++ }) ++ assert.equal(open.a.url.isWellFormed(), true) ++ assert.equal(open.a.title.isWellFormed(), true) ++ assert.equal(open.a.url, 'a'.repeat(1999)) ++ assert.equal(open.a.title, 'b'.repeat(299)) ++ // A cut that lands on a whole character keeps it. ++ const roomy = rt.browserOpen({ a: { url: `${'a'.repeat(1998)}\u{1F600}`, title: `${'b'.repeat(298)}\u{1F600}` } }) ++ assert.equal(roomy.a.url, `${'a'.repeat(1998)}\u{1F600}`) ++ assert.equal(roomy.a.title, `${'b'.repeat(298)}\u{1F600}`) ++ assert.equal(registered.length > 0, true) ++}) diff --git a/test/capabilities.test.js b/test/capabilities.test.js --- a/test/capabilities.test.js +++ b/test/capabilities.test.js diff --git a/patches/ds-bot/files/src/shared/remote-label.js b/patches/ds-bot/files/src/shared/remote-label.js index 7312732e0..f229208c2 100644 --- a/patches/ds-bot/files/src/shared/remote-label.js +++ b/patches/ds-bot/files/src/shared/remote-label.js @@ -1,10 +1,12 @@ // Display labels for a friend's published Bot and for a room the owner did not start. // Peer-controlled strings (names, homes, handles) are untrusted: strip controls and // markup so a name cannot look like instructions when it later reaches a model. +import { cut } from './text-boundary.js' export function cleanPeerName(name, fallback = '') { - const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim().slice(0, 60) - return text || fallback + const text = String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"`<>]/g, '').replace(/\s+/g, ' ').trim() + const label = cut(text, 60).text + return label || fallback } export function normRemoteHandle(handle) { diff --git a/patches/ds-bot/files/src/shared/text-boundary.js b/patches/ds-bot/files/src/shared/text-boundary.js new file mode 100644 index 000000000..af6579e24 --- /dev/null +++ b/patches/ds-bot/files/src/shared/text-boundary.js @@ -0,0 +1,16 @@ +// Where a bounded prefix may end. Both sides bound text (#1908/#1912/#1916): the host cuts what a +// Bot reads back and what it stores, and a shared label cuts a peer's name. A cut that lands between +// the halves of a surrogate pair reaches a UTF-8 encoder as U+FFFD, so every bound goes through here +// instead of `slice`, and `host/text.js` builds its `cut` on this one implementation. + +const isHighSurrogate = code => code >= 0xd800 && code <= 0xdbff +const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff + +// The start of a bounded window, moved back onto a code point boundary. +export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at) + +// The end of a bounded prefix, moved back onto a code point boundary. +export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) + +// A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. +export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) diff --git a/patches/ds-bot/files/test/claim-cue.test.js b/patches/ds-bot/files/test/claim-cue.test.js index 0aa31cf61..feabe4a11 100644 --- a/patches/ds-bot/files/test/claim-cue.test.js +++ b/patches/ds-bot/files/test/claim-cue.test.js @@ -10,7 +10,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' import { createRuntime } from '../index.js' -import { CLAIM_CUE_REPEAT_MS } from '../src/host/groups.js' +import { CLAIM_CUE_REPEAT_MS, claimCueText } from '../src/host/groups.js' import { fakeContext } from './fake-ctx.js' const team = () => ({ @@ -180,3 +180,20 @@ test('a claim outside a room posts no line (#1253)', async (t) => { test('the repeat window is a stated number of minutes, not forever (#1253)', () => { assert.equal(CLAIM_CUE_REPEAT_MS, 30 * 60 * 1000) }) + +// #1916: the reference of a title with no issue number is a bounded prefix, and the bound is a +// code-unit one: 60 must not land between the halves of an emoji, or the room notice and the +// room log both keep an ill-formed string. +const utf8Prefix = (whole, part) => Buffer.from(part, 'utf8').equals(Buffer.from(whole, 'utf8').subarray(0, Buffer.byteLength(part))) + +test('a claim cue reference cut at 60 never splits a surrogate pair (#1916)', () => { + const title = `${'a'.repeat(59)}\u{1F600}tail` + const ref = claimCueText({ title }).replace(/^接了 /, '') + assert.equal(ref.isWellFormed(), true) + assert.equal(ref.includes('\uFFFD'), false) + assert.ok(ref.length <= 60, `ref is ${ref.length} units`) + assert.equal(utf8Prefix(title, ref), true, 'the ref is a byte prefix of the title') + // A cut that lands on a whole character keeps the whole character. + const roomy = claimCueText({ title: `${'a'.repeat(58)}\u{1F600}tail` }).replace(/^接了 /, '') + assert.equal(roomy, `${'a'.repeat(58)}\u{1F600}`) +}) diff --git a/patches/ds-bot/files/test/group-bot-label.test.js b/patches/ds-bot/files/test/group-bot-label.test.js index 08ae245fd..f9c37ff94 100644 --- a/patches/ds-bot/files/test/group-bot-label.test.js +++ b/patches/ds-bot/files/test/group-bot-label.test.js @@ -117,3 +117,11 @@ test('#526: real names stay as they are', () => { const view = roster([seatMine, { ...reviewer, label: undefined }], 'group') assert.equal(remoteSpeaker({ remote: { handle }, senderName: 'Reviewer' }, view, 'r').name, 'Reviewer (flybot682b8a43)') }) + +// #1916: a friend's Bot or room name reaches a display label, and its 60-unit bound must not land +// inside an emoji: the half pair would show as U+FFFD wherever the label is drawn. +test('a peer name cut at 60 never splits a surrogate pair (#1916)', () => { + assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`), 'b'.repeat(59)) + assert.equal(labels.cleanPeerName(`${'b'.repeat(58)}\u{1F600}tail`), `${'b'.repeat(58)}\u{1F600}`) + assert.equal(labels.cleanPeerName(`${'b'.repeat(59)}\u{1F600}tail`, 'fallback').isWellFormed(), true) +}) diff --git a/patches/ds-bot/topics.json b/patches/ds-bot/topics.json index 705dba5c6..73290f013 100644 --- a/patches/ds-bot/topics.json +++ b/patches/ds-bot/topics.json @@ -142,7 +142,8 @@ "test/dsh-update.test.js", "test/store.test.js", "test/secrets.test.js", - "test/capabilities.test.js" + "test/capabilities.test.js", + "test/browser-channel.test.js" ], "0016-room-collaboration-smoke": [ "dev/mock-llm.mjs"