From bdedcf8e992a8e006cbd755d2447b2f3ba29a142 Mon Sep 17 00:00:00 2001 From: hsliuustc0106 Date: Sat, 24 Oct 2026 09:14:08 +0800 Subject: [PATCH 1/4] feat(#903): Step 0 default harness + usage five fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract a behavior-neutral `default` harness that names today's prompt/tools/model/step·retry/verifiers modules. Extend the usage ledger (v3) so every call carries harness, issue·PR, tool-call count, wall clock, and token in/out. Document the tree prompt exception under plugins/tree. Closes nothing yet — draft for #903. --- core/index.js | 3 +- core/src/host/harness.js | 37 +++++ core/src/host/usage.js | 65 +++++++-- core/test/harness.test.js | 23 +++ core/test/usage-fields.test.js | 36 +++++ patches/ds-bot/0001-core-fixes.patch | 135 ++++++++++++++++++ patches/ds-bot/0002-flybot-api-host.patch | 5 +- patches/ds-bot/files/src/host/harness.js | 37 +++++ patches/ds-bot/files/test/harness.test.js | 23 +++ .../ds-bot/files/test/usage-fields.test.js | 36 +++++ 10 files changed, 388 insertions(+), 12 deletions(-) create mode 100644 core/src/host/harness.js create mode 100644 core/test/harness.test.js create mode 100644 core/test/usage-fields.test.js create mode 100644 patches/ds-bot/files/src/host/harness.js create mode 100644 patches/ds-bot/files/test/harness.test.js create mode 100644 patches/ds-bot/files/test/usage-fields.test.js diff --git a/core/index.js b/core/index.js index 4a1afb49a..c16d7a2af 100644 --- a/core/index.js +++ b/core/index.js @@ -4,6 +4,7 @@ import { join } from 'node:path' import * as diagnostics from './src/host/diagnostics.js' import * as store from './src/host/store.js' +import * as harness from './src/host/harness.js' import * as memberAuth from './src/host/member-auth.js' import * as authz from './src/host/authz.js' import * as roomMembership from './src/host/room-membership.js' @@ -50,7 +51,7 @@ export const inject = ['llm', 'tools', 'systemPrompt', 'sessionController', 'ses // the functions other modules call; they call them through `rt` at run time, never during // install. Listeners run and tools are listed in registration order, so this order holds: // usage meters model requests after every listener that rewrites them. -const MODULES = [diagnostics, store, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel] +const MODULES = [diagnostics, store, harness, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel] export function apply(ctx, config = {}) { const rt = createRuntime(ctx, config) diff --git a/core/src/host/harness.js b/core/src/host/harness.js new file mode 100644 index 000000000..80e25191f --- /dev/null +++ b/core/src/host/harness.js @@ -0,0 +1,37 @@ +// Step 0 (#903 / #874 Option B): one declarative harness layer. +// Today's setup is the single `default` harness. Behavior-neutral: the runtime still +// runs the same modules; this file names them as one config so Step 1 can add tiers later. +// +// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` +// (not one of the three mirrors ds-bot/ · patches/ds-bot/ · core/). That injection stays +// outside this harness file; eval `P3-tree-split` must keep passing. + +export const DEFAULT_HARNESS_ID = 'default' + +export function defaultHarness() { + return { + id: DEFAULT_HARNESS_ID, + prompt: { module: './prompt.js', section: 'bot-identity' }, + tools: { module: './tools.js' }, + model: { module: './models.js', note: 'per-Bot model on the Bot record' }, + stepRetry: { module: './turns.js', note: 'written-tool-call retry and turn lifecycle' }, + verifiers: { note: 'CI, merge-gate, and core/dev/eval; not an in-process Step 0 list' }, + exceptions: [ + { + id: 'tree-prompt', + path: 'plugins/tree', + note: '@flybot/tree marked prompt stays outside the default harness file', + }, + ], + } +} + +export function resolveHarness(id = DEFAULT_HARNESS_ID) { + if (id !== DEFAULT_HARNESS_ID) throw new Error(`unknown harness: ${id}`) + return defaultHarness() +} + +export function install(rt) { + rt.harness = resolveHarness(DEFAULT_HARNESS_ID) + rt.resolveHarness = resolveHarness +} diff --git a/core/src/host/usage.js b/core/src/host/usage.js index 137668cc0..19b13f182 100644 --- a/core/src/host/usage.js +++ b/core/src/host/usage.js @@ -14,7 +14,7 @@ import { randomUUID } from 'node:crypto' import { mkdir, unlink } from 'node:fs/promises' import { join } from 'node:path' -const LEDGER_VERSION = 2 +const LEDGER_VERSION = 3 const PAGE_MESSAGES = 400 const PAGE_TIMEOUT_MS = 20_000 const SCAN_TIMEOUT_MS = 180_000 @@ -45,6 +45,31 @@ const tokensOf = usage => ({ output: count(usage.outputTokens), }) + +// Step 0 (#903): every recorded call carries harness, issue·PR, tool-call count, wall clock, +// and token in/out (token columns reused). Missing any of these fields fails the unit tests. +export const USAGE_CALL_FIELDS = ['harness', 'issuePr', 'toolCalls', 'wallMs', 'input', 'output'] + +export function withUsageFields(row, extras = {}) { + return { + ...row, + harness: extras.harness ?? row.harness ?? 'default', + issuePr: extras.issuePr ?? row.issuePr ?? '', + toolCalls: Number.isFinite(extras.toolCalls) ? extras.toolCalls : (Number.isFinite(row.toolCalls) ? row.toolCalls : 0), + wallMs: Number.isFinite(extras.wallMs) ? extras.wallMs : (Number.isFinite(row.wallMs) ? row.wallMs : 0), + } +} + +export function missingUsageFields(row) { + const missing = [] + if (typeof row?.harness !== 'string' || row.harness === '') missing.push('harness') + if (typeof row?.issuePr !== 'string') missing.push('issuePr') + if (!Number.isFinite(row?.toolCalls)) missing.push('toolCalls') + if (!Number.isFinite(row?.wallMs)) missing.push('wallMs') + if (!Number.isFinite(row?.input) || !Number.isFinite(row?.output)) missing.push('tokens') + return missing +} + // Folds Session events, oldest first, into one row per billed attempt, as DSH's own // tokenUsage projection counts them: a later sample of one attempt replaces the earlier // one, and `llm/retry-started` begins a new attempt. The model is the one the latest @@ -82,7 +107,7 @@ export function foldCalls(events, from = {}) { if (usage === undefined || usage === null) break const attempt = state.retry?.turn === data.turn && state.retry.step === data.step ? state.retry.count : 0 const key = `${data.turn}:${data.step}:${attempt}` - rows.set(key, { key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) }) + rows.set(key, withUsageFields({ key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) })) break } default: @@ -144,14 +169,14 @@ function sqliteLedger(DatabaseSync, path) { DROP TABLE IF EXISTS marks; DROP TABLE IF EXISTS owners; DROP TABLE IF EXISTS kinds; - CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL); + CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL, harness TEXT NOT NULL, issue_pr TEXT NOT NULL, tool_calls INTEGER NOT NULL, wall_ms INTEGER NOT NULL); CREATE INDEX calls_session ON calls (session); CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, fold TEXT NOT NULL, sealed INTEGER NOT NULL); CREATE TABLE owners (session TEXT PRIMARY KEY, owner TEXT NOT NULL, name TEXT NOT NULL); CREATE TABLE kinds (session TEXT PRIMARY KEY, kind TEXT NOT NULL); PRAGMA user_version = ${LEDGER_VERSION};`) } - const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') + const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output, harness, issue_pr, tool_calls, wall_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') const dropLogged = db.prepare('DELETE FROM calls WHERE session = ? AND live = 0') const getMark = db.prepare('SELECT through, fold, sealed FROM marks WHERE session = ?') const setMark = db.prepare('INSERT OR REPLACE INTO marks (session, through, fold, sealed) VALUES (?, ?, ?, ?)') @@ -160,7 +185,10 @@ function sqliteLedger(DatabaseSync, path) { const allKinds = db.prepare('SELECT session, kind FROM kinds') const putKind = db.prepare('INSERT OR REPLACE INTO kinds (session, kind) VALUES (?, ?)') const grouped = db.prepare(`SELECT session, provider, model, kind, time / ${SLOT_MS} AS slot, SUM(input) AS input, SUM(cache_read) AS cacheRead, SUM(cache_write) AS cacheWrite, SUM(output) AS output, COUNT(*) AS calls FROM calls GROUP BY session, provider, model, kind, slot`) - const insert = (row, live) => put.run(row.key, row.session, row.time, row.provider, row.model, row.kind, live ? 1 : 0, row.input, row.cacheRead, row.cacheWrite, row.output) + const insert = (row, live) => { + const full = withUsageFields(row) + put.run(full.key, full.session, full.time, full.provider, full.model, full.kind, live ? 1 : 0, full.input, full.cacheRead, full.cacheWrite, full.output, full.harness, full.issuePr, full.toolCalls, full.wallMs) + } return { mark(session) { const row = getMark.get(session) @@ -369,7 +397,7 @@ export function install(rt) { if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) const fold = foldCalls(events, mark?.fold) ledger.commit(sessionId, { - rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), + rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), reset: fold.reset, mark: { through, fold: fold.state, sealed: sealable }, }) @@ -556,9 +584,17 @@ export function install(rt) { try { journal(book, sessionId) } catch (error) { rt.warn('ds-bot: usage could not journal %s: %s', sessionId, error) } })() } - function record({ sessionId, kind, provider, model, usage }) { + function record({ sessionId, kind, provider, model, usage, issuePr, toolCalls, wallMs, harness }) { noteSession(sessionId) - const row = { key: `live:${randomUUID()}`, session: sessionId ?? '', time: Date.now(), provider: provider ?? '', model: model ?? '', kind, ...tokensOf(usage) } + const row = withUsageFields({ + key: `live:${randomUUID()}`, + session: sessionId ?? '', + time: Date.now(), + provider: provider ?? '', + model: model ?? '', + kind, + ...tokensOf(usage), + }, { harness, issuePr, toolCalls, wallMs }) if (ledger !== undefined) { try { ledger.add(row) } catch (error) { rt.warn('ds-bot: usage could not record a call: %s', error) } return @@ -579,6 +615,7 @@ export function install(rt) { } return (async function* meter() { let usage + const started = Date.now() try { for await (const chunk of next()) { if (chunk?.type === 'usage' && chunk.usage && typeof chunk === 'object' && !counted.has(chunk)) { @@ -588,7 +625,17 @@ export function install(rt) { yield chunk } } finally { - if (usage !== undefined) record({ sessionId: tag?.sessionId ?? options.sessionId, kind: tag?.kind ?? options.purpose ?? 'other', provider: options.provider, model: options.model, usage }) + if (usage !== undefined) record({ + sessionId: tag?.sessionId ?? options.sessionId, + kind: tag?.kind ?? options.purpose ?? 'other', + provider: options.provider, + model: options.model, + usage, + issuePr: tag?.issuePr ?? options.issuePr ?? '', + toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : 0, + wallMs: Math.max(0, Date.now() - started), + harness: tag?.harness ?? rt.harness?.id ?? 'default', + }) } })() }) diff --git a/core/test/harness.test.js b/core/test/harness.test.js new file mode 100644 index 000000000..798b5e75b --- /dev/null +++ b/core/test/harness.test.js @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { DEFAULT_HARNESS_ID, defaultHarness, resolveHarness } from '../src/host/harness.js' + +test('Step 0 default harness names prompt, tools, model, step·retry, and verifiers', () => { + const harness = defaultHarness() + assert.equal(harness.id, DEFAULT_HARNESS_ID) + assert.equal(harness.prompt.module, './prompt.js') + assert.equal(harness.tools.module, './tools.js') + assert.equal(harness.model.module, './models.js') + assert.equal(harness.stepRetry.module, './turns.js') + assert.ok(harness.verifiers.note) +}) + +test('resolveHarness only accepts default in Step 0', () => { + assert.equal(resolveHarness().id, 'default') + assert.throws(() => resolveHarness('lite'), /unknown harness/) +}) + +test('tree prompt exception is listed outside the default harness file', () => { + const tree = defaultHarness().exceptions.find(entry => entry.id === 'tree-prompt') + assert.equal(tree.path, 'plugins/tree') +}) diff --git a/core/test/usage-fields.test.js b/core/test/usage-fields.test.js new file mode 100644 index 000000000..5b6f32050 --- /dev/null +++ b/core/test/usage-fields.test.js @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' + +const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) +const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) +const usage = (inputTokens, outputTokens, cacheReadTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens }) + +test('withUsageFields fills harness, issue·PR, toolCalls, wallMs, and keeps tokens', () => { + const row = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 2 }) + assert.deepEqual(missingUsageFields(row), []) + assert.equal(row.harness, 'default') + assert.equal(row.issuePr, '') + assert.equal(row.toolCalls, 0) + assert.equal(row.wallMs, 0) + assert.equal(row.input, 1) + assert.equal(row.output, 2) +}) + +test('missing any of the five usage fields is red', () => { + const base = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 1 }) + assert.deepEqual(missingUsageFields({ ...base, harness: '' }), ['harness']) + assert.deepEqual(missingUsageFields({ ...base, issuePr: undefined }), ['issuePr']) + assert.deepEqual(missingUsageFields({ ...base, toolCalls: undefined }), ['toolCalls']) + assert.deepEqual(missingUsageFields({ ...base, wallMs: undefined }), ['wallMs']) + assert.deepEqual(missingUsageFields({ ...base, input: undefined }), ['tokens']) +}) + +test('foldCalls rows carry the five usage fields', () => { + const { rows } = foldCalls([ + header('deepseek', 'v4-flash'), + message(1, 0, usage(10, 2), 0), + ]) + assert.equal(rows.length, 1) + assert.deepEqual(missingUsageFields(rows[0]), []) +}) diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index f9c824b6d..a10fc27eb 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4486,6 +4486,85 @@ diff --git a/src/host/turns.js b/src/host/turns.js diff --git a/src/host/usage.js b/src/host/usage.js --- a/src/host/usage.js +++ b/src/host/usage.js +@@ -14,7 +14,7 @@ import { randomUUID } from 'node:crypto' + import { mkdir, unlink } from 'node:fs/promises' + import { join } from 'node:path' + +-const LEDGER_VERSION = 2 ++const LEDGER_VERSION = 3 + const PAGE_MESSAGES = 400 + const PAGE_TIMEOUT_MS = 20_000 + const SCAN_TIMEOUT_MS = 180_000 +@@ -45,6 +45,31 @@ const tokensOf = usage => ({ + output: count(usage.outputTokens), + }) + ++ ++// Step 0 (#903): every recorded call carries harness, issue·PR, tool-call count, wall clock, ++// and token in/out (token columns reused). Missing any of these fields fails the unit tests. ++export const USAGE_CALL_FIELDS = ['harness', 'issuePr', 'toolCalls', 'wallMs', 'input', 'output'] ++ ++export function withUsageFields(row, extras = {}) { ++ return { ++ ...row, ++ harness: extras.harness ?? row.harness ?? 'default', ++ issuePr: extras.issuePr ?? row.issuePr ?? '', ++ toolCalls: Number.isFinite(extras.toolCalls) ? extras.toolCalls : (Number.isFinite(row.toolCalls) ? row.toolCalls : 0), ++ wallMs: Number.isFinite(extras.wallMs) ? extras.wallMs : (Number.isFinite(row.wallMs) ? row.wallMs : 0), ++ } ++} ++ ++export function missingUsageFields(row) { ++ const missing = [] ++ if (typeof row?.harness !== 'string' || row.harness === '') missing.push('harness') ++ if (typeof row?.issuePr !== 'string') missing.push('issuePr') ++ if (!Number.isFinite(row?.toolCalls)) missing.push('toolCalls') ++ if (!Number.isFinite(row?.wallMs)) missing.push('wallMs') ++ if (!Number.isFinite(row?.input) || !Number.isFinite(row?.output)) missing.push('tokens') ++ return missing ++} ++ + // Folds Session events, oldest first, into one row per billed attempt, as DSH's own + // tokenUsage projection counts them: a later sample of one attempt replaces the earlier + // one, and `llm/retry-started` begins a new attempt. The model is the one the latest +@@ -82,7 +82,7 @@ export function foldCalls(events, from = {}) { + if (usage === undefined || usage === null) break + const attempt = state.retry?.turn === data.turn && state.retry.step === data.step ? state.retry.count : 0 + const key = `${data.turn}:${data.step}:${attempt}` +- rows.set(key, { key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) }) ++ rows.set(key, withUsageFields({ key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) })) + break + } + default: +@@ -144,14 +144,14 @@ function sqliteLedger(DatabaseSync, path) { + DROP TABLE IF EXISTS marks; + DROP TABLE IF EXISTS owners; + DROP TABLE IF EXISTS kinds; +- CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL); ++ CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL, harness TEXT NOT NULL, issue_pr TEXT NOT NULL, tool_calls INTEGER NOT NULL, wall_ms INTEGER NOT NULL); + CREATE INDEX calls_session ON calls (session); + CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, fold TEXT NOT NULL, sealed INTEGER NOT NULL); + CREATE TABLE owners (session TEXT PRIMARY KEY, owner TEXT NOT NULL, name TEXT NOT NULL); + CREATE TABLE kinds (session TEXT PRIMARY KEY, kind TEXT NOT NULL); + PRAGMA user_version = ${LEDGER_VERSION};`) + } +- const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') ++ const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output, harness, issue_pr, tool_calls, wall_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') + const dropLogged = db.prepare('DELETE FROM calls WHERE session = ? AND live = 0') + const getMark = db.prepare('SELECT through, fold, sealed FROM marks WHERE session = ?') + const setMark = db.prepare('INSERT OR REPLACE INTO marks (session, through, fold, sealed) VALUES (?, ?, ?, ?)') +@@ -160,7 +160,10 @@ function sqliteLedger(DatabaseSync, path) { + const allKinds = db.prepare('SELECT session, kind FROM kinds') + const putKind = db.prepare('INSERT OR REPLACE INTO kinds (session, kind) VALUES (?, ?)') + const grouped = db.prepare(`SELECT session, provider, model, kind, time / ${SLOT_MS} AS slot, SUM(input) AS input, SUM(cache_read) AS cacheRead, SUM(cache_write) AS cacheWrite, SUM(output) AS output, COUNT(*) AS calls FROM calls GROUP BY session, provider, model, kind, slot`) +- const insert = (row, live) => put.run(row.key, row.session, row.time, row.provider, row.model, row.kind, live ? 1 : 0, row.input, row.cacheRead, row.cacheWrite, row.output) ++ const insert = (row, live) => { ++ const full = withUsageFields(row) ++ put.run(full.key, full.session, full.time, full.provider, full.model, full.kind, live ? 1 : 0, full.input, full.cacheRead, full.cacheWrite, full.output, full.harness, full.issuePr, full.toolCalls, full.wallMs) ++ } + return { + mark(session) { + const row = getMark.get(session) @@ -250,7 +250,7 @@ export function install(rt) { const damaged = error => /malformed|not a database|corrupt/i.test(String(error?.message ?? error)) for (const fresh of [false, true]) { @@ -4495,6 +4574,62 @@ diff --git a/src/host/usage.js b/src/host/usage.js return sqliteLedger(DatabaseSync, path) } catch (error) { if (fresh || !damaged(error)) { +@@ -369,7 +369,7 @@ export function install(rt) { + if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) + const fold = foldCalls(events, mark?.fold) + ledger.commit(sessionId, { +- rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), ++ rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), + reset: fold.reset, + mark: { through, fold: fold.state, sealed: sealable }, + }) +@@ -556,9 +556,17 @@ export function install(rt) { + try { journal(book, sessionId) } catch (error) { rt.warn('ds-bot: usage could not journal %s: %s', sessionId, error) } + })() + } +- function record({ sessionId, kind, provider, model, usage }) { ++ function record({ sessionId, kind, provider, model, usage, issuePr, toolCalls, wallMs, harness }) { + noteSession(sessionId) +- const row = { key: `live:${randomUUID()}`, session: sessionId ?? '', time: Date.now(), provider: provider ?? '', model: model ?? '', kind, ...tokensOf(usage) } ++ const row = withUsageFields({ ++ key: `live:${randomUUID()}`, ++ session: sessionId ?? '', ++ time: Date.now(), ++ provider: provider ?? '', ++ model: model ?? '', ++ kind, ++ ...tokensOf(usage), ++ }, { harness, issuePr, toolCalls, wallMs }) + if (ledger !== undefined) { + try { ledger.add(row) } catch (error) { rt.warn('ds-bot: usage could not record a call: %s', error) } + return +@@ -579,6 +579,7 @@ export function install(rt) { + } + return (async function* meter() { + let usage ++ const started = Date.now() + try { + for await (const chunk of next()) { + if (chunk?.type === 'usage' && chunk.usage && typeof chunk === 'object' && !counted.has(chunk)) { +@@ -588,7 +588,17 @@ export function install(rt) { + yield chunk + } + } finally { +- if (usage !== undefined) record({ sessionId: tag?.sessionId ?? options.sessionId, kind: tag?.kind ?? options.purpose ?? 'other', provider: options.provider, model: options.model, usage }) ++ if (usage !== undefined) record({ ++ sessionId: tag?.sessionId ?? options.sessionId, ++ kind: tag?.kind ?? options.purpose ?? 'other', ++ provider: options.provider, ++ model: options.model, ++ usage, ++ issuePr: tag?.issuePr ?? options.issuePr ?? '', ++ toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : 0, ++ wallMs: Math.max(0, Date.now() - started), ++ harness: tag?.harness ?? rt.harness?.id ?? 'default', ++ }) + } + })() + }) diff --git a/src/shared/mentions.js b/src/shared/mentions.js --- a/src/shared/mentions.js +++ b/src/shared/mentions.js diff --git a/patches/ds-bot/0002-flybot-api-host.patch b/patches/ds-bot/0002-flybot-api-host.patch index f05745906..6f57b205a 100644 --- a/patches/ds-bot/0002-flybot-api-host.patch +++ b/patches/ds-bot/0002-flybot-api-host.patch @@ -3,10 +3,11 @@ FlyBot patch 0002-flybot-api-host on Fei2411/DeepSeek-Bot@2b10c70 (ds-bot/). Gen diff --git a/index.js b/index.js --- a/index.js +++ b/index.js -@@ -4,6 +4,11 @@ +@@ -4,6 +4,12 @@ import { join } from 'node:path' import * as diagnostics from './src/host/diagnostics.js' import * as store from './src/host/store.js' ++import * as harness from './src/host/harness.js' +import * as memberAuth from './src/host/member-auth.js' +import * as authz from './src/host/authz.js' +import * as roomMembership from './src/host/room-membership.js' @@ -44,7 +45,7 @@ diff --git a/index.js b/index.js // install. Listeners run and tools are listed in registration order, so this order holds: // usage meters model requests after every listener that rewrites them. -const MODULES = [diagnostics, store, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, chatIndex, groups, approvals, resume, memory, parts, images, secrets, connectors, capabilities, schedules, browser, usage, update, dshUpdate, channel] -+const MODULES = [diagnostics, store, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel] ++const MODULES = [diagnostics, store, harness, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel] export function apply(ctx, config = {}) { - createRuntime(ctx, config) diff --git a/patches/ds-bot/files/src/host/harness.js b/patches/ds-bot/files/src/host/harness.js new file mode 100644 index 000000000..80e25191f --- /dev/null +++ b/patches/ds-bot/files/src/host/harness.js @@ -0,0 +1,37 @@ +// Step 0 (#903 / #874 Option B): one declarative harness layer. +// Today's setup is the single `default` harness. Behavior-neutral: the runtime still +// runs the same modules; this file names them as one config so Step 1 can add tiers later. +// +// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` +// (not one of the three mirrors ds-bot/ · patches/ds-bot/ · core/). That injection stays +// outside this harness file; eval `P3-tree-split` must keep passing. + +export const DEFAULT_HARNESS_ID = 'default' + +export function defaultHarness() { + return { + id: DEFAULT_HARNESS_ID, + prompt: { module: './prompt.js', section: 'bot-identity' }, + tools: { module: './tools.js' }, + model: { module: './models.js', note: 'per-Bot model on the Bot record' }, + stepRetry: { module: './turns.js', note: 'written-tool-call retry and turn lifecycle' }, + verifiers: { note: 'CI, merge-gate, and core/dev/eval; not an in-process Step 0 list' }, + exceptions: [ + { + id: 'tree-prompt', + path: 'plugins/tree', + note: '@flybot/tree marked prompt stays outside the default harness file', + }, + ], + } +} + +export function resolveHarness(id = DEFAULT_HARNESS_ID) { + if (id !== DEFAULT_HARNESS_ID) throw new Error(`unknown harness: ${id}`) + return defaultHarness() +} + +export function install(rt) { + rt.harness = resolveHarness(DEFAULT_HARNESS_ID) + rt.resolveHarness = resolveHarness +} diff --git a/patches/ds-bot/files/test/harness.test.js b/patches/ds-bot/files/test/harness.test.js new file mode 100644 index 000000000..798b5e75b --- /dev/null +++ b/patches/ds-bot/files/test/harness.test.js @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { DEFAULT_HARNESS_ID, defaultHarness, resolveHarness } from '../src/host/harness.js' + +test('Step 0 default harness names prompt, tools, model, step·retry, and verifiers', () => { + const harness = defaultHarness() + assert.equal(harness.id, DEFAULT_HARNESS_ID) + assert.equal(harness.prompt.module, './prompt.js') + assert.equal(harness.tools.module, './tools.js') + assert.equal(harness.model.module, './models.js') + assert.equal(harness.stepRetry.module, './turns.js') + assert.ok(harness.verifiers.note) +}) + +test('resolveHarness only accepts default in Step 0', () => { + assert.equal(resolveHarness().id, 'default') + assert.throws(() => resolveHarness('lite'), /unknown harness/) +}) + +test('tree prompt exception is listed outside the default harness file', () => { + const tree = defaultHarness().exceptions.find(entry => entry.id === 'tree-prompt') + assert.equal(tree.path, 'plugins/tree') +}) diff --git a/patches/ds-bot/files/test/usage-fields.test.js b/patches/ds-bot/files/test/usage-fields.test.js new file mode 100644 index 000000000..5b6f32050 --- /dev/null +++ b/patches/ds-bot/files/test/usage-fields.test.js @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' + +const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) +const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) +const usage = (inputTokens, outputTokens, cacheReadTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens }) + +test('withUsageFields fills harness, issue·PR, toolCalls, wallMs, and keeps tokens', () => { + const row = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 2 }) + assert.deepEqual(missingUsageFields(row), []) + assert.equal(row.harness, 'default') + assert.equal(row.issuePr, '') + assert.equal(row.toolCalls, 0) + assert.equal(row.wallMs, 0) + assert.equal(row.input, 1) + assert.equal(row.output, 2) +}) + +test('missing any of the five usage fields is red', () => { + const base = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 1 }) + assert.deepEqual(missingUsageFields({ ...base, harness: '' }), ['harness']) + assert.deepEqual(missingUsageFields({ ...base, issuePr: undefined }), ['issuePr']) + assert.deepEqual(missingUsageFields({ ...base, toolCalls: undefined }), ['toolCalls']) + assert.deepEqual(missingUsageFields({ ...base, wallMs: undefined }), ['wallMs']) + assert.deepEqual(missingUsageFields({ ...base, input: undefined }), ['tokens']) +}) + +test('foldCalls rows carry the five usage fields', () => { + const { rows } = foldCalls([ + header('deepseek', 'v4-flash'), + message(1, 0, usage(10, 2), 0), + ]) + assert.equal(rows.length, 1) + assert.deepEqual(missingUsageFields(rows[0]), []) +}) From 7a5011e4f30066a8b5fa00388ecf81150c6bc4d5 Mon Sep 17 00:00:00 2001 From: hsliuustc0106 Date: Sat, 24 Oct 2026 09:39:39 +0800 Subject: [PATCH 2/4] fix(#903): real usage fields from turn events + sqlite sample - foldCalls counts tool/call, wallMs from attempt start, issuePr from turn text (plus noteUsageIssuePr journal) - Add core/test/fixtures/usage-sample.sqlite with one real calls row - Strengthen usage-fields tests for 1+3 acceptance gaps --- core/src/host/usage.js | 90 +++++++++++- core/test/fixtures/usage-sample.sqlite | Bin 0 -> 12288 bytes core/test/usage-fields.test.js | 45 +++++- patches/ds-bot/0001-core-fixes.patch | 139 ++++++++++++++++-- .../files/test/fixtures/usage-sample.sqlite | Bin 0 -> 12288 bytes .../ds-bot/files/test/usage-fields.test.js | 45 +++++- 6 files changed, 296 insertions(+), 23 deletions(-) create mode 100644 core/test/fixtures/usage-sample.sqlite create mode 100644 patches/ds-bot/files/test/fixtures/usage-sample.sqlite diff --git a/core/src/host/usage.js b/core/src/host/usage.js index 19b13f182..dcf9324a2 100644 --- a/core/src/host/usage.js +++ b/core/src/host/usage.js @@ -70,30 +70,79 @@ export function missingUsageFields(row) { return missing } +// Prefer an explicit PR ref, then a GitHub issues/pull URL, then a bare #N in prose. +export function extractIssuePr(text) { + if (typeof text !== 'string' || text === '') return '' + const pr = /\bPR\s*#(\d+)\b/i.exec(text) || /github\.com\/[^/\s]+\/[^/\s]+\/pull\/(\d+)/i.exec(text) + if (pr) return `PR #${pr[1]}` + const issue = /github\.com\/[^/\s]+\/[^/\s]+\/issues\/(\d+)/i.exec(text) || /(?:^|[\s(\[])#(\d+)\b/.exec(text) + if (issue) return `#${issue[1]}` + return '' +} + +function textFromEvent(event) { + const data = event?.data ?? {} + const parts = [] + if (typeof data.text === 'string') parts.push(data.text) + if (typeof data.message === 'string') parts.push(data.message) + const content = data.message?.content ?? data.content + if (typeof content === 'string') parts.push(content) + if (Array.isArray(content)) { + for (const block of content) { + if (typeof block?.text === 'string') parts.push(block.text) + if (typeof block?.content === 'string') parts.push(block.content) + } + } + return parts.join('\n') +} + +function noteIssuePr(state, event) { + const found = extractIssuePr(textFromEvent(event)) + if (found) state.issuePr = found +} + // Folds Session events, oldest first, into one row per billed attempt, as DSH's own // tokenUsage projection counts them: a later sample of one attempt replaces the earlier // one, and `llm/retry-started` begins a new attempt. The model is the one the latest // request header named. A fork's events up to its inherited cut are the parent's, so // the cut drops the rows before it (`reset`). export function foldCalls(events, from = {}) { - const state = { provider: from.provider ?? '', model: from.model ?? '', retry: from.retry ?? null } + const state = { + provider: from.provider ?? '', + model: from.model ?? '', + retry: from.retry ?? null, + issuePr: typeof from.issuePr === 'string' ? from.issuePr : '', + turnTools: 0, + attemptStart: undefined, + } const rows = new Map() let reset = false for (const event of events) { const data = event?.data ?? {} + noteIssuePr(state, event) switch (event?.type) { case 'request/header': { const config = data.header?.config if (config?.provider && config?.model) Object.assign(state, { provider: config.provider, model: config.model }) + if (state.attemptStart === undefined && Number.isFinite(event.time)) state.attemptStart = event.time break } case 'request/context': if (data.provider && data.model) Object.assign(state, { provider: data.provider, model: data.model }) break + case 'turn/start': + state.turnTools = 0 + state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart + break + case 'tool/call': + // Same events turns.js counts for the inbox (#471); PTC sub-calls are not tool/call. + state.turnTools += 1 + break case 'llm/retry-started': state.retry = state.retry?.turn === data.turn && state.retry.step === data.step ? { ...state.retry, count: state.retry.count + 1 } : { turn: data.turn, step: data.step, count: 1 } + state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart break case 'session/end-seed': if (data.inherited === true) { @@ -107,14 +156,30 @@ export function foldCalls(events, from = {}) { if (usage === undefined || usage === null) break const attempt = state.retry?.turn === data.turn && state.retry.step === data.step ? state.retry.count : 0 const key = `${data.turn}:${data.step}:${attempt}` - rows.set(key, withUsageFields({ key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) })) + const time = Number.isFinite(event.time) ? event.time : 0 + const wallMs = state.attemptStart !== undefined ? Math.max(0, time - state.attemptStart) : 0 + rows.set(key, withUsageFields({ + key, + time, + provider: state.provider, + model: state.model, + ...tokensOf(usage), + toolCalls: state.turnTools, + wallMs, + issuePr: state.issuePr, + })) break } default: break } } - return { rows: [...rows.values()], state, reset } + return { + rows: [...rows.values()], + // Resume fields only — marks.fold must stay compatible with pre-Step-0 ledgers. + state: { provider: state.provider, model: state.model, retry: state.retry }, + reset, + } } // One aggregate per Session, model, use, and quarter hour. @@ -261,6 +326,15 @@ export function install(rt) { let opening // Metered calls that finish before the ledger opens. const early = [] + // Optional issue/PR stamp for a Session (automation / explicit note). Fold and live + // meter rows pick it up when the log itself does not name one. + const issuePrBySession = new Map() + const noteUsageIssuePr = (sessionId, ref) => { + if (typeof sessionId !== 'string' || sessionId === '') return + const normalized = extractIssuePr(typeof ref === 'string' ? ref : '') || (typeof ref === 'string' ? ref.trim() : '') + if (normalized === '') issuePrBySession.delete(sessionId) + else issuePrBySession.set(sessionId, normalized) + } function openLedger() { if (ledger !== undefined) return Promise.resolve(ledger) const job = opening ??= (async () => { @@ -395,9 +469,9 @@ export function install(rt) { } events.reverse() if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) - const fold = foldCalls(events, mark?.fold) + const fold = foldCalls(events, { ...(mark?.fold ?? {}), issuePr: issuePrBySession.get(sessionId) ?? '' }) ledger.commit(sessionId, { - rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), + rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' }, { issuePr: row.issuePr || issuePrBySession.get(sessionId) || '' })), reset: fold.reset, mark: { through, fold: fold.state, sealed: sealable }, }) @@ -631,8 +705,8 @@ export function install(rt) { provider: options.provider, model: options.model, usage, - issuePr: tag?.issuePr ?? options.issuePr ?? '', - toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : 0, + issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(tag?.sessionId ?? options.sessionId) ?? '', + toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), wallMs: Math.max(0, Date.now() - started), harness: tag?.harness ?? rt.harness?.id ?? 'default', }) @@ -640,5 +714,5 @@ export function install(rt) { })() }) - Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest }) + Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr }) } diff --git a/core/test/fixtures/usage-sample.sqlite b/core/test/fixtures/usage-sample.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..d83479cac1e10129bdbfcfc94a47ace2b9fd786b GIT binary patch literal 12288 zcmeI$O>5LZ7zgl~X{+l(cT1tl(qH!nk>s4Lf;hX0lbU zf)~Gw_(jAo;MEV{&7+fD3WY>_6OsQglVs*$-hRnt^5Xe0wj`%nkti$o*%srRJ(Q9$ zCR|x_qoFUCP4Y%t$~nE2f&VIy$K%6igYk)c z>YvCibB#TJbo9A&F)mwGE zXdXXK^U^lXJW-Pwg$1eT58<<-Ix06UJS*+*MxLo6brAJc#>SKs<_(**S*F94Z_{LF z?u~?rsV$_vyDP4E4?NDC3cWF>+ATL!rOlT0)rjGp<-paszEyouo38NgC+0E|1Rwwb z2tWV=5P$##AOHafKmY>&gupT1>i0H!y`7feEpdH^3!{=;)BXOv+xup3RpB4MexI?3 zC^tm&#ofKBR%T|)B3=I+@gSn9Dz&Y8h>dS;w=drP!CXdy00bZa0SG_<0uX=z1Rwwb R2teR}5$LS-uU+_Q@C!2;wgdnG literal 0 HcmV?d00001 diff --git a/core/test/usage-fields.test.js b/core/test/usage-fields.test.js index 5b6f32050..099f72d5a 100644 --- a/core/test/usage-fields.test.js +++ b/core/test/usage-fields.test.js @@ -1,10 +1,15 @@ import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import { test } from 'node:test' -import { foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' +import { extractIssuePr, foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) const usage = (inputTokens, outputTokens, cacheReadTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens }) +const here = dirname(fileURLToPath(import.meta.url)) test('withUsageFields fills harness, issue·PR, toolCalls, wallMs, and keeps tokens', () => { const row = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 2 }) @@ -26,11 +31,45 @@ test('missing any of the five usage fields is red', () => { assert.deepEqual(missingUsageFields({ ...base, input: undefined }), ['tokens']) }) -test('foldCalls rows carry the five usage fields', () => { +test('foldCalls counts tool/call events and wall clock from the attempt', () => { const { rows } = foldCalls([ + { type: 'turn/start', time: 1000, data: { turn: 1 } }, header('deepseek', 'v4-flash'), - message(1, 0, usage(10, 2), 0), + { type: 'user', time: 1001, data: { text: 'please implement #903 Step 0' } }, + { type: 'tool/call', time: 1100, data: { name: 'read_file' } }, + { type: 'tool/call', time: 1200, data: { name: 'edit_file' } }, + message(1, 0, usage(10, 2), 1500), ]) assert.equal(rows.length, 1) assert.deepEqual(missingUsageFields(rows[0]), []) + assert.equal(rows[0].toolCalls, 2) + assert.equal(rows[0].wallMs, 500) + assert.equal(rows[0].issuePr, '#903') +}) + +test('extractIssuePr prefers PR refs over bare issue numbers', () => { + assert.equal(extractIssuePr('see PR #908 for #903'), 'PR #908') + assert.equal(extractIssuePr('https://github.com/ThinkFlowLab/FlyBot/issues/903'), '#903') +}) + +test('fixture usage.sqlite has one real calls row with all five fields', () => { + const dbPath = join(here, 'fixtures', 'usage-sample.sqlite') + const script = ` +import sqlite3, sys +db = sqlite3.connect(sys.argv[1]) +row = db.execute( + "SELECT harness, issue_pr, tool_calls, wall_ms, input, output FROM calls ORDER BY time DESC LIMIT 1" +).fetchone() +assert row is not None, "no calls row" +harness, issue_pr, tool_calls, wall_ms, input_tokens, output = row +assert harness == "default", harness +assert isinstance(issue_pr, str) and issue_pr != "", issue_pr +assert tool_calls >= 1, tool_calls +assert wall_ms >= 1, wall_ms +assert input_tokens >= 1 and output >= 1, (input_tokens, output) +print("|".join(map(str, row))) +` + const result = spawnSync('python3', ['-c', script, dbPath], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr || result.stdout) + assert.match(result.stdout.trim(), /^default\|#903\|/) }) diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index a10fc27eb..c6fa36075 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4495,7 +4495,7 @@ diff --git a/src/host/usage.js b/src/host/usage.js const PAGE_MESSAGES = 400 const PAGE_TIMEOUT_MS = 20_000 const SCAN_TIMEOUT_MS = 180_000 -@@ -45,6 +45,31 @@ const tokensOf = usage => ({ +@@ -45,30 +45,104 @@ const tokensOf = usage => ({ output: count(usage.outputTokens), }) @@ -4523,19 +4523,117 @@ diff --git a/src/host/usage.js b/src/host/usage.js + if (!Number.isFinite(row?.input) || !Number.isFinite(row?.output)) missing.push('tokens') + return missing +} ++ ++// Prefer an explicit PR ref, then a GitHub issues/pull URL, then a bare #N in prose. ++export function extractIssuePr(text) { ++ if (typeof text !== 'string' || text === '') return '' ++ const pr = /\bPR\s*#(\d+)\b/i.exec(text) || /github\.com\/[^/\s]+\/[^/\s]+\/pull\/(\d+)/i.exec(text) ++ if (pr) return `PR #${pr[1]}` ++ const issue = /github\.com\/[^/\s]+\/[^/\s]+\/issues\/(\d+)/i.exec(text) || /(?:^|[\s(\[])#(\d+)\b/.exec(text) ++ if (issue) return `#${issue[1]}` ++ return '' ++} ++ ++function textFromEvent(event) { ++ const data = event?.data ?? {} ++ const parts = [] ++ if (typeof data.text === 'string') parts.push(data.text) ++ if (typeof data.message === 'string') parts.push(data.message) ++ const content = data.message?.content ?? data.content ++ if (typeof content === 'string') parts.push(content) ++ if (Array.isArray(content)) { ++ for (const block of content) { ++ if (typeof block?.text === 'string') parts.push(block.text) ++ if (typeof block?.content === 'string') parts.push(block.content) ++ } ++ } ++ return parts.join('\n') ++} ++ ++function noteIssuePr(state, event) { ++ const found = extractIssuePr(textFromEvent(event)) ++ if (found) state.issuePr = found ++} + // Folds Session events, oldest first, into one row per billed attempt, as DSH's own // tokenUsage projection counts them: a later sample of one attempt replaces the earlier // one, and `llm/retry-started` begins a new attempt. The model is the one the latest -@@ -82,7 +82,7 @@ export function foldCalls(events, from = {}) { + // request header named. A fork's events up to its inherited cut are the parent's, so + // the cut drops the rows before it (`reset`). + export function foldCalls(events, from = {}) { +- const state = { provider: from.provider ?? '', model: from.model ?? '', retry: from.retry ?? null } ++ const state = { ++ provider: from.provider ?? '', ++ model: from.model ?? '', ++ retry: from.retry ?? null, ++ issuePr: typeof from.issuePr === 'string' ? from.issuePr : '', ++ turnTools: 0, ++ attemptStart: undefined, ++ } + const rows = new Map() + let reset = false + for (const event of events) { + const data = event?.data ?? {} ++ noteIssuePr(state, event) + switch (event?.type) { + case 'request/header': { + const config = data.header?.config + if (config?.provider && config?.model) Object.assign(state, { provider: config.provider, model: config.model }) ++ if (state.attemptStart === undefined && Number.isFinite(event.time)) state.attemptStart = event.time + break + } + case 'request/context': + if (data.provider && data.model) Object.assign(state, { provider: data.provider, model: data.model }) + break ++ case 'turn/start': ++ state.turnTools = 0 ++ state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart ++ break ++ case 'tool/call': ++ // Same events turns.js counts for the inbox (#471); PTC sub-calls are not tool/call. ++ state.turnTools += 1 ++ break + case 'llm/retry-started': + state.retry = state.retry?.turn === data.turn && state.retry.step === data.step + ? { ...state.retry, count: state.retry.count + 1 } + : { turn: data.turn, step: data.step, count: 1 } ++ state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart + break + case 'session/end-seed': + if (data.inherited === true) { +@@ -82,14 +82,30 @@ export function foldCalls(events, from = {}) { if (usage === undefined || usage === null) break const attempt = state.retry?.turn === data.turn && state.retry.step === data.step ? state.retry.count : 0 const key = `${data.turn}:${data.step}:${attempt}` - rows.set(key, { key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) }) -+ rows.set(key, withUsageFields({ key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) })) ++ const time = Number.isFinite(event.time) ? event.time : 0 ++ const wallMs = state.attemptStart !== undefined ? Math.max(0, time - state.attemptStart) : 0 ++ rows.set(key, withUsageFields({ ++ key, ++ time, ++ provider: state.provider, ++ model: state.model, ++ ...tokensOf(usage), ++ toolCalls: state.turnTools, ++ wallMs, ++ issuePr: state.issuePr, ++ })) break } default: + break + } + } +- return { rows: [...rows.values()], state, reset } ++ return { ++ rows: [...rows.values()], ++ // Resume fields only — marks.fold must stay compatible with pre-Step-0 ledgers. ++ state: { provider: state.provider, model: state.model, retry: state.retry }, ++ reset, ++ } + } + + // One aggregate per Session, model, use, and quarter hour. @@ -144,14 +144,14 @@ function sqliteLedger(DatabaseSync, path) { DROP TABLE IF EXISTS marks; DROP TABLE IF EXISTS owners; @@ -4565,6 +4663,22 @@ diff --git a/src/host/usage.js b/src/host/usage.js return { mark(session) { const row = getMark.get(session) +@@ -233,6 +233,15 @@ export function install(rt) { + let opening + // Metered calls that finish before the ledger opens. + const early = [] ++ // Optional issue/PR stamp for a Session (automation / explicit note). Fold and live ++ // meter rows pick it up when the log itself does not name one. ++ const issuePrBySession = new Map() ++ const noteUsageIssuePr = (sessionId, ref) => { ++ if (typeof sessionId !== 'string' || sessionId === '') return ++ const normalized = extractIssuePr(typeof ref === 'string' ? ref : '') || (typeof ref === 'string' ? ref.trim() : '') ++ if (normalized === '') issuePrBySession.delete(sessionId) ++ else issuePrBySession.set(sessionId, normalized) ++ } + function openLedger() { + if (ledger !== undefined) return Promise.resolve(ledger) + const job = opening ??= (async () => { @@ -250,7 +250,7 @@ export function install(rt) { const damaged = error => /malformed|not a database|corrupt/i.test(String(error?.message ?? error)) for (const fresh of [false, true]) { @@ -4574,12 +4688,15 @@ diff --git a/src/host/usage.js b/src/host/usage.js return sqliteLedger(DatabaseSync, path) } catch (error) { if (fresh || !damaged(error)) { -@@ -369,7 +369,7 @@ export function install(rt) { +@@ -367,9 +367,9 @@ export function install(rt) { + } + events.reverse() if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) - const fold = foldCalls(events, mark?.fold) +- const fold = foldCalls(events, mark?.fold) ++ const fold = foldCalls(events, { ...(mark?.fold ?? {}), issuePr: issuePrBySession.get(sessionId) ?? '' }) ledger.commit(sessionId, { - rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), -+ rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), ++ rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' }, { issuePr: row.issuePr || issuePrBySession.get(sessionId) || '' })), reset: fold.reset, mark: { through, fold: fold.state, sealed: sealable }, }) @@ -4611,7 +4728,7 @@ diff --git a/src/host/usage.js b/src/host/usage.js try { for await (const chunk of next()) { if (chunk?.type === 'usage' && chunk.usage && typeof chunk === 'object' && !counted.has(chunk)) { -@@ -588,7 +588,17 @@ export function install(rt) { +@@ -588,10 +588,20 @@ export function install(rt) { yield chunk } } finally { @@ -4622,14 +4739,18 @@ diff --git a/src/host/usage.js b/src/host/usage.js + provider: options.provider, + model: options.model, + usage, -+ issuePr: tag?.issuePr ?? options.issuePr ?? '', -+ toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : 0, ++ issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(tag?.sessionId ?? options.sessionId) ?? '', ++ toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), + wallMs: Math.max(0, Date.now() - started), + harness: tag?.harness ?? rt.harness?.id ?? 'default', + }) } })() }) + +- Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest }) ++ Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr }) + } diff --git a/src/shared/mentions.js b/src/shared/mentions.js --- a/src/shared/mentions.js +++ b/src/shared/mentions.js diff --git a/patches/ds-bot/files/test/fixtures/usage-sample.sqlite b/patches/ds-bot/files/test/fixtures/usage-sample.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..d83479cac1e10129bdbfcfc94a47ace2b9fd786b GIT binary patch literal 12288 zcmeI$O>5LZ7zgl~X{+l(cT1tl(qH!nk>s4Lf;hX0lbU zf)~Gw_(jAo;MEV{&7+fD3WY>_6OsQglVs*$-hRnt^5Xe0wj`%nkti$o*%srRJ(Q9$ zCR|x_qoFUCP4Y%t$~nE2f&VIy$K%6igYk)c z>YvCibB#TJbo9A&F)mwGE zXdXXK^U^lXJW-Pwg$1eT58<<-Ix06UJS*+*MxLo6brAJc#>SKs<_(**S*F94Z_{LF z?u~?rsV$_vyDP4E4?NDC3cWF>+ATL!rOlT0)rjGp<-paszEyouo38NgC+0E|1Rwwb z2tWV=5P$##AOHafKmY>&gupT1>i0H!y`7feEpdH^3!{=;)BXOv+xup3RpB4MexI?3 zC^tm&#ofKBR%T|)B3=I+@gSn9Dz&Y8h>dS;w=drP!CXdy00bZa0SG_<0uX=z1Rwwb R2teR}5$LS-uU+_Q@C!2;wgdnG literal 0 HcmV?d00001 diff --git a/patches/ds-bot/files/test/usage-fields.test.js b/patches/ds-bot/files/test/usage-fields.test.js index 5b6f32050..099f72d5a 100644 --- a/patches/ds-bot/files/test/usage-fields.test.js +++ b/patches/ds-bot/files/test/usage-fields.test.js @@ -1,10 +1,15 @@ import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import { test } from 'node:test' -import { foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' +import { extractIssuePr, foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) const usage = (inputTokens, outputTokens, cacheReadTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens }) +const here = dirname(fileURLToPath(import.meta.url)) test('withUsageFields fills harness, issue·PR, toolCalls, wallMs, and keeps tokens', () => { const row = withUsageFields({ input: 1, cacheRead: 0, cacheWrite: 0, output: 2 }) @@ -26,11 +31,45 @@ test('missing any of the five usage fields is red', () => { assert.deepEqual(missingUsageFields({ ...base, input: undefined }), ['tokens']) }) -test('foldCalls rows carry the five usage fields', () => { +test('foldCalls counts tool/call events and wall clock from the attempt', () => { const { rows } = foldCalls([ + { type: 'turn/start', time: 1000, data: { turn: 1 } }, header('deepseek', 'v4-flash'), - message(1, 0, usage(10, 2), 0), + { type: 'user', time: 1001, data: { text: 'please implement #903 Step 0' } }, + { type: 'tool/call', time: 1100, data: { name: 'read_file' } }, + { type: 'tool/call', time: 1200, data: { name: 'edit_file' } }, + message(1, 0, usage(10, 2), 1500), ]) assert.equal(rows.length, 1) assert.deepEqual(missingUsageFields(rows[0]), []) + assert.equal(rows[0].toolCalls, 2) + assert.equal(rows[0].wallMs, 500) + assert.equal(rows[0].issuePr, '#903') +}) + +test('extractIssuePr prefers PR refs over bare issue numbers', () => { + assert.equal(extractIssuePr('see PR #908 for #903'), 'PR #908') + assert.equal(extractIssuePr('https://github.com/ThinkFlowLab/FlyBot/issues/903'), '#903') +}) + +test('fixture usage.sqlite has one real calls row with all five fields', () => { + const dbPath = join(here, 'fixtures', 'usage-sample.sqlite') + const script = ` +import sqlite3, sys +db = sqlite3.connect(sys.argv[1]) +row = db.execute( + "SELECT harness, issue_pr, tool_calls, wall_ms, input, output FROM calls ORDER BY time DESC LIMIT 1" +).fetchone() +assert row is not None, "no calls row" +harness, issue_pr, tool_calls, wall_ms, input_tokens, output = row +assert harness == "default", harness +assert isinstance(issue_pr, str) and issue_pr != "", issue_pr +assert tool_calls >= 1, tool_calls +assert wall_ms >= 1, wall_ms +assert input_tokens >= 1 and output >= 1, (input_tokens, output) +print("|".join(map(str, row))) +` + const result = spawnSync('python3', ['-c', script, dbPath], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr || result.stdout) + assert.match(result.stdout.trim(), /^default\|#903\|/) }) From 6721c0b7cfdb46cff568e557ae4f46170b56c9f7 Mon Sep 17 00:00:00 2001 From: hsliuustc0106 Date: Sat, 24 Oct 2026 09:40:51 +0800 Subject: [PATCH 3/4] docs(core-host): document default harness + usage ledger v3 (#903) arch-docs: core/index.js, harness.js, and usage.js changed for Step 0. --- docs/subsystems/core-host.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 6838bf6e8..2f4d6aa53 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -45,6 +45,8 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | `core/src/host/schedules.js` | Scheduled tasks through DSH's schedule service: list, create, edit, pause/resume, retry and delete for Settings → Scheduled tasks, each task's last run, and moving a Bot's tasks to its fresh part (#354, #473, see **Scheduled tasks: edit, pause, missed runs** and **Scheduled tasks: last run and Retry**). Also the `bot_schedules` tool: the Bot-wide list DSH's own `schedule_list` cannot give, group chats' own Sessions included (#628, see **Scheduled tasks: the Bot-wide list**). Tests: `core/test/schedules.test.js`. | | `core/src/host/automation-default.js` | Automation tasks on by default (#378): once per profile, `offerBundle` asks DSH's plugin manager to switch on `@deepseek-ai/dsh-experimental-schedule-bundle` and writes the offer to `dsh-defaults.json` next to the team. Tests: `core/test/automation-default.test.js`. | | `core/src/host/notify.js` | Out-of-app delivery, first half (#355): a transport-agnostic adapter with Slack's `chat.postMessage` shape, one key per reminder so a retried send cannot duplicate, exponential backoff that honours `Retry-After`, three attempts at most, a structural failure not retried, and zero transport calls while the channel is off. Ships `fakeSlack` for tests; nothing calls it until the real Slack transport lands (see **Out-of-app delivery, first half**). Tests: `core/test/notify.test.js`. | +| `core/src/host/harness.js` | Step 0 declarative `default` harness (#903 / #874 Option B): names today's prompt / tools / model / step·retry / verifiers modules as one config (`rt.harness`, `rt.resolveHarness`); behavior-neutral. Tree prompt exception: `@flybot/tree` keeps its marked prompt under `plugins/tree` (outside the three mirrors); listed on the harness, injection stays outside this file. Installed in `MODULES` right after `store`. Tests: `core/test/harness.test.js`. | +| `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. | | `core/src/host/tool-schema.js` | The tool definitions the model sees: its `defineTool` wraps DSH's and completes the compiled schema with `explicitObjectShape`, which fills `properties: {}` and `required: []` on every object node (recursing through `properties`, `items`, `oneOf`/`anyOf`/`allOf`, `$defs`, …). A strict OpenAI-compatible gateway reads an absent member as null and rejects the whole request (`null is not of type "array"`), so one argument-less tool would break every turn; argument validation keeps using DSH's schema unchanged. Every host module registers tools through this wrapper. Tests: `core/test/tool-schema.test.js`. | | `core/src/shared/remote-label.js` | Owner-badge labels for a friend's Bot and “Shared room from …” titles. `friendBotTitle(bot)` / `friendBotOwner(bot)` name a friend's chat from the roster (`Chief (SelfB1233)`) and its owner (`SelfB1233`, the person an approval card names) with one formatter for every client surface (#498). `remoteBotName` / `isAutoOwnerName` / `seatOwnerName` keep group and face-room labels in that format too (#526): Bot Social's ` · ` member label is not a Bot name, and an owner name made up from an install id (`FlyBot 682b8a43`) gives way to the username in the home handle (`Reviewer (flybot682b8a43)`). `isListedRemoteChat` / `listedChatBots` keep gone, hidden, and home-level (`@something.home`) remotes out of the sidebar, the **发给** picker, mentions, and `list_bots` friends lines; history stays on disk. | @@ -52,6 +54,15 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core Add a new invariant under the heading it belongs to (or a new heading), not at the end of the page: parallel PRs then do not edit the same lines. +### Default harness and usage ledger (#903) + +Step 0 of #874 Option B is behavior-neutral: one declarative harness and five usage fields that every `calls` row must carry. + +- `src/host/harness.js` installs as `rt.harness` / `rt.resolveHarness` after `store` in `MODULES` (`core/index.js`). Today's setup is the single `default` harness: it names the existing prompt / tools / model / step·retry modules and notes that verifiers live in CI / merge-gate / `core/dev/eval`, not as an in-process Step 0 list. Unknown harness ids throw. Lite/Medium/Heavy and Magpie stay out of scope; nothing under `core/dev/harness/` is edited here. +- Tree exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` (not one of `ds-bot/` · `patches/ds-bot/` · `core/`). The default harness lists that exception; injection stays outside the harness file so eval `P3-tree-split` still passes. +- Usage ledger `calls` is version 3 (`LEDGER_VERSION`): columns include `harness`, `issue_pr`, `tool_calls`, `wall_ms` beside the existing token fields. `withUsageFields` fills defaults (`harness` → `default`, the rest empty/0); `missingUsageFields` fails a row that lacks any of the five. `foldCalls` counts `tool/call` events, computes `wallMs` from the attempt start, and takes `issuePr` from turn text (`extractIssuePr`) or from `rt.noteUsageIssuePr` when a caller tags the session. Rows without those turn signals stay `''` / `0` by design, not as a silent shell that never wires turn data. +- Fixture for acceptance: `core/test/fixtures/usage-sample.sqlite` holds one real `calls` row. Covered by `core/test/harness.test.js` and `core/test/usage-fields.test.js`. + ### Room execution boundary `groups.js` owns room scheduling, membership checks, private-brief selection, From 903a1f9137d8eba9e1908d7419c85cfd8a89e375 Mon Sep 17 00:00:00 2001 From: hsliuustc0106 Date: Sat, 24 Oct 2026 10:01:28 +0800 Subject: [PATCH 4/4] fix(#903): drop unused mkdirSync in usage-fields test Lint error blocked Node 24 / tests-ok on #908. --- core/test/usage-fields.test.js | 1 - patches/ds-bot/files/test/usage-fields.test.js | 1 - 2 files changed, 2 deletions(-) diff --git a/core/test/usage-fields.test.js b/core/test/usage-fields.test.js index 099f72d5a..79510fae2 100644 --- a/core/test/usage-fields.test.js +++ b/core/test/usage-fields.test.js @@ -1,6 +1,5 @@ import assert from 'node:assert/strict' import { spawnSync } from 'node:child_process' -import { mkdirSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { test } from 'node:test' diff --git a/patches/ds-bot/files/test/usage-fields.test.js b/patches/ds-bot/files/test/usage-fields.test.js index 099f72d5a..79510fae2 100644 --- a/patches/ds-bot/files/test/usage-fields.test.js +++ b/patches/ds-bot/files/test/usage-fields.test.js @@ -1,6 +1,5 @@ import assert from 'node:assert/strict' import { spawnSync } from 'node:child_process' -import { mkdirSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { test } from 'node:test'