Step 0: default harness + usage five fields (#903)

core/index.js

@@ -4,6 +4,7 @@
 import { join } from 'node:path'
 import * as diagnostics from './src/host/diagnostics.js'
 import * as store from './src/host/store.js'
+import * as harness from './src/host/harness.js'
 import * as memberAuth from './src/host/member-auth.js'
 import * as authz from './src/host/authz.js'
 import * as roomMembership from './src/host/room-membership.js'
@@ -50,7 +51,7 @@ export const inject = ['llm', 'tools', 'systemPrompt', 'sessionController', 'ses
 // the functions other modules call; they call them through `rt` at run time, never during
 // install. Listeners run and tools are listed in registration order, so this order holds:
 // usage meters model requests after every listener that rewrites them.
-const MODULES = [diagnostics, store, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel]
+const MODULES = [diagnostics, store, harness, memberAuth, authz, models, permissions, turns, delivery, roster, folders, agents, prompt, timeContext, tools, decisions, chatIndex, groups, roomMembership, reach, tasks, approvals, resume, memory, parts, images, secrets, connectors, connectorAccess, capabilities, schedules, automationDefault, browser, usage, update, dshUpdate, hooks, remoteChat, channel]
 
 export function apply(ctx, config = {}) {
   const rt = createRuntime(ctx, config)

core/src/host/harness.js

@@ -0,0 +1,37 @@
+// Step 0 (#903 / #874 Option B): one declarative harness layer.
+// Today's setup is the single `default` harness. Behavior-neutral: the runtime still
+// runs the same modules; this file names them as one config so Step 1 can add tiers later.
+//
+// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree`
+// (not one of the three mirrors ds-bot/ · patches/ds-bot/ · core/). That injection stays
+// outside this harness file; eval `P3-tree-split` must keep passing.
+
+export const DEFAULT_HARNESS_ID = 'default'
+
+export function defaultHarness() {
+  return {
+    id: DEFAULT_HARNESS_ID,
+    prompt: { module: './prompt.js', section: 'bot-identity' },
+    tools: { module: './tools.js' },
+    model: { module: './models.js', note: 'per-Bot model on the Bot record' },
+    stepRetry: { module: './turns.js', note: 'written-tool-call retry and turn lifecycle' },
+    verifiers: { note: 'CI, merge-gate, and core/dev/eval; not an in-process Step 0 list' },
+    exceptions: [
+      {
+        id: 'tree-prompt',
+        path: 'plugins/tree',
+        note: '@flybot/tree marked prompt stays outside the default harness file',
+      },
+    ],
+  }
+}
+
+export function resolveHarness(id = DEFAULT_HARNESS_ID) {
+  if (id !== DEFAULT_HARNESS_ID) throw new Error(`unknown harness: ${id}`)
+  return defaultHarness()
+}
+
+export function install(rt) {
+  rt.harness = resolveHarness(DEFAULT_HARNESS_ID)
+  rt.resolveHarness = resolveHarness
+}

core/src/host/usage.js

@@ -14,7 +14,7 @@ import { randomUUID } from 'node:crypto'
 import { mkdir, unlink } from 'node:fs/promises'
 import { join } from 'node:path'
 
-const LEDGER_VERSION = 2
+const LEDGER_VERSION = 3
 const PAGE_MESSAGES = 400
 const PAGE_TIMEOUT_MS = 20_000
 const SCAN_TIMEOUT_MS = 180_000
@@ -45,30 +45,104 @@ const tokensOf = usage => ({
   output: count(usage.outputTokens),
 })
 
+
+// Step 0 (#903): every recorded call carries harness, issue·PR, tool-call count, wall clock,
+// and token in/out (token columns reused). Missing any of these fields fails the unit tests.
+export const USAGE_CALL_FIELDS = ['harness', 'issuePr', 'toolCalls', 'wallMs', 'input', 'output']
+
+export function withUsageFields(row, extras = {}) {
+  return {
+    ...row,
+    harness: extras.harness ?? row.harness ?? 'default',
+    issuePr: extras.issuePr ?? row.issuePr ?? '',
+    toolCalls: Number.isFinite(extras.toolCalls) ? extras.toolCalls : (Number.isFinite(row.toolCalls) ? row.toolCalls : 0),
+    wallMs: Number.isFinite(extras.wallMs) ? extras.wallMs : (Number.isFinite(row.wallMs) ? row.wallMs : 0),
+  }
+}
+
+export function missingUsageFields(row) {
+  const missing = []
+  if (typeof row?.harness !== 'string' || row.harness === '') missing.push('harness')
+  if (typeof row?.issuePr !== 'string') missing.push('issuePr')
+  if (!Number.isFinite(row?.toolCalls)) missing.push('toolCalls')
+  if (!Number.isFinite(row?.wallMs)) missing.push('wallMs')
+  if (!Number.isFinite(row?.input) || !Number.isFinite(row?.output)) missing.push('tokens')
+  return missing
+}
+
+// Prefer an explicit PR ref, then a GitHub issues/pull URL, then a bare #N in prose.
+export function extractIssuePr(text) {
+  if (typeof text !== 'string' || text === '') return ''
+  const pr = /\bPR\s*#(\d+)\b/i.exec(text) || /github\.com\/[^/\s]+\/[^/\s]+\/pull\/(\d+)/i.exec(text)
+  if (pr) return `PR #${pr[1]}`
+  const issue = /github\.com\/[^/\s]+\/[^/\s]+\/issues\/(\d+)/i.exec(text) || /(?:^|[\s(\[])#(\d+)\b/.exec(text)
+  if (issue) return `#${issue[1]}`
+  return ''
+}
+
+function textFromEvent(event) {
+  const data = event?.data ?? {}
+  const parts = []
+  if (typeof data.text === 'string') parts.push(data.text)
+  if (typeof data.message === 'string') parts.push(data.message)
+  const content = data.message?.content ?? data.content
+  if (typeof content === 'string') parts.push(content)
+  if (Array.isArray(content)) {
+    for (const block of content) {
+      if (typeof block?.text === 'string') parts.push(block.text)
+      if (typeof block?.content === 'string') parts.push(block.content)
+    }
+  }
+  return parts.join('\n')
+}
+
+function noteIssuePr(state, event) {
+  const found = extractIssuePr(textFromEvent(event))
+  if (found) state.issuePr = found
+}
+
 // Folds Session events, oldest first, into one row per billed attempt, as DSH's own
 // tokenUsage projection counts them: a later sample of one attempt replaces the earlier
 // one, and `llm/retry-started` begins a new attempt. The model is the one the latest
 // request header named. A fork's events up to its inherited cut are the parent's, so
 // the cut drops the rows before it (`reset`).
 export function foldCalls(events, from = {}) {
-  const state = { provider: from.provider ?? '', model: from.model ?? '', retry: from.retry ?? null }
+  const state = {
+    provider: from.provider ?? '',
+    model: from.model ?? '',
+    retry: from.retry ?? null,
+    issuePr: typeof from.issuePr === 'string' ? from.issuePr : '',
+    turnTools: 0,
+    attemptStart: undefined,
+  }
   const rows = new Map()
   let reset = false
   for (const event of events) {
     const data = event?.data ?? {}
+    noteIssuePr(state, event)
     switch (event?.type) {
       case 'request/header': {
         const config = data.header?.config
         if (config?.provider && config?.model) Object.assign(state, { provider: config.provider, model: config.model })
+        if (state.attemptStart === undefined && Number.isFinite(event.time)) state.attemptStart = event.time
         break
       }
       case 'request/context':
         if (data.provider && data.model) Object.assign(state, { provider: data.provider, model: data.model })
         break
+      case 'turn/start':
+        state.turnTools = 0
+        state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart
+        break
+      case 'tool/call':
+        // Same events turns.js counts for the inbox (#471); PTC sub-calls are not tool/call.
+        state.turnTools += 1
+        break
       case 'llm/retry-started':
         state.retry = state.retry?.turn === data.turn && state.retry.step === data.step
           ? { ...state.retry, count: state.retry.count + 1 }
           : { turn: data.turn, step: data.step, count: 1 }
+        state.attemptStart = Number.isFinite(event.time) ? event.time : state.attemptStart
         break
       case 'session/end-seed':
         if (data.inherited === true) {
@@ -82,14 +156,30 @@ export function foldCalls(events, from = {}) {
         if (usage === undefined || usage === null) break
         const attempt = state.retry?.turn === data.turn && state.retry.step === data.step ? state.retry.count : 0
         const key = `${data.turn}:${data.step}:${attempt}`
-        rows.set(key, { key, time: Number.isFinite(event.time) ? event.time : 0, provider: state.provider, model: state.model, ...tokensOf(usage) })
+        const time = Number.isFinite(event.time) ? event.time : 0
+        const wallMs = state.attemptStart !== undefined ? Math.max(0, time - state.attemptStart) : 0
+        rows.set(key, withUsageFields({
+          key,
+          time,
+          provider: state.provider,
+          model: state.model,
+          ...tokensOf(usage),
+          toolCalls: state.turnTools,
+          wallMs,
+          issuePr: state.issuePr,
+        }))
         break
       }
       default:
         break
     }
   }
-  return { rows: [...rows.values()], state, reset }
+  return {
+    rows: [...rows.values()],
+    // Resume fields only — marks.fold must stay compatible with pre-Step-0 ledgers.
+    state: { provider: state.provider, model: state.model, retry: state.retry },
+    reset,
+  }
 }
 
 // One aggregate per Session, model, use, and quarter hour.
@@ -144,14 +234,14 @@ function sqliteLedger(DatabaseSync, path) {
         DROP TABLE IF EXISTS marks;
         DROP TABLE IF EXISTS owners;
         DROP TABLE IF EXISTS kinds;
-        CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL);
+        CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL, harness TEXT NOT NULL, issue_pr TEXT NOT NULL, tool_calls INTEGER NOT NULL, wall_ms INTEGER NOT NULL);
         CREATE INDEX calls_session ON calls (session);
         CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, fold TEXT NOT NULL, sealed INTEGER NOT NULL);
         CREATE TABLE owners (session TEXT PRIMARY KEY, owner TEXT NOT NULL, name TEXT NOT NULL);
         CREATE TABLE kinds (session TEXT PRIMARY KEY, kind TEXT NOT NULL);
         PRAGMA user_version = ${LEDGER_VERSION};`)
     }
-    const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+    const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output, harness, issue_pr, tool_calls, wall_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
     const dropLogged = db.prepare('DELETE FROM calls WHERE session = ? AND live = 0')
     const getMark = db.prepare('SELECT through, fold, sealed FROM marks WHERE session = ?')
     const setMark = db.prepare('INSERT OR REPLACE INTO marks (session, through, fold, sealed) VALUES (?, ?, ?, ?)')
@@ -160,7 +250,10 @@ function sqliteLedger(DatabaseSync, path) {
     const allKinds = db.prepare('SELECT session, kind FROM kinds')
     const putKind = db.prepare('INSERT OR REPLACE INTO kinds (session, kind) VALUES (?, ?)')
     const grouped = db.prepare(`SELECT session, provider, model, kind, time / ${SLOT_MS} AS slot, SUM(input) AS input, SUM(cache_read) AS cacheRead, SUM(cache_write) AS cacheWrite, SUM(output) AS output, COUNT(*) AS calls FROM calls GROUP BY session, provider, model, kind, slot`)
-    const insert = (row, live) => put.run(row.key, row.session, row.time, row.provider, row.model, row.kind, live ? 1 : 0, row.input, row.cacheRead, row.cacheWrite, row.output)
+    const insert = (row, live) => {
+      const full = withUsageFields(row)
+      put.run(full.key, full.session, full.time, full.provider, full.model, full.kind, live ? 1 : 0, full.input, full.cacheRead, full.cacheWrite, full.output, full.harness, full.issuePr, full.toolCalls, full.wallMs)
+    }
     return {
       mark(session) {
         const row = getMark.get(session)
@@ -233,6 +326,15 @@ export function install(rt) {
   let opening
   // Metered calls that finish before the ledger opens.
   const early = []
+  // Optional issue/PR stamp for a Session (automation / explicit note). Fold and live
+  // meter rows pick it up when the log itself does not name one.
+  const issuePrBySession = new Map()
+  const noteUsageIssuePr = (sessionId, ref) => {
+    if (typeof sessionId !== 'string' || sessionId === '') return
+    const normalized = extractIssuePr(typeof ref === 'string' ? ref : '') || (typeof ref === 'string' ? ref.trim() : '')
+    if (normalized === '') issuePrBySession.delete(sessionId)
+    else issuePrBySession.set(sessionId, normalized)
+  }
   function openLedger() {
     if (ledger !== undefined) return Promise.resolve(ledger)
     const job = opening ??= (async () => {
@@ -367,9 +469,9 @@ export function install(rt) {
     }
     events.reverse()
     if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`)
-    const fold = foldCalls(events, mark?.fold)
+    const fold = foldCalls(events, { ...(mark?.fold ?? {}), issuePr: issuePrBySession.get(sessionId) ?? '' })
     ledger.commit(sessionId, {
-      rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })),
+      rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' }, { issuePr: row.issuePr || issuePrBySession.get(sessionId) || '' })),
       reset: fold.reset,
       mark: { through, fold: fold.state, sealed: sealable },
     })
@@ -556,9 +658,17 @@ export function install(rt) {
       try { journal(book, sessionId) } catch (error) { rt.warn('ds-bot: usage could not journal %s: %s', sessionId, error) }
     })()
   }
-  function record({ sessionId, kind, provider, model, usage }) {
+  function record({ sessionId, kind, provider, model, usage, issuePr, toolCalls, wallMs, harness }) {
     noteSession(sessionId)
-    const row = { key: `live:${randomUUID()}`, session: sessionId ?? '', time: Date.now(), provider: provider ?? '', model: model ?? '', kind, ...tokensOf(usage) }
+    const row = withUsageFields({
+      key: `live:${randomUUID()}`,
+      session: sessionId ?? '',
+      time: Date.now(),
+      provider: provider ?? '',
+      model: model ?? '',
+      kind,
+      ...tokensOf(usage),
+    }, { harness, issuePr, toolCalls, wallMs })
     if (ledger !== undefined) {
       try { ledger.add(row) } catch (error) { rt.warn('ds-bot: usage could not record a call: %s', error) }
       return
@@ -579,6 +689,7 @@ export function install(rt) {
     }
     return (async function* meter() {
       let usage
+      const started = Date.now()
       try {
         for await (const chunk of next()) {
           if (chunk?.type === 'usage' && chunk.usage && typeof chunk === 'object' && !counted.has(chunk)) {
@@ -588,10 +699,20 @@ export function install(rt) {
           yield chunk
         }
       } finally {
-        if (usage !== undefined) record({ sessionId: tag?.sessionId ?? options.sessionId, kind: tag?.kind ?? options.purpose ?? 'other', provider: options.provider, model: options.model, usage })
+        if (usage !== undefined) record({
+          sessionId: tag?.sessionId ?? options.sessionId,
+          kind: tag?.kind ?? options.purpose ?? 'other',
+          provider: options.provider,
+          model: options.model,
+          usage,
+          issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(tag?.sessionId ?? options.sessionId) ?? '',
+          toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0),
+          wallMs: Math.max(0, Date.now() - started),
+          harness: tag?.harness ?? rt.harness?.id ?? 'default',
+        })
       }
     })()
   })
 
-  Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest })
+  Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr })
 }

core/test/harness.test.js

@@ -0,0 +1,23 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { DEFAULT_HARNESS_ID, defaultHarness, resolveHarness } from '../src/host/harness.js'
+
+test('Step 0 default harness names prompt, tools, model, step·retry, and verifiers', () => {
+  const harness = defaultHarness()
+  assert.equal(harness.id, DEFAULT_HARNESS_ID)
+  assert.equal(harness.prompt.module, './prompt.js')
+  assert.equal(harness.tools.module, './tools.js')
+  assert.equal(harness.model.module, './models.js')
+  assert.equal(harness.stepRetry.module, './turns.js')
+  assert.ok(harness.verifiers.note)
+})
+
+test('resolveHarness only accepts default in Step 0', () => {
+  assert.equal(resolveHarness().id, 'default')
+  assert.throws(() => resolveHarness('lite'), /unknown harness/)
+})
+
+test('tree prompt exception is listed outside the default harness file', () => {
+  const tree = defaultHarness().exceptions.find(entry => entry.id === 'tree-prompt')
+  assert.equal(tree.path, 'plugins/tree')
+})