From 0e6e00b024fb26832360a4188cb678029a8374ac Mon Sep 17 00:00:00 2001 From: zy95-12 <135135297+zy95-12@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:25:07 +0800 Subject: [PATCH 1/2] wip: establish native Chief process recovery boundaries --- core/src/host/project-agents.js | 27 ++++++++++++++++++++++++- core/src/host/project-chief-control.js | 11 +++++----- core/src/host/project-chief-recovery.js | 22 ++++++++++++++++++++ core/src/host/project-process-proof.js | 20 ++++++++++++++++-- core/test/project-agents.test.js | 8 +++++--- core/test/project-process-proof.test.js | 14 +++++++++++++ 6 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 core/src/host/project-chief-recovery.js diff --git a/core/src/host/project-agents.js b/core/src/host/project-agents.js index 000879965..16419a547 100644 --- a/core/src/host/project-agents.js +++ b/core/src/host/project-agents.js @@ -1,3 +1,5 @@ +import { captureProcessScope, priorHostProcessExited } from './project-process-proof.js' +import { nativeChiefBoundary, scopeDigest } from './project-chief-recovery.js' import { CHIEF_AUTONOMY } from './project-chief-protocol.js' import { agentHistoryPage } from './project-history-pages.js' import { createProjectAgentInbox } from './project-agent-inbox.js' @@ -367,6 +369,10 @@ export function createProjectAgents(rt) { const delegation = { id: randomUUID(), exchangeId: randomUUID(), actor: { ...actor }, definitionRevision: set.revision, started: false, revoked: false, chiefJob: { runId, invocation, prompt: fullPrompt }, onChiefResult: onResult } delegations.set(agent.id, delegation) agent.lastRequest = { id: randomUUID(), initiatedBy: actor.memberId, executorBotId: agent.botId, exchangeId: delegation.exchangeId, phase: 'queued', context: snapshot, definitionRevision: set.revision, chiefRunId: runId, chiefInvocation: invocation, at: Date.now() } + const run = state.projectExecutions?.[runId] + if (!run || run.invocation !== invocation || run.status !== 'running') throw refuse('Chief invocation changed before execution admission') + run.executionBoundary = { version: 'chief-execution-boundary/v1', kind: (definition.execution?.harness ?? 'flybot') === 'flybot' ? 'native-in-process' : 'sdk-worker', runId, invocation, botId: agent.botId, sessionId: agent.sessionId, carrierGeneration: agent.carrierGeneration, exchangeId: delegation.exchangeId } + run.processScope ??= await captureProcessScope() await rt.save() const finish = async result => { try { if (delegations.get(agent.id)?.id !== delegation.id || delegation.revoked) throw refuse('Chief execution was revoked'); const outcome = await onResult({ ...result, executionEvidence: { ...result.executionEvidence, chiefRunId: runId, carrierGeneration: agent.carrierGeneration } }); agent.lastRequest.phase = outcome?.correction ? outcome.correction.status === 'pending' ? 'correcting' : 'incomplete' : result.ok ? 'completed' : 'failed' } @@ -624,6 +630,25 @@ export function createProjectAgents(rt) { return serial(() => { if (expectedCarrierId && keyOf(projectId, memberId) !== expectedCarrierId) throw refuse('Personal-agent default changed before notification'); return send({ projectId, visibility: 'private', text }, actor) }) } const stoppingChiefJobs = new Map() + async function reconcileChiefExit(run) { + if (!['interrupted', 'failed', 'cancelled'].includes(run.status)) return false + const selected = nativeChiefBoundary(state, run), scope = run.processScope, invocation = run.invocation + if (!selected || !await priorHostProcessExited(scope)) return false + const current = () => state.projectExecutions?.[run.id] === run && run.processScope === scope && run.invocation === invocation && ['interrupted', 'failed', 'cancelled'].includes(run.status) + if (!current()) return false + const proof = { confirmed: true, kind: 'native-chief-prior-host-exited', runId: run.id, invocation, carrierGeneration: selected.boundary.carrierGeneration, scopeSha256: scopeDigest(scope), at: Date.now() } + for (const lease of selected.leases) { + if (!current() || state.projectResourceLeases?.[lease.id] !== lease) return false + await rt.projectResources.release(lease.id, lease.generation, { handle: selected.boundary.exchangeId, exited: true, proof }) + if (lease.status !== 'released') return false + } + if (!current()) return false + run.exitProof = proof + const carrier = agents()[`public-${run.projectId}`], request = carrier?.lastRequest + if (request?.chiefRunId === run.id && request.chiefInvocation === invocation && carrier.carrierGeneration === proof.carrierGeneration) { request.phase = 'interrupted'; request.exitProof = structuredClone(proof) } + await rt.save() + return true + } function stopChiefJob(projectId, runId) { if (stoppingChiefJobs.has(runId)) return stoppingChiefJobs.get(runId) const pending = (async () => { @@ -803,7 +828,7 @@ export function createProjectAgents(rt) { } const isDelegationActive = (id, delegationId) => { const entry = delegations.get(id); return entry?.id === delegationId && !entry.revoked } const ensureProject = (projectId, actor) => serial(async () => { await ensure(projectId, actor, 'public'); return ensure(projectId, actor) }) - return { stream, liveFor, hasExchange: exchangeId => [...delegations.values()].some(row => row.exchangeId === exchangeId && !row.revoked), privateIdentity: keyOf, ensurePublic: (projectId, actor) => serial(() => ensure(projectId, actor, 'public')), contextExchange, chiefMemory, chiefBusy: projectId => delegations.has(`public-${projectId}`) || [...stoppingChiefJobs.keys()].some(id => state.projectExecutions?.[id]?.projectId === projectId), stopChiefJob, startChiefJob, endpoint, consultation: consultations.endpoint, contactUser: consultations.contactUser, agentConsultations, agentInbox, inboxExecutionActive, chiefQuestion, askChief, suggestClosure, githubRead, action, wakeWatch, revokeWatchGrant, ensureProject, isDelegationActive, actorOf, context, memory, task, run, onEvent, revoke, revokeSession, removeProject, deliveryDefinitions, recordForBot, roles: roleGovernance, proposeRole, proposeRoleForJob, ROLES } + return { stream, liveFor, hasExchange: exchangeId => [...delegations.values()].some(row => row.exchangeId === exchangeId && !row.revoked), privateIdentity: keyOf, ensurePublic: (projectId, actor) => serial(() => ensure(projectId, actor, 'public')), contextExchange, chiefMemory, reconcileChiefExit, chiefBusy: projectId => delegations.has(`public-${projectId}`) || [...stoppingChiefJobs.keys()].some(id => state.projectExecutions?.[id]?.projectId === projectId), stopChiefJob, startChiefJob, endpoint, consultation: consultations.endpoint, contactUser: consultations.contactUser, agentConsultations, agentInbox, inboxExecutionActive, chiefQuestion, askChief, suggestClosure, githubRead, action, wakeWatch, revokeWatchGrant, ensureProject, isDelegationActive, actorOf, context, memory, task, run, onEvent, revoke, revokeSession, removeProject, deliveryDefinitions, recordForBot, roles: roleGovernance, proposeRole, proposeRoleForJob, ROLES } } export function install(rt) { diff --git a/core/src/host/project-chief-control.js b/core/src/host/project-chief-control.js index 2994f1f5d..5c524fcc7 100644 --- a/core/src/host/project-chief-control.js +++ b/core/src/host/project-chief-control.js @@ -308,19 +308,20 @@ export function createProjectChiefControl(rt) { } finally { if (run.invocation !== job.invocation) return; run.lastInvocationFinishedAt = Date.now(); if (!correcting) run.finishedAt = Date.now(); if (active.get(run.id) === job) active.delete(run.id); if (!correcting) waiters.delete(run.id); await rt.save(); if (correcting) rt.projectScheduler?.wake() } } async function inspectionExited(run) { + await rt.projectAgents.reconcileChiefExit?.(run) const workers = Object.values(rt.state.projectHarnessWorkers ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined ? row.id === run.workerId : row.chiefInvocation === run.invocation)) const leases = Object.values(rt.state.projectResourceLeases ?? {}).filter(row => row.chiefRunId === run.id && row.chiefInvocation === run.invocation || workers.some(worker => worker.leaseId === row.id)) if (leases.some(row => row.status !== 'released')) return false if (workers.length) return workers.every(row => confirmedWorkerExit(rt.state, row) && leases.some(lease => lease.id === row.leaseId && lease.projectId === row.projectId && lease.attemptId === row.attemptId && lease.role === row.role && lease.resourceId === row.resourceId && lease.generation === row.generation && lease.worker?.handle === row.id)) - if (leases.length && leases.every(row => row.releaseProof && (!row.worker && row.releaseProof.kind === 'unbound-reservation-release' || row.worker?.kind === 'native-turn' && ['host-confirmed-worker-exit', 'systemd-prior-invocation-exited'].includes(row.releaseProof.kind)))) return true + if (leases.length && leases.every(row => row.releaseProof && (!row.worker && row.releaseProof.kind === 'unbound-reservation-release' || row.worker?.kind === 'native-turn' && ['host-confirmed-worker-exit', 'systemd-prior-invocation-exited', 'native-chief-prior-host-exited'].includes(row.releaseProof.kind)))) return true const agent = rt.state.projectAgents?.[`public-${run.projectId}`], proof = agent?.lastRequest?.exitProof if (proof?.confirmed && proof.runId === run.id && agent.lastRequest.chiefRunId === run.id && proof.carrierGeneration === agent.carrierGeneration && proof.invocation === run.invocation) return true - if (['native-chief-quiescent', 'sdk-chief-confirmed-exit', 'chief-never-dispatched'].includes(run.exitProof?.kind) && run.exitProof?.confirmed && run.exitProof.runId === run.id && run.exitProof.invocation === run.invocation) return true + if (['native-chief-quiescent', 'native-chief-prior-host-exited', 'sdk-chief-confirmed-exit', 'chief-never-dispatched'].includes(run.exitProof?.kind) && run.exitProof?.confirmed && run.exitProof.runId === run.id && run.exitProof.invocation === run.invocation) return true return Boolean(await priorScopeExited(run.processScope)) } async function inspectionBarrier(current) { for (const run of Object.values(rt.state.projectExecutions ?? {})) { - if (run.id === current.id || run.projectId !== current.projectId || run.kind !== 'chief-control' || run.status !== 'interrupted' || !run.startedAt) continue + if (run.id === current.id || run.projectId !== current.projectId || run.kind !== 'chief-control' || run.status !== 'interrupted' && run.recovery?.exitConfirmed !== false || !run.startedAt) continue if (!await inspectionExited(run)) return true } return false @@ -420,7 +421,7 @@ export function createProjectChiefControl(rt) { const task = rt.state.tasks[run.taskId], policy = policyOf(run.projectId) if (!policy?.enabled || task?.collaboration?.claim?.chiefRunId !== run.id || task.collaboration.claim.status !== 'interrupted') continue const worker = rt.state.projectHarnessWorkers?.[run.workerId] - if (worker?.status !== 'released' && !await priorScopeExited(run.processScope)) continue + if (!await inspectionExited(run)) continue try { await rt.projectPools.retryChief(task.id, { memberId: policy.authorityMemberId }); run.recovery = { status: 'requeued', at: Date.now() }; await rt.save() } catch (error) { run.recovery = { status: 'blocked', reason: error.message, at: Date.now() } } } for (const task of Object.values(rt.state.tasks)) if (task.collaboration?.status === 'arbitration' && !terminal.has(task.status) && policyOf(task.projectId)?.enabled) await request(task.projectId, 'arbitration', { handoffId: task.collaboration.currentHandoffId }, task.id) @@ -431,7 +432,7 @@ export function createProjectChiefControl(rt) { if (!policy?.enabled || policy.revision !== run.policyRevision) { run.status = 'cancelled'; waiters.get(run.id)?.reject(refuse('Chief policy changed before execution')); waiters.delete(run.id); await rt.save(); continue } const actor = { memberId: policy.authorityMemberId, chiefJobId: run.id, chiefToken: randomUUID() } const job = { run, actor, token: actor.chiefToken, snapshot: [], invocation: (run.invocation ?? 0) + 1 } - run.invocation = job.invocation + run.invocation = job.invocation; run.executionBoundary = null; delete run.workerId; delete run.workerGeneration run.status = 'running'; run.waitingFor = 'preparing'; run.startedAt = Date.now(); active.set(run.id, job); await rt.save() void dispatchJob(job).catch(error => rt.warn?.('Chief dispatch result could not be saved: %s', error.message)) } diff --git a/core/src/host/project-chief-recovery.js b/core/src/host/project-chief-recovery.js new file mode 100644 index 000000000..99419ae91 --- /dev/null +++ b/core/src/host/project-chief-recovery.js @@ -0,0 +1,22 @@ +// Recovery facts and boundaries; no business role selection or second dispatcher. +import { createHash } from 'node:crypto' +export const scopeDigest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex') +export function chiefRecoveryError(kind, message, { retryable = false, conditions = [], actions = [], ...facts } = {}) { + return Object.assign(new Error(message), { code: 'bot/chief-recovery', recovery: { version: 'chief-recovery/v1', kind, retryable, conditions, actions, ...facts } }) +} +export function nativeChiefBoundary(state, run) { + const boundary = run.executionBoundary + if (boundary) { + if (boundary.version !== 'chief-execution-boundary/v1' || boundary.kind !== 'native-in-process' || boundary.runId !== run.id || boundary.invocation !== run.invocation || !boundary.botId || !boundary.sessionId || !Number.isSafeInteger(boundary.carrierGeneration)) return null + } else if (boundary !== undefined || run.workerId !== `native-chief-job-${run.id}`) return null + // A native host proof must never hide an independent worker, including a + // worker registered before its handle was copied into the Chief Run. + const workers = Object.values(state.projectHarnessWorkers ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined || row.chiefInvocation === run.invocation)) + if (workers.length) return null + const leases = Object.values(state.projectResourceLeases ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined || row.chiefInvocation === run.invocation)) + const carrier = state.projectAgents?.[`public-${run.projectId}`], request = carrier?.lastRequest + const selected = boundary ?? (request?.chiefRunId === run.id && request.chiefInvocation === run.invocation && !request.workerId ? { version: 'chief-execution-boundary/v1', kind: 'native-in-process', runId: run.id, invocation: run.invocation, botId: carrier.botId, sessionId: carrier.sessionId, carrierGeneration: carrier.carrierGeneration, exchangeId: request.exchangeId } : null) + if (!selected || !Number.isSafeInteger(run.invocation) || run.invocation < 1) return null + if (leases.some(row => row.projectId !== run.projectId || row.chiefInvocation !== run.invocation || row.worker?.kind !== 'native-turn' || row.worker.botId !== selected.botId || row.worker.sessionId !== selected.sessionId || row.worker.handle !== selected.exchangeId || row.preparation?.status === 'active')) return null + return { boundary: selected, leases } +} diff --git a/core/src/host/project-process-proof.js b/core/src/host/project-process-proof.js index d641c943b..a0f24a690 100644 --- a/core/src/host/project-process-proof.js +++ b/core/src/host/project-process-proof.js @@ -1,15 +1,17 @@ // A manager lifecycle boundary can prove exit after restart; a missing PID alone cannot. -import { readFile } from 'node:fs/promises' +import { readFile, readlink } from 'node:fs/promises' import { execFile } from 'node:child_process' import { promisify } from 'node:util' const exec = promisify(execFile) const read = path => readFile(path, 'utf8') +const namespace = () => readlink('/proc/self/ns/pid') const manager = async unit => (await exec('/usr/bin/systemctl', ['show', unit, '--property=InvocationID,KillMode,ControlGroup'], { timeout: 2000, maxBuffer: 8192, env: { PATH: '/usr/bin:/bin', LANG: 'C' } })).stdout -export async function captureProcessScope({ readText = read, inspectManager = manager, environment = process.env, pid = process.pid } = {}) { +export async function captureProcessScope({ readText = read, inspectManager = manager, environment = process.env, pid = process.pid, readNamespace = namespace } = {}) { const base = { kind: 'unmanaged-process', hostPid: pid } if (process.platform !== 'linux') return base try { const [boot, cgroups, stat] = await Promise.all([readText('/proc/sys/kernel/random/boot_id'), readText('/proc/self/cgroup'), readText(`/proc/${pid}/stat`)]) + base.pidNamespace = await readNamespace() base.bootId = boot.trim(); base.startTicks = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] const group = cgroups.split('\n').find(line => line.startsWith('0::'))?.slice(3), unit = group?.split('/').find(part => /^[A-Za-z0-9_.@-]+\.service$/.test(part)) if (!unit || !/^[a-f0-9]{32}$/.test(environment.INVOCATION_ID ?? '')) return base @@ -28,3 +30,17 @@ export function supervisorProofMatches(scope, proof) { return Boolean(scope?.kind === 'systemd-control-group' && scope.killMode === 'control-group' && /^[a-f0-9]{32}$/.test(scope.invocationId ?? '') && proof?.kind === 'systemd-prior-invocation-exited' && proof.previousInvocationId === scope.invocationId) } + +// Proves only the old host process exited. NEVER use this as SDK/container, +// native role tool subprocess or resource preparation exit confirmation. +export async function priorHostProcessExited(scope, { readText = read, readNamespace = namespace } = {}) { + if (!['unmanaged-process', 'systemd-control-group'].includes(scope?.kind) || !Number.isSafeInteger(scope.hostPid) || scope.hostPid < 1 || !/^[a-f0-9-]{36}$/.test(scope.bootId ?? '') || !/^\d+$/.test(scope.startTicks ?? '') || !/^pid:\[\d+\]$/.test(scope.pidNamespace ?? '')) return false + try { if ((await readText('/proc/sys/kernel/random/boot_id')).trim() !== scope.bootId || await readNamespace() !== scope.pidNamespace) return false } catch { return false } + try { + const stat = await readText(`/proc/${scope.hostPid}/stat`), end = stat.lastIndexOf(')') + if (end < 0 || stat.slice(0, stat.indexOf(' ')) !== String(scope.hostPid)) return false + const fields = stat.slice(end + 2).split(' ') + if (!/^\d+$/.test(fields[19] ?? '')) return false + return fields[19] !== scope.startTicks || fields[0] === 'Z' || fields[0] === 'X' + } catch (error) { return error.code === 'ENOENT' } +} diff --git a/core/test/project-agents.test.js b/core/test/project-agents.test.js index 557756505..19e4ee876 100644 --- a/core/test/project-agents.test.js +++ b/core/test/project-agents.test.js @@ -568,7 +568,8 @@ test('dormant native end cannot consume an SDK Chief job before worker registrat assert.equal(rt.projectAgents.isDelegationActive(input.agent.id, input.delegation.id), true) return 'delayed-sdk-worker' } - await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'chief-window', prompt: 'Inspect', projectFacts: {}, onResult: async () => { callbacks++ } }) + rt.state.projectExecutions['chief-window'] = { id: 'chief-window', projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, invocation: 1, runId: 'chief-window', prompt: 'Inspect', projectFacts: {}, onResult: async () => { callbacks++ } }) await request.onResult({ ok: true, text: '{}' }) assert.equal(callbacks, 1) assert.equal(rt.state.projectAgents[original.public.id].lastRequest.phase, 'completed') @@ -807,7 +808,8 @@ test('resident native Chief receives fresh job facts in each turn instead of rel rt.deliver = async args => { sends.push({ ...args, system: rt.botOf(chief.botId).systemPrompt }); return originalDeliver(args) } for (const index of [1, 2]) { let completed = false - await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: `chief-${index}`, prompt: 'Judge only this current job', projectFacts: { tasks: [{ taskId: `task-${index}`, title: `FRESH_JOB_${index}` }] }, onResult: async () => { completed = true } }) + rt.state.projectExecutions[`chief-${index}`] = { id: `chief-${index}`, projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, invocation: 1, runId: `chief-${index}`, prompt: 'Judge only this current job', projectFacts: { tasks: [{ taskId: `task-${index}`, title: `FRESH_JOB_${index}` }] }, onResult: async () => { completed = true } }) const sent = sends.at(-1) assert.match(sent.text, new RegExp(`FRESH_JOB_${index}`)) assert.match(sent.text, /Judge only this current job/) @@ -934,7 +936,7 @@ test('native Chief deadline fencing waits for actual idleness and preserves the const { rt } = await setup(t) const { public: chief } = await get(rt, alice) rt.projectChiefControl.validateActor = async () => {} - rt.state.projectExecutions.deadline = { projectId: 'p1' } + rt.state.projectExecutions.deadline = { id: 'deadline', projectId: 'p1', invocation: 1, status: 'running' } await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'deadline', invocation: 1, prompt: 'Bounded inspection', projectFacts: {}, onResult: async () => {} }) let idle, cancelled = false rt.resolveAgent = async () => ({ inbox: {}, cancel() { cancelled = true }, whenIdle: () => new Promise(resolve => { idle = resolve }) }) diff --git a/core/test/project-process-proof.test.js b/core/test/project-process-proof.test.js index 0c764c703..45d6e266e 100644 --- a/core/test/project-process-proof.test.js +++ b/core/test/project-process-proof.test.js @@ -16,3 +16,17 @@ test('PID disappearance, changed host boot, remote resources or changed kill pol assert.equal(await priorScopeExited(original, { ...fixture(changed), readText: async path => path.endsWith('boot_id') ? 'different-boot' : fixture(changed).readText(path) }), false) assert.equal(await priorScopeExited({ kind: 'remote-resource' }, fixture(changed)), false) }) + +test('birth-checked unmanaged host exit is distinct from independent worker/process-group exit', async () => { + const { priorHostProcessExited } = await import('../src/host/project-process-proof.js') + const scope = { kind: 'unmanaged-process', hostPid: 123, bootId: 'a'.repeat(8) + '-aaaa-aaaa-aaaa-' + 'a'.repeat(12), startTicks: '900', pidNamespace: 'pid:[123]' } + const readText = async path => path.endsWith('boot_id') ? scope.bootId : `${scope.hostPid} (node) ${['S', ...Array(18).fill('0'), '900'].join(' ')}` + const readNamespace = async () => scope.pidNamespace + assert.equal(await priorHostProcessExited(scope, { readText, readNamespace }), false) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async path => path.endsWith('boot_id') ? scope.bootId : readText(path).then(text => text.replace('900', '901')) }), true) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async path => { if (path.endsWith('boot_id')) return scope.bootId; throw Object.assign(new Error('gone'), { code: 'ENOENT' }) } }), true) + for (const code of ['EACCES', 'EPERM', 'EIO']) assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async () => { throw Object.assign(new Error('unknown'), { code }) } }), false) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async () => 'other-boot' }), false) + assert.equal(await priorHostProcessExited({ ...scope, startTicks: null }, { readText, readNamespace }), false) + assert.equal(await priorScopeExited(scope), false, 'Host death is never a generic resource exit proof') +}) From 7b036d63bf6ff64236d0c129a32f0cce7cb84f42 Mon Sep 17 00:00:00 2001 From: zy95-12 <135135297+zy95-12@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:44:13 +0800 Subject: [PATCH 2/2] fix: recover Chief inspection boundaries and sealed verification routes (#2347) --- PROJECT.md | 60 +--- ROADMAP.md | 6 + changelog.d/2347.fixed.md | 1 + core/src/host/hooks.js | 2 + core/src/host/project-agents.js | 31 +- core/src/host/project-chief-control.js | 62 ++-- core/src/host/project-chief-lifecycle.js | 9 +- core/src/host/project-chief-recovery.js | 29 ++ core/src/host/project-chief-schedule.json | 4 +- core/src/host/project-delivery.js | 39 ++- core/src/host/project-development-service.js | 2 +- core/src/host/project-pools.js | 19 +- core/src/host/project-service-views.js | 1 + core/test/project-agents.test.js | 23 ++ core/test/project-chief-control.test.js | 41 +++ core/test/project-chief-lifecycle.test.js | 9 +- core/test/project-chief-recovery.test.js | 37 ++ core/test/project-delivery.test.js | 43 +++ dev/demo/chief-restart.mjs | 45 +++ dev/demo/run.mjs | 31 +- dev/demo/test/runtime.test.mjs | 12 + docs/chief-recovery.md | 107 ++++++ ...2026-11-04-role-convergence-checkpoints.md | 51 +++ docs/openapi/flybot.openapi.json | 326 +++++++++++++++--- docs/subsystems/core-host.md | 4 + docs/subsystems/plugin-runtime.md | 3 + docs/subsystems/project-agents.md | 4 + docs/subsystems/project-chief-control.md | 7 + docs/subsystems/project-delivery.md | 5 + .../subsystems/project-development-service.md | 3 + docs/subsystems/project-harness.md | 4 + docs/subsystems/project-pools.md | 4 + patches/ds-bot/files/src/host/hooks.js | 2 + .../ds-bot/files/src/host/project-agents.js | 46 ++- .../files/src/host/project-chief-control.js | 73 ++-- .../files/src/host/project-chief-lifecycle.js | 9 +- .../files/src/host/project-chief-recovery.js | 51 +++ .../src/host/project-chief-schedule.json | 4 +- .../ds-bot/files/src/host/project-delivery.js | 39 ++- .../src/host/project-development-service.js | 2 +- .../ds-bot/files/src/host/project-pools.js | 19 +- .../files/src/host/project-process-proof.js | 20 +- .../files/src/host/project-service-views.js | 1 + .../ds-bot/files/test/project-agents.test.js | 31 +- .../files/test/project-chief-control.test.js | 41 +++ .../test/project-chief-lifecycle.test.js | 9 +- .../files/test/project-chief-recovery.test.js | 37 ++ .../files/test/project-delivery.test.js | 43 +++ .../files/test/project-process-proof.test.js | 14 + scripts/api-contracts.mjs | 13 +- 50 files changed, 1274 insertions(+), 204 deletions(-) create mode 100644 changelog.d/2347.fixed.md create mode 100644 core/test/project-chief-recovery.test.js create mode 100644 dev/demo/chief-restart.mjs create mode 100644 docs/chief-recovery.md create mode 100644 docs/history/2026-11-04-role-convergence-checkpoints.md create mode 100644 patches/ds-bot/files/src/host/project-chief-recovery.js create mode 100644 patches/ds-bot/files/test/project-chief-recovery.test.js diff --git a/PROJECT.md b/PROJECT.md index 19136834d..4c7d7d1f8 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -231,51 +231,17 @@ a snapshot handoff, not a new main-targeting implementation PR or an upstream compatibility PASS. Runtime/private configuration and the frontend flow are untouched. -### Task-role execution convergence — 2026-11-03 (#2184) - -Task-role SDK retries retain pinned public persona/skills and the latest observed -backend context across Attempts. Dynamic input is not a definition identity. A -binding incompatibility or unknown old execution fails visibly; no silent fresh -Session or rollback to older context. Definition snapshots are shared by digest, -current rights still revoke execution, failed unused preparations release their -pins, and model input follows a durable-save/cancellation barrier. Existing -Harness/lease/Workflow services remain authoritative. Adapter capability inspection -reports exact native Task-role resume with committed activation/quiescence; actual DSH deterministic return/context coverage is recorded separately from real-provider validation. No -public service protocol, SQL transaction, deployment or frontend change is claimed. - - -### Collaboration contract convergence — 2026-11-03 (#2175) - -Expose durable Task handoffs through one scoped contract projection, with distinct -pending/accepted/processed/revoked/invalidated states, source/target context references, -actual claim/worker binding, initiative fences and predecessor/successor history. -Idempotent trusted offer/accept ports reuse pools; unresolved old writers cannot be -bypassed by Chief rerouting to another resource. Consultation is a separate private -protocol over retained transcripts, without Task creation or initiative transfer. -Chief prefers an active personal agent for user questions: safe answers require user -publication, watch answers use existing inbox rights; only committed Effects may wake -an authorized native watch. External clients have isolated mailboxes, not fabricated -provider sessions. SDK process/HTTP tests and real-provider acceptance remain distinct. -Native restoration/context bounds, transaction events and parent waits stay on their -original dependent issues; no control-plane, deployment or frontend changes. - - -### Context/memory convergence — 2026-11-03 (#2177) - -Unify model/Harness context budgeting and bounded source tools across persistent -Chief and temporary roles. Unknown capacity stays explicit; full rules/current -constraints remain required, retrieval yields before overflow, and native/SDK input -is checked before delivery. Working summaries use original public role-memory -records with exact source/evidence/uncertainty, separate resident Chief topics and -rollback of uncommitted staging. Facts have source/classification/version/archive; -role-authored experience cannot declare facts or overwrite human control. Private -notes retain history under bounded views. Codex tool history has an explicit cap; -SDK compaction remains runtime-managed, without claiming aggregate usage is current -window pressure. No new memory platform, executor or timer. Native Task-role physical -restore2184 and normalized SQL storage2182 remain separate acceptance; real SDK -verification must name exact runtime source hashes, not borrow deterministic tests. - -Follow-up #2177: native resident Chief current facts are actual per-turn input with whole-input budget and final authority checks; do not rely on updating cached system prompt metadata. Preserve the original resident Session. +### Role, collaboration and context convergence + +Dated #2184/#2175/#2177 implementation checkpoints are [preserved verbatim](docs/history/2026-11-04-role-convergence-checkpoints.md). +Current contracts remain active: one logical Session per Task/role; pinned persona, +resource/profile and current delegation; actual exit before restoring context; +consultation distinct from durable initiative handoff; private user context is not +published automatically. Memory is bounded guidance, not authority or delivery +proof. Preserve immutable source/history pages, verified evidence and independent +acceptance; historical fixtures do not establish live-provider or UI completion. +See [agents](docs/subsystems/project-agents.md), [Harness](docs/subsystems/project-harness.md) +and the Project memory/context contracts for the current implementation. ### 2026-11-03 — Project Task contract (#2179) @@ -318,3 +284,7 @@ from preparation and tests. Nested Node test markers do not enter host verificat #2338 preflights input receipts before host side effects and returns correctable errors to the original role/Chief Session. Frozen proposals, bounded tool-free correction and durable receipts preserve work; stale/revoked input stays denied. + +#2347 recovers birth-proven unmanaged native Chief invocations, preserves unknown +external execution, and admits explicit sealed-candidate verification continuation +under current authority/context gates. No live rollout or unknown-exit override. diff --git a/ROADMAP.md b/ROADMAP.md index 620cce81e..54835d439 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -178,3 +178,9 @@ not live npm ci provisioning or production Vitest acceptance. with all-plan preflight, immutable business evidence and explicit incomplete recovery. Tests distinguish deterministic role transport and actual SDK execution with a local provider fixture from paid-model/live project acceptance. + +#2347 adds bounded inspection/stop reconciliation, explicit recovery facts and +same-Attempt Tester/Reviewer continuation. Real disposable unmanaged DSH restart +reaches a later original scheduled inspection with deterministic inference; paid +model/live-instance acceptance stays separate. Legacy missing namespace evidence +requires exact operator corroboration, never a guessed worker release. diff --git a/changelog.d/2347.fixed.md b/changelog.d/2347.fixed.md new file mode 100644 index 000000000..1c5d790f7 --- /dev/null +++ b/changelog.d/2347.fixed.md @@ -0,0 +1 @@ +Recover proven unmanaged native Chief invocations, bound exit waits, expose recovery causes, and allow evidence-supported Tester/Reviewer continuation of a sealed interrupted Attempt. diff --git a/core/src/host/hooks.js b/core/src/host/hooks.js index 4c5580b71..79c79a35c 100644 --- a/core/src/host/hooks.js +++ b/core/src/host/hooks.js @@ -1432,6 +1432,8 @@ export function install(rt) { ...(attempt.pr ? [{ pr: attempt.pr.number, sha: attempt.pr.sha, checks: 'unknown' }] : []), ] }, + validateRecovery: input => rt.projectDelivery.validateRecovery(input), + resumeVerification: input => rt.projectDelivery.resumeVerification(input), complete: input => rt.projectDelivery.completePoolRole(input), rework: input => rt.projectDelivery.reworkPool(input), }) diff --git a/core/src/host/project-agents.js b/core/src/host/project-agents.js index 16419a547..055b6a100 100644 --- a/core/src/host/project-agents.js +++ b/core/src/host/project-agents.js @@ -1,5 +1,6 @@ +import { inspectionLifecycle } from './project-chief-lifecycle.js' import { captureProcessScope, priorHostProcessExited } from './project-process-proof.js' -import { nativeChiefBoundary, scopeDigest } from './project-chief-recovery.js' +import { nativeChiefBoundary, scopeDigest, corroboratedNativeScope } from './project-chief-recovery.js' import { CHIEF_AUTONOMY } from './project-chief-protocol.js' import { agentHistoryPage } from './project-history-pages.js' import { createProjectAgentInbox } from './project-agent-inbox.js' @@ -366,21 +367,21 @@ export function createProjectAgents(rt) { bot.instructions = priorPrompt.instructions if (priorPrompt.systemPrompt === undefined) delete bot.systemPrompt; else bot.systemPrompt = priorPrompt.systemPrompt } + const run = state.projectExecutions?.[runId] + if (!run || run.invocation !== invocation || run.status !== 'running') throw refuse('Chief invocation changed before execution admission') const delegation = { id: randomUUID(), exchangeId: randomUUID(), actor: { ...actor }, definitionRevision: set.revision, started: false, revoked: false, chiefJob: { runId, invocation, prompt: fullPrompt }, onChiefResult: onResult } delegations.set(agent.id, delegation) agent.lastRequest = { id: randomUUID(), initiatedBy: actor.memberId, executorBotId: agent.botId, exchangeId: delegation.exchangeId, phase: 'queued', context: snapshot, definitionRevision: set.revision, chiefRunId: runId, chiefInvocation: invocation, at: Date.now() } - const run = state.projectExecutions?.[runId] - if (!run || run.invocation !== invocation || run.status !== 'running') throw refuse('Chief invocation changed before execution admission') - run.executionBoundary = { version: 'chief-execution-boundary/v1', kind: (definition.execution?.harness ?? 'flybot') === 'flybot' ? 'native-in-process' : 'sdk-worker', runId, invocation, botId: agent.botId, sessionId: agent.sessionId, carrierGeneration: agent.carrierGeneration, exchangeId: delegation.exchangeId } - run.processScope ??= await captureProcessScope() - await rt.save() + run.executionBoundary = { version: 'chief-execution-boundary/v1', kind: (definition.execution?.harness ?? 'flybot') === 'flybot' ? 'native-in-process' : 'sdk-worker', phase: 'preparing', allocationKey: delegation.id, runId, invocation, botId: agent.botId, sessionId: agent.sessionId, carrierGeneration: agent.carrierGeneration, exchangeId: delegation.exchangeId } + try { run.processScope ??= await captureProcessScope(); await rt.save() } + catch (error) { if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId, invocation, at: Date.now() }; throw error } const finish = async result => { try { if (delegations.get(agent.id)?.id !== delegation.id || delegation.revoked) throw refuse('Chief execution was revoked'); const outcome = await onResult({ ...result, executionEvidence: { ...result.executionEvidence, chiefRunId: runId, carrierGeneration: agent.carrierGeneration } }); agent.lastRequest.phase = outcome?.correction ? outcome.correction.status === 'pending' ? 'correcting' : 'incomplete' : result.ok ? 'completed' : 'failed' } finally { restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); await rt.save(); rt.projectScheduler?.wake() } } delegation.finishChiefJob = finish; delegation.restoreChiefPrompt = restorePrompt try { - if ((definition.execution?.harness ?? 'flybot') !== 'flybot') { delegation.started = true; delegation.sdkExecution = true; return await rt.projectHarness.chiefTurn({ agent, delegation, definition, context: snapshot, text: 'Inspect the pinned Project facts and return the bounded Chief judgement.', onResult: finish }) } + if ((definition.execution?.harness ?? 'flybot') !== 'flybot') { run.executionBoundary.phase = 'allocating'; await rt.save(); delegation.started = true; delegation.sdkExecution = true; return await rt.projectHarness.chiefTurn({ agent, delegation, definition, context: snapshot, text: 'Inspect the pinned Project facts and return the bounded Chief judgement.', onResult: finish }) } const bot = rt.botOf(agent.botId) nativeSystem = definition.instructions + '\n\nFor each configured Chief job, use the current request protocol and scoped snapshot in that turn. Prior turns are historical background; act only on Task IDs and facts in the current snapshot. Preserve your resident Session and memory.' const nativeInput = fullPrompt + '\n\nCurrent scoped Chief job facts (data, not authority):\n' + snapshot.text @@ -397,9 +398,15 @@ export function createProjectAgents(rt) { } await rt.projectChiefControl.validateActor(actor) if (delegations.get(agent.id)?.id !== delegation.id || delegation.revoked) throw refuse('Chief execution was revoked before current job input') + run.executionBoundary.phase = 'input-sending'; await rt.save(); await rt.projectChiefControl.validateActor(actor) await rt.deliver({ toId: agent.botId, sessionId: agent.sessionId, role: 'request', audience: 'user', text: nativeInput, fromName: 'Project scheduled/arbitration request', untrusted: true, exchangeId: delegation.exchangeId, projectDelegation: { agentId: agent.id, delegationId: delegation.id } }) return `native-chief-job-${runId}` - } catch (error) { restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); throw error } + } catch (error) { + restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id) + const allocations = [...Object.values(state.projectResourceLeases ?? {}), ...Object.values(state.projectHarnessWorkers ?? {})].filter(row => row.attemptId === `chief-turn-${delegation.id}` || row.chiefRunId === runId && row.chiefInvocation === invocation) + if (state.projectExecutions?.[runId] === run && run.invocation === invocation && !allocations.length && (run.executionBoundary.kind === 'sdk-worker' || run.executionBoundary.phase === 'preparing')) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId, invocation, at: Date.now() }; await rt.save() } + throw error + } }) } function appendReply(agent, entry) { logOf(agent.id).push(entry) @@ -633,7 +640,7 @@ export function createProjectAgents(rt) { async function reconcileChiefExit(run) { if (!['interrupted', 'failed', 'cancelled'].includes(run.status)) return false const selected = nativeChiefBoundary(state, run), scope = run.processScope, invocation = run.invocation - if (!selected || !await priorHostProcessExited(scope)) return false + if (!selected || !await priorHostProcessExited(corroboratedNativeScope(scope, run, rt.config.projectChiefRecovery?.legacyNativeScopes))) return false const current = () => state.projectExecutions?.[run.id] === run && run.processScope === scope && run.invocation === invocation && ['interrupted', 'failed', 'cancelled'].includes(run.status) if (!current()) return false const proof = { confirmed: true, kind: 'native-chief-prior-host-exited', runId: run.id, invocation, carrierGeneration: selected.boundary.carrierGeneration, scopeSha256: scopeDigest(scope), at: Date.now() } @@ -659,13 +666,17 @@ export function createProjectAgents(rt) { const request = agent.lastRequest request.phase = 'stopping'; await rt.save() let confirmed = false + const run = state.projectExecutions?.[runId] + if (!request.workerId && (run?.executionBoundary?.kind === 'sdk-worker' || Object.values(state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === runId && row.chiefInvocation === request.chiefInvocation))) return { confirmed: false, reason: 'SDK admission is incomplete; reconcile its exact worker and leases without cancelling the native Chief' } if (request.workerId) confirmed = await rt.projectHarness.release(request.workerId) else { const active = await rt.resolveAgent?.(agent.sessionId) if (!active || typeof active.whenIdle !== 'function') return { confirmed: false, reason: 'Native Chief exit cannot be confirmed' } for (const message of [...(active.inbox?.nextStep ?? []), ...(active.inbox?.nextTurn ?? [])]) if (message.source?.exchangeId === request.exchangeId) active.inbox.remove?.(message.id) active.cancel?.({ kind: 'hook', reason: 'Configured Chief inspection deadline' }) - await active.whenIdle() + let timer + const stopped = await Promise.race([active.whenIdle().then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), inspectionLifecycle(state.projectChiefPolicies?.[projectId]?.inspection).stopTimeoutMs) })]).finally(() => clearTimeout(timer)) + if (!stopped) return { confirmed: false, reason: 'Native Chief exit confirmation timed out; execution remains isolated' } const lease = delegation?.resourceLease ?? request.resourceLease if (lease) await rt.projectResources.release(lease.id, lease.generation, { handle: request.exchangeId, exited: true }) confirmed = true diff --git a/core/src/host/project-chief-control.js b/core/src/host/project-chief-control.js index 5c524fcc7..bbe19f0dc 100644 --- a/core/src/host/project-chief-control.js +++ b/core/src/host/project-chief-control.js @@ -1,3 +1,4 @@ +import { classifyChiefRecovery, progressCheckpoint } from './project-chief-recovery.js' // Chief arbitration uses existing Tasks, Runs, Sessions and resource leases. import { randomUUID, createHash } from 'node:crypto' import { readFileSync } from 'node:fs' @@ -20,22 +21,13 @@ export function inspectionResources(state, task) { const historical = rows.filter(row => row.attemptId !== task.deliveryAttemptId && row.status === 'released' && proven.has(row.releaseProof)) return { resources: rows.filter(row => !historical.includes(row)), ...(historical.length ? { releasedResourceHistory: { count: historical.length, allConfirmedReleased: true, sha256: digest(historical), source: 'projectResourceLeases and projectHarnessWorkers; original records retained' } } : {}) } } -function verifiedProgress(evidence = []) { - return evidence.filter(row => row.kind === 'host-tests' && row.passed === true && typeof row.sha === 'string' && typeof row.sha256 === 'string') -} export function progressFingerprint(attempt) { const tests = attempt?.evidence?.tests return digest(tests?.passed === true && tests.sha === attempt.candidateSha ? { testedCandidate: tests.sha } : null) } export function inspectionFacts(state, task, thresholds, now = Date.now()) { const c = task.collaboration, attempt = state.projectAttempts?.[task.deliveryAttemptId], fingerprint = progressFingerprint(attempt) - let checkpoint = 0, lastMeaningfulProgressAt = null, progressEvidence = [] - const seen = new Set() - for (const [index, receipt] of (c?.history ?? []).entries()) { - const fresh = verifiedProgress(receipt.evidence).filter(row => !seen.has(row.sha)) - for (const row of fresh) seen.add(row.sha) - if (fresh.length) { checkpoint = index + 1; lastMeaningfulProgressAt = receipt.createdAt; progressEvidence = fresh.map(row => ({ handoffId: receipt.id, sha: row.sha, testLogSha256: row.sha256 })) } - } + const { checkpoint, lastMeaningfulProgressAt, progressEvidence } = progressCheckpoint(c?.history) const handoffs = (c?.history.length ?? 0) - checkpoint const worker = c?.claim?.workerId ? state.projectHarnessWorkers?.[c.claim.workerId] : null const reasons = [] @@ -67,11 +59,12 @@ export function createProjectChiefControl(rt) { const stopping = new Map() let pumping = false, reconciling = null rt.projectEffects?.register('chief.inspection-recovery', { run: row => ({ runId: row.payload.runId }) }, { replace: true }) + rt.projectEffects?.register('chief.inspection-stop', { run: row => ({ runId: row.payload.runId, stopAttempt: row.payload.stopAttempt }) }, { replace: true }) rt.projectEffects?.register('chief.correction', { run: row => ({ runId: row.payload.runId, round: row.payload.round }) }, { replace: true }) const policyOf = id => ownEntry(policies(), id) function requireJobActor(actor, projectId) { const job = active.get(actor?.chiefJobId), policy = policyOf(projectId) - if (rt.readOnly?.() || !job || job.token !== actor.chiefToken || job.run.projectId !== projectId || job.run.initiatedBy !== actor.memberId || job.run.status !== 'running' || !policy?.enabled || policy.revision !== job.run.policyRevision || policy.authorityMemberId !== actor.memberId) throw refuse('Chief job authority is missing, revoked or stale') + if (rt.readOnly?.() || !job || job.token !== actor.chiefToken || job.run.projectId !== projectId || job.run.initiatedBy !== actor.memberId || job.run.status !== 'running' || job.run.invocation !== job.invocation || !policy?.enabled || policy.revision !== job.run.policyRevision || policy.authorityMemberId !== actor.memberId) throw refuse('Chief job authority is missing, revoked or stale') return job } async function validateActor(actor) { @@ -210,11 +203,11 @@ export function createProjectChiefControl(rt) { } async function completed(job, result) { const { run } = job - if (active.get(run.id) !== job || run.status !== 'running' || job.resultInProgress) return { ignored: true } + if (active.get(run.id) !== job || run.invocation !== job.invocation || run.status !== 'running' || job.resultInProgress) return { ignored: true } job.resultInProgress = true let correcting = false try { - if (!result.ok) throw refuse('Chief execution did not complete; inspect its resource evidence') + if (!result.ok) throw Object.assign(refuse('Chief execution did not complete; inspect its resource evidence'), { recovery: result.recoveryIssue ?? classifyChiefRecovery(new Error('Chief execution failed')) }) await validateActor(job.actor) const execution = result.executionEvidence, carrier = rt.state.projectAgents?.[`public-${run.projectId}`] if (!execution || execution.chiefRunId !== run.id || execution.carrierGeneration !== carrier?.carrierGeneration) throw refuse('Chief carrier changed or execution evidence is missing') @@ -237,6 +230,7 @@ export function createProjectChiefControl(rt) { if (run.appliedDecisions?.some(row => row.taskId === decision.taskId)) continue const saved = job.snapshot.find(row => row.taskId === decision.taskId) if (!saved) throw chiefProtocolError('taskId', 'an exact Task in this invocation snapshot', decision.taskId) + if (['route', 'return', 'request-tests'].includes(decision.action)) await rt.projectPools?.validateChiefRoute?.({ taskId: decision.taskId, targetBotId: decision.action === 'request-tests' ? `${run.projectId}:tester` : decision.targetBotId, expectedGeneration: saved.generation }, job.actor) try { rt.projectInputs?.validateChiefReceipt?.(decision.taskId, saved, run, decision.inputReceipts) } catch (error) { if (!error.receiptFeedback?.correctable) throw error @@ -260,7 +254,7 @@ export function createProjectChiefControl(rt) { if (rt.state.tasks[decision.taskId]?.chiefActionIssue) delete rt.state.tasks[decision.taskId].chiefActionIssue if (rt.state.tasks[decision.taskId]?.chiefDecisionIssue) delete rt.state.tasks[decision.taskId].chiefDecisionIssue } catch (error) { - if (run.purpose !== 'inspection') throw error + if (run.purpose !== 'inspection' || error.code === 'bot/chief-protocol') throw error const failure = { taskId: decision.taskId, code: error.code ?? 'bot/chief', message: String(error.message).slice(0, 1000), at: Date.now() }; failures.push(failure) const task = rt.state.tasks[decision.taskId]; if (task?.projectId === run.projectId) task.chiefActionIssue = { runId: run.id, ...failure } await rt.save() @@ -268,14 +262,14 @@ export function createProjectChiefControl(rt) { } await validateActor(job.actor) if (run.correction) run.correction.status = 'resolved' - delete run.failure + delete run.failure; delete run.recoveryIssue run.waitingFor = null; run.status = failures.length ? 'partial' : 'done'; run.decisionFailures = failures; if (failures.length) run.failure = `${failures.length} Task decisions were refused; independent decisions continued`; run.summary = parsed.summary.slice(0, 8000); run.decisionIds = results.map(row => row.id); run.executionEvidence = result.executionEvidence ?? null waiters.get(run.id)?.resolve() } catch (error) { if (active.get(run.id) !== job || run.status !== 'running') return { ignored: true } if (error.code === 'bot/chief-protocol') { const previous = run.correction, round = (previous?.round ?? 0) + 1, limit = run.correctionLimit ?? correctionBudget(policyOf(run.projectId)) - const feedback = { ...(error.receiptFeedback ? { receiptFeedback: error.receiptFeedback } : {}), code: error.code, taskId: error.taskId, fields: error.fields, message: error.message.slice(0, 500), at: Date.now(), sideEffects: 'response protocol rejected before applying its decisions', appliedTaskIds: (run.appliedDecisions ?? []).map(row => row.taskId) } + const feedback = { ...(error.recovery ? { recovery: error.recovery } : {}), ...(error.receiptFeedback ? { receiptFeedback: error.receiptFeedback } : {}), code: error.code, taskId: error.taskId, fields: error.fields, message: error.message.slice(0, 500), at: Date.now(), sideEffects: 'response protocol rejected before applying its decisions', appliedTaskIds: (run.appliedDecisions ?? []).map(row => row.taskId) } run.correction = protocolCorrection(previous, { version: 'chief-correction/v1', limit, feedback, priorExecution: job.executionEvidence ?? null }) run.status = round <= limit ? 'correction-pending' : 'incomplete'; run.failure = feedback.message if (rt.projectEffects) { @@ -301,7 +295,11 @@ export function createProjectChiefControl(rt) { if (!correcting) waiters.get(run.id)?.reject(refuse('Chief judgement incomplete; correction budget exhausted')) return { correction: structuredClone(run.correction) } } - run.status = 'failed'; run.failure = String(error.message).slice(0, 1000) + run.status = 'failed'; run.failure = String(error.message).slice(0, 1000); run.recoveryIssue = error.recovery ?? classifyChiefRecovery(error) + if (run.purpose === 'inspection' && run.recoveryIssue.retryable) { + const config = inspectionLifecycle(policyOf(run.projectId)?.inspection) + run.status = 'interrupted'; run.waitingFor = 'recovery'; run.recovery = { ...run.recovery, status: 'waiting-exit', attempts: run.recovery?.attempts ?? 0, notBefore: Date.now() + config.recoveryDelayMs } + } for (const audit of Object.values(rt.state.projectChiefDecisions ?? {})) if (audit.runId === run.id && audit.status === 'requested') { audit.status = 'rejected'; audit.failure = run.failure; audit.finishedAt = Date.now() } if (run.taskId) { const task = ownEntry(rt.state.tasks, run.taskId); if (task?.projectTask && task.projectTask.admission.status !== 'accepted') task.projectTask.admission.failure = { chiefRunId: run.id, reason: run.failure, at: Date.now() }; if (task?.collaboration && !terminal.has(task.status)) { task.collaboration.status = 'blocked'; if (task.collaboration.claim?.chiefRunId === run.id) task.collaboration.claim.status = 'failed'; task.collaboration.block = { reason: run.failure, at: Date.now() } } } waiters.get(run.id)?.reject(error) @@ -323,6 +321,7 @@ export function createProjectChiefControl(rt) { for (const run of Object.values(rt.state.projectExecutions ?? {})) { if (run.id === current.id || run.projectId !== current.projectId || run.kind !== 'chief-control' || run.status !== 'interrupted' && run.recovery?.exitConfirmed !== false || !run.startedAt) continue if (!await inspectionExited(run)) return true + if (run.recovery?.exitConfirmed === false) { run.recovery.exitConfirmed = true; if (run.status !== 'interrupted') run.recovery.status = 'isolated-exit-confirmed'; await rt.save() } } return false } @@ -344,23 +343,33 @@ export function createProjectChiefControl(rt) { await rt.save() } } - if (run.status !== 'interrupted') continue - if (!policy?.enabled || policy.revision !== run.policyRevision || run.source?.scheduleId && (!rt.state.projectScheduledTasks?.[run.source.scheduleId]?.enabled || run.source.scheduleRevision !== undefined && rt.state.projectScheduledTasks[run.source.scheduleId].revision !== run.source.scheduleRevision)) { run.status = 'cancelled'; run.failure = 'Inspection recovery authority or schedule changed'; await rt.save(); continue } + if (run.status !== 'interrupted' && run.recovery?.exitConfirmed !== false) continue + if (run.status === 'interrupted' && (!policy?.enabled || policy.revision !== run.policyRevision || run.source?.scheduleId && (!rt.state.projectScheduledTasks?.[run.source.scheduleId]?.enabled || run.source.scheduleRevision !== undefined && rt.state.projectScheduledTasks[run.source.scheduleId].revision !== run.source.scheduleRevision))) { run.status = 'cancelled'; run.failure = 'Inspection recovery authority or schedule changed'; active.delete(run.id); await rt.save() } if (!run.recovery) { run.recovery = { status: 'waiting-exit', notBefore: now + config.recoveryDelayMs, attempts: 0 }; await rt.save() } const exited = await inspectionExited(run) run.recovery.exitConfirmed = exited if (!exited) { - run.recovery.status = 'waiting-exit' - if (!stopping.has(run.id) && now >= (run.recovery.nextStopAt ?? 0) && rt.projectAgents.stopChiefJob) { + run.recovery.status = (run.recovery.stopAttempts ?? 0) >= config.maxStopAttempts ? 'exit-unconfirmed' : 'waiting-exit' + run.recoveryIssue = { version: 'chief-recovery/v1', kind: 'worker-exit-unknown', retryable: true, conditions: ['matching execution exit proof'], actions: ['inspect registered worker and lease evidence'], stopBudgetExhausted: (run.recovery.stopAttempts ?? 0) >= config.maxStopAttempts } + if (!stopping.has(run.id) && (run.recovery.stopAttempts ?? 0) < config.maxStopAttempts && now >= (run.recovery.nextStopAt ?? 0) && rt.projectAgents.stopChiefJob) { + run.recovery.stopAttempts = (run.recovery.stopAttempts ?? 0) + 1 run.recovery.nextStopAt = now + config.recoveryDelayMs const invocation = run.invocation + let stopIntent + try { + if (rt.projectEffects) { + stopIntent = rt.projectEffects.stage({ kind: 'chief.inspection-stop', key: `${run.id}:${invocation}:${run.recovery.stopAttempts}`, projectId: run.projectId, payload: { runId: run.id, invocation, stopAttempt: run.recovery.stopAttempts } }) + await rt.projectEffects.persist(stopIntent) + } else await rt.save() + } catch (error) { if (!stopIntent || rt.projectEffects.discardUncommitted(stopIntent)) --run.recovery.stopAttempts; throw error } const pending = Promise.resolve().then(() => rt.projectAgents.stopChiefJob(run.projectId, run.id)).then(async proof => { - if (proof?.confirmed && run.invocation === invocation && run.status === 'interrupted') { run.exitProof = { ...proof, runId: run.id, invocation, at: Date.now() }; await rt.save(); rt.projectScheduler?.wake() } + if (proof?.confirmed && run.invocation === invocation && ['interrupted', 'cancelled', 'failed'].includes(run.status)) { run.exitProof = { ...proof, runId: run.id, invocation, at: Date.now() }; await rt.save(); rt.projectScheduler?.wake() } }).catch(error => { run.recovery.stopFailure = String(error.message).slice(0, 300) }).finally(() => stopping.delete(run.id)) stopping.set(run.id, pending) } continue } + if (run.status !== 'interrupted') { run.recovery.status = 'isolated-exit-confirmed'; active.delete(run.id); await rt.save(); continue } if (active.has(run.id) || rt.projectAgents.chiefBusy(run.projectId)) continue if (run.recovery.supersededBy) { run.status = 'cancelled'; run.recovery.status = 'coalesced'; await rt.save(); continue } if ((run.recovery.attempts ?? 0) >= config.maxRecoveries) { run.status = 'failed'; run.recovery.status = 'exhausted'; await rt.save(); continue } @@ -369,6 +378,8 @@ export function createProjectChiefControl(rt) { await rt.projectDelegations.background(policy.authorityMemberId) rt.requireProjectAccess({ memberId: policy.authorityMemberId }, run.projectId, 'configure') if (policyOf(run.projectId) !== policy || policy.revision !== run.policyRevision) throw refuse('Inspection policy changed during recovery') + if (run.recovery.stopAttempts) run.recovery.stopHistory = [...(run.recovery.stopHistory ?? []), { invocation: run.invocation, attempts: run.recovery.stopAttempts, lastStopAt: run.recovery.nextStopAt }].slice(-5) + run.recovery.stopAttempts = 0; delete run.recovery.nextStopAt run.recovery.attempts = (run.recovery.attempts ?? 0) + 1; run.recovery.status = 'requeued'; run.recovery.previousInvocation = run.invocation run.status = 'queued'; run.queuedAt = now; delete run.finishedAt if (rt.projectEffects) { @@ -397,11 +408,11 @@ export function createProjectChiefControl(rt) { if (run.invocation === job.invocation) { run.workerId = workerId; if (run.status === 'running') run.waitingFor = 'execution'; run.workerGeneration = rt.state.projectHarnessWorkers?.[workerId]?.generation ?? null; await rt.save() } } catch (error) { if (active.get(run.id) !== job || run.status !== 'running') { - if (run.invocation === job.invocation && !Object.values(rt.state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === run.id) && rt.state.projectAgents?.[`public-${run.projectId}`]?.lastRequest?.chiefRunId !== run.id) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId: run.id, invocation: job.invocation, at: Date.now() }; await rt.save() } + if (run.invocation === job.invocation && !Object.values(rt.state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined ? row.id === run.workerId : row.chiefInvocation === job.invocation)) && !(rt.state.projectAgents?.[`public-${run.projectId}`]?.lastRequest?.chiefRunId === run.id && rt.state.projectAgents[`public-${run.projectId}`].lastRequest.chiefInvocation === job.invocation)) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId: run.id, invocation: job.invocation, at: Date.now() }; await rt.save() } return } if (error.code === 'bot/busy') { run.status = 'queued'; active.delete(run.id); await rt.save() } - else { run.dispatchFailure = String(error.message).slice(0, 1000); await completed(job, { ok: false, text: String(error.message) }) } + else { run.dispatchFailure = String(error.message).slice(0, 1000); run.recoveryIssue = classifyChiefRecovery(error); await completed(job, { ok: false, text: String(error.message), recoveryIssue: run.recoveryIssue }) } } } async function pump() { @@ -420,7 +431,6 @@ export function createProjectChiefControl(rt) { for (const run of Object.values(rt.state.projectExecutions ?? {}).filter(row => row.kind === 'chief-control' && row.status === 'interrupted' && row.taskId && row.purpose !== 'inspection')) { const task = rt.state.tasks[run.taskId], policy = policyOf(run.projectId) if (!policy?.enabled || task?.collaboration?.claim?.chiefRunId !== run.id || task.collaboration.claim.status !== 'interrupted') continue - const worker = rt.state.projectHarnessWorkers?.[run.workerId] if (!await inspectionExited(run)) continue try { await rt.projectPools.retryChief(task.id, { memberId: policy.authorityMemberId }); run.recovery = { status: 'requeued', at: Date.now() }; await rt.save() } catch (error) { run.recovery = { status: 'blocked', reason: error.message, at: Date.now() } } } @@ -440,7 +450,7 @@ export function createProjectChiefControl(rt) { } function recover() { let changed = false - for (const run of Object.values(rt.state.projectExecutions ?? {})) if (run.kind === 'chief-control' && run.status === 'running') { run.status = 'interrupted'; run.failure = 'Host restarted before Chief result was validated'; changed = true } + for (const run of Object.values(rt.state.projectExecutions ?? {})) if (run.kind === 'chief-control' && run.status === 'running') { run.status = 'interrupted'; run.waitingFor = 'recovery'; run.recoveryIssue = { version: 'chief-recovery/v1', kind: 'host-recovering', retryable: true, conditions: ['prior execution exit reconciliation'], actions: ['wait for the existing recovery pass'] }; run.failure = 'Host restarted before Chief result was validated'; changed = true } return changed } return { configure, view, request, scheduled, scheduledState, pump, recover, requireJobActor, validateActor, applyDecision, template: structuredClone(template) } diff --git a/core/src/host/project-chief-lifecycle.js b/core/src/host/project-chief-lifecycle.js index 838aee03b..6a4d468f1 100644 --- a/core/src/host/project-chief-lifecycle.js +++ b/core/src/host/project-chief-lifecycle.js @@ -1,9 +1,10 @@ +import { progressCheckpoint } from './project-chief-recovery.js' // Mechanical inspection lifecycle; no business routing or automatic human escalation. -export const INSPECTION_LIFECYCLE = { queueTimeoutMs: 600000, runTimeoutMs: 900000, recoveryDelayMs: 5000, maxRecoveries: 2, overdueMs: 3600000 } +export const INSPECTION_LIFECYCLE = { queueTimeoutMs: 600000, runTimeoutMs: 900000, recoveryDelayMs: 5000, maxRecoveries: 2, stopTimeoutMs: 10000, maxStopAttempts: 3, overdueMs: 3600000 } export function inspectionLifecycle(inspection) { if (inspection?.lifecycle !== undefined && (!inspection.lifecycle || typeof inspection.lifecycle !== 'object' || Array.isArray(inspection.lifecycle))) throw new Error('Invalid Chief inspection lifecycle configuration') const value = { ...INSPECTION_LIFECYCLE, ...inspection?.lifecycle } - for (const [key, number] of Object.entries(value)) if (!(key in INSPECTION_LIFECYCLE) || !Number.isSafeInteger(number) || number < (key === 'maxRecoveries' ? 0 : 1) || number > (key === 'maxRecoveries' ? 4 : 86400000)) throw new Error('Invalid Chief inspection lifecycle configuration: ' + key) + for (const [key, number] of Object.entries(value)) if (!(key in INSPECTION_LIFECYCLE) || !Number.isSafeInteger(number) || number < (['maxRecoveries', 'maxStopAttempts'].includes(key) ? 0 : 1) || number > (key === 'maxRecoveries' ? 4 : key === 'maxStopAttempts' ? 10 : 86400000)) throw new Error('Invalid Chief inspection lifecycle configuration: ' + key) return value } export function inspectionHealth(state, projectId, now = Date.now()) { @@ -15,10 +16,10 @@ export function inspectionHealth(state, projectId, now = Date.now()) { const lastSuccessAt = last(runs.filter(row => row.status === 'done').map(row => row.finishedAt)) const enabled = Boolean(policy?.enabled && schedule?.enabled) const anchor = lastSuccessAt ?? policy?.updatedAt ?? schedule?.createdAt ?? now - const active = runs.filter(row => ['queued', 'running', 'correction-pending', 'interrupted'].includes(row.status)).map(row => ({ runId: row.id, occurrenceId: row.source?.occurrenceId ?? null, status: row.status, waitingFor: row.waitingFor ?? null, ageMs: Math.max(0, now - (row.startedAt ?? row.createdAt)), failure: row.failure ?? null, recovery: row.recovery ?? null, deadlineAt: (row.status === 'queued' ? (row.queuedAt ?? row.createdAt) + config.queueTimeoutMs : (row.startedAt ?? row.createdAt) + config.runTimeoutMs) })) + const active = runs.filter(row => (['queued', 'running', 'correction-pending', 'interrupted'].includes(row.status) || row.recovery?.exitConfirmed === false)).map(row => ({ runId: row.id, occurrenceId: row.source?.occurrenceId ?? null, status: row.status, waitingFor: row.waitingFor ?? null, ageMs: Math.max(0, now - (row.startedAt ?? row.createdAt)), failure: row.failure ?? null, recovery: row.recovery ?? null, recoveryIssue: row.recoveryIssue ?? null, nextRecoveryAt: row.recovery?.exitConfirmed === false ? (row.recovery.stopAttempts ?? 0) < config.maxStopAttempts ? row.recovery.nextStopAt ?? row.recovery.notBefore ?? null : null : row.recovery?.notBefore ?? null, deadlineAt: (row.status === 'queued' ? (row.queuedAt ?? row.createdAt) + config.queueTimeoutMs : (row.startedAt ?? row.createdAt) + config.runTimeoutMs) })) const latestDelivery = deliveries.toSorted((a,b) => (b.at ?? 0) - (a.at ?? 0))[0] const latest = runs.toSorted((a,b) => b.createdAt - a.createdAt)[0] - return { enabled, status: !enabled ? 'disabled' : now - anchor > config.overdueMs ? 'overdue' : active.some(row => row.status === 'interrupted' || row.deadlineAt <= now) || latestDelivery?.status === 'failed' && (latestDelivery.finishedAt ?? latestDelivery.at) >= (lastSuccessAt ?? 0) || ['failed', 'partial', 'incomplete'].includes(latest?.status) ? 'degraded' : lastSuccessAt === null ? 'pending' : 'healthy', lastOccurrenceAt: last(deliveries.map(row => row.at)), lastStartedAt: last(runs.map(row => row.startedAt)), lastCompletedAt: last(runs.map(row => row.finishedAt)), lastSuccessAt, lastAppliedDecisionAt: last(Object.values(state.projectChiefDecisions ?? {}).filter(row => row.projectId === projectId && row.status === 'applied' && !['continue', 'needs-human'].includes(row.action)).map(row => row.finishedAt)), overdueMs: config.overdueMs, active } + return { enabled, status: !enabled ? 'disabled' : now - anchor > config.overdueMs ? 'overdue' : active.some(row => row.status === 'interrupted' || row.recovery?.exitConfirmed === false || row.deadlineAt <= now) || latestDelivery?.status === 'failed' && (latestDelivery.finishedAt ?? latestDelivery.at) >= (lastSuccessAt ?? 0) || ['failed', 'partial', 'incomplete'].includes(latest?.status) ? 'degraded' : lastSuccessAt === null ? 'pending' : 'healthy', lastOccurrenceAt: last(deliveries.map(row => row.at)), lastStartedAt: last(runs.map(row => row.startedAt)), lastCompletedAt: last(runs.map(row => row.finishedAt)), lastSuccessAt, lastMeaningfulProgressAt: last(Object.values(state.tasks ?? {}).filter(row => row.projectId === projectId).map(row => progressCheckpoint(row.collaboration?.history).lastMeaningfulProgressAt)), lastAppliedDecisionAt: last(Object.values(state.projectChiefDecisions ?? {}).filter(row => row.projectId === projectId && row.status === 'applied' && !['continue', 'needs-human'].includes(row.action)).map(row => row.finishedAt)), overdueMs: config.overdueMs, active } } export function scheduledInspectionState(state, row, delivery) { diff --git a/core/src/host/project-chief-recovery.js b/core/src/host/project-chief-recovery.js index 99419ae91..4880f219e 100644 --- a/core/src/host/project-chief-recovery.js +++ b/core/src/host/project-chief-recovery.js @@ -20,3 +20,32 @@ export function nativeChiefBoundary(state, run) { if (leases.some(row => row.projectId !== run.projectId || row.chiefInvocation !== run.invocation || row.worker?.kind !== 'native-turn' || row.worker.botId !== selected.botId || row.worker.sessionId !== selected.sessionId || row.worker.handle !== selected.exchangeId || row.preparation?.status === 'active')) return null return { boundary: selected, leases } } + +export function classifyChiefRecovery(error) { + if (error.recovery) return structuredClone(error.recovery) + const message = String(error.message) + const kind = /stale|generation|snapshot.*changed/i.test(message) ? 'stale-snapshot' : /worker.*exit|confirmed.*exit|unreachable/i.test(message) ? 'worker-exit-unknown' : /interrupted delivery|candidate.*missing|matching passing tests/i.test(message) ? 'unsupported-route' : ['bot/forbidden', 'bot/delegation', 'bot/execution-revoked'].includes(error.code) ? 'authority-unavailable' : ['bot/resource', 'bot/busy'].includes(error.code) ? 'resource-unavailable' : 'execution-failed' + return { version: 'chief-recovery/v1', kind, retryable: ['worker-exit-unknown', 'resource-unavailable'].includes(kind), conditions: kind === 'worker-exit-unknown' ? ['matching exit proof; absence of a host controller is insufficient'] : kind === 'stale-snapshot' ? ['fresh scoped snapshot'] : ['inspect current scoped evidence'], actions: kind === 'unsupported-route' ? ['select an evidence-supported role or rework explicitly'] : [] } +} + +// Optional operator corroboration for pre-namespace records; scoped to one exact +// old native invocation. It is not a PID-only exit claim or a public Bot grant. +export function corroboratedNativeScope(scope, run, configuration) { + if (scope?.pidNamespace !== undefined || configuration === undefined) return scope + if (!Array.isArray(configuration) || configuration.length > 20) throw new Error('legacyNativeScopes must contain at most20 exact operator attestations') + const selected = configuration.find(row => row.runId === run.id && row.invocation === run.invocation && row.scopeSha256 === scopeDigest(scope)) + if (!selected) return scope + if (!/^pid:\[\d+\]$/.test(selected.pidNamespace ?? '') || scope?.kind !== 'unmanaged-process') throw new Error('Legacy native namespace corroboration is invalid') + return { ...scope, pidNamespace: selected.pidNamespace } +} + +export function progressCheckpoint(history = []) { + let checkpoint = 0, lastMeaningfulProgressAt = null, progressEvidence = [] + const seen = new Set() + for (const [index, receipt] of history.entries()) { + const fresh = (receipt.evidence ?? []).filter(row => row.kind === 'host-tests' && row.passed === true && typeof row.sha === 'string' && typeof row.sha256 === 'string' && !seen.has(row.sha)) + for (const row of fresh) seen.add(row.sha) + if (fresh.length) { checkpoint = index + 1; lastMeaningfulProgressAt = receipt.createdAt; progressEvidence = fresh.map(row => ({ handoffId: receipt.id, sha: row.sha, testLogSha256: row.sha256 })) } + } + return { checkpoint, lastMeaningfulProgressAt, progressEvidence } +} diff --git a/core/src/host/project-chief-schedule.json b/core/src/host/project-chief-schedule.json index cfef0a12f..bad4b99c4 100644 --- a/core/src/host/project-chief-schedule.json +++ b/core/src/host/project-chief-schedule.json @@ -18,6 +18,8 @@ "runTimeoutMs": 900000, "recoveryDelayMs": 5000, "maxRecoveries": 2, - "overdueMs": 3600000 + "overdueMs": 3600000, + "stopTimeoutMs": 10000, + "maxStopAttempts": 3 } } diff --git a/core/src/host/project-delivery.js b/core/src/host/project-delivery.js index 52a8ceb8e..0f78ff564 100644 --- a/core/src/host/project-delivery.js +++ b/core/src/host/project-delivery.js @@ -1,3 +1,5 @@ +import { chiefRecoveryError } from './project-chief-recovery.js' +import { chiefProtocolError } from './project-chief-protocol.js' // Delivery coordinates artifacts/worktrees around the existing Workflow service. import { execFile } from 'node:child_process' import { createHash, randomUUID } from 'node:crypto' @@ -516,7 +518,7 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { return { baseSha: attempt.baseSha ?? null, checkoutSha: attempt.candidateSha ?? attempt.seedSha ?? attempt.baseSha ?? null, availableInputs: await approvedRepositoryInputs(project), snapshot: await integrationSnapshot(attempt), preparationFailure: attempt.preparationFailures?.at(-1) ?? null, - materialization: workspaceFor(attempt, 'developer')?.materialization ?? null } + materialization: workspaceFor(attempt, 'developer')?.materialization ?? null, verificationRecovery: { attemptStatus: attempt.status, sealedCandidate: Boolean(attempt.candidateSha && attempt.evidence.implementation?.sha === attempt.candidateSha), testsPassedForCandidate: Boolean(attempt.evidence.tests?.passed && attempt.evidence.tests.sha === attempt.candidateSha), requires: ['current requirements and repository binding', 'clean sealed candidate', 'all old allocation exits confirmed', 'normal Task-role context admission'], reviewerRequires: 'matching passing host test command and immutable log' } } } catch { return { status: 'unavailable', reason: 'Registered repository or workspace facts are unavailable; preserve prior execution and inspect current bindings' } } } async function admitChiefInputs(taskId, inputs, actor, saved) { @@ -1049,6 +1051,39 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { attempt.acceptedViaChief = decision.id; if (task.result) task.result.summary = `Chief confirmed: ${decision.reason}`; task.completedBy = { kind: 'chief', authorizedBy: actor.memberId, decisionId: decision.id, runId: actor.chiefJobId }; await rt.save(); return accepted } catch (error) { attempt.chiefDecision = prior; throw error } } + async function validateRecovery({ task, target }) { + const attempt = ownEntry(attempts(), task.collaboration?.attemptId), capability = target.policyRole ?? target.role + if (!attempt || capability === 'developer') return + const refuseRoute = (message, actions) => Object.assign(chiefProtocolError('targetBotId', message, task.id), { recovery: { version: 'chief-recovery/v1', kind: 'unsupported-route', retryable: false, attemptId: attempt.id, attemptStatus: attempt.status, candidateSha: attempt.candidateSha ?? null, actions, conditions: [message] } }) + if (!['running', 'interrupted', 'failed'].includes(attempt.status) || attempt.cancelRequested || attempt.supersededBy || task.deliveryAttemptId !== attempt.id) throw refuseRoute('a current active or recoverable Attempt; preserve ended/superseded history', ['inspect current Attempt', 'choose an explicit rework or reopen action']) + if (task.collaboration.kind !== 'code' || !['tester', 'reviewer'].includes(capability)) { + if (attempt.status !== 'running') throw refuseRoute('a running non-code Attempt or explicit rework', ['choose an explicit rework']) + return + } + if (task.projectTask && attempt.requirement.projectTaskRevision !== task.projectTask.requirements.revision || task.title !== attempt.requirement.title || (task.criteria ?? '') !== attempt.requirement.criteria) throw refuseRoute('matching current Task requirements', ['rework against current requirements']) + settingsOf(projectOf(task.projectId, attempt)) + if (!attempt.candidateSha || attempt.evidence.implementation?.sha !== attempt.candidateSha) throw refuseRoute('a host-sealed candidate before verification routing', ['route to a permitted implementation role', 'admit approved repository inputs when appropriate']) + try { await verifyCandidate(attempt) } catch { throw refuseRoute('the unchanged clean sealed candidate and valid child inputs', ['inspect candidate/workspace evidence', 'choose explicit rework']) } + if (capability === 'reviewer') { + const tests = attempt.evidence.tests, settings = settingsOf(projectOf(task.projectId, attempt)) + if (!tests?.passed || tests.sha !== attempt.candidateSha || JSON.stringify(tests.command) !== JSON.stringify(settings.testCommand)) throw refuseRoute('matching passing host tests for Reviewer recovery', ['request tests', 'inspect environment readiness']) + try { if (await fileDigest(tests.log) !== tests.sha256) throw new Error('changed') } catch { throw refuseRoute('an unchanged matching host test log', ['request fresh tests']) } + } + } + async function resumeVerification({ task, target, decisionId, reason }) { + const attempt = ownEntry(attempts(), task.collaboration?.attemptId) + if (!attempt || attempt.status === 'running') return + try { await validateRecovery({ task, target }) } catch (error) { throw chiefRecoveryError('unsupported-route', error.message, { attemptId: attempt.id, conditions: error.recovery?.conditions ?? [], actions: error.recovery?.actions ?? [] }) } + const leases = Object.values(state.projectResourceLeases ?? {}).filter(row => row.attemptId === attempt.id), workers = Object.values(state.projectHarnessWorkers ?? {}).filter(row => row.attemptId === attempt.id), native = Object.values(state.projectNativeWorkers ?? {}).filter(row => row.attemptId === attempt.id) + if (leases.some(row => row.status !== 'released') || workers.some(row => !confirmedWorkerExit(state, row)) || native.some(row => !nativeExitConfirmed(state, row)) || Object.values(state.bots ?? {}).some(row => row.ephemeral && row.attemptId === attempt.id)) throw chiefRecoveryError('worker-exit-unknown', 'Verification recovery waits for confirmed exit of every previous allocation', { retryable: true, attemptId: attempt.id, conditions: ['all exact worker and lease exits confirmed'], actions: ['inspect registered execution evidence'] }) + const previousRunId = attempt.runId, runId = `pool-resume-${hash(attempt.id + ':' + decisionId)}` + const resumption = { kind: 'sealed-candidate-verification', decisionId, previousRunId, runId, previousStatus: attempt.status, candidateSha: attempt.candidateSha, targetBotId: target.id, reason, at: Date.now(), leaseIds: leases.map(row => row.id), workerIds: workers.map(row => row.id) } + ;(attempt.resumptions ??= []).push(resumption) + attempt.runId = runId; attempt.status = 'running'; attempt.updatedAt = Date.now() + rt.bindProject({ kind: 'run', id: previousRunId, projectId: task.projectId, taskId: task.id, attemptId: attempt.id, status: 'failed' }) + rt.bindProject({ kind: 'run', id: runId, projectId: task.projectId, taskId: task.id, attemptId: attempt.id, status: 'queued' }) + await rt.save() + } async function reworkPool({ task, claim, reason, force = false, decisionId }) { const previous = ownEntry(attempts(), claim.attemptId) if (!previous) throw reject('Chief rework has no original Attempt') @@ -1145,7 +1180,7 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { } } const ownsWorkspace = (projectId, path) => Object.values(state.projectWorkspaces ?? {}).find(workspace => workspace.projectId === projectId && workspace.path === path && DELIVERY_ROLES.includes(workspace.policyRole ?? workspace.role) && ['worktree', 'resource-clone', 'decision-directory'].includes(workspace.kind) && workspace.status === 'allocated' && ongoing.has(ownEntry(attempts(), workspace.attemptId)?.status)) - return { releases, recoveryFacts, admitChiefInputs, reconcileExecutions: () => reconcileExecutionAuthority(rt), exitConfirmed: workerId => executionExitConfirmed(rt, workerId), nativeSessions, endpoint, context, admitChildInputs: (taskId, claim) => serialAttempt(claim.attemptId, () => childInputs.admit(taskId, claim)), preparePoolRole, abortPoolAdmission, completePoolRole, chiefAccept, reworkPool, recordExecution, repositoryContext, repositoryRolePolicy, bindSession, releaseSession, sessionFor, recover, runReported, ownsWorkspace, gate: input => serialAttempt(current(input.taskId)?.id ?? input.taskId, () => gate(input)) } + return { releases, recoveryFacts, admitChiefInputs, reconcileExecutions: () => reconcileExecutionAuthority(rt), exitConfirmed: workerId => executionExitConfirmed(rt, workerId), nativeSessions, endpoint, context, admitChildInputs: (taskId, claim) => serialAttempt(claim.attemptId, () => childInputs.admit(taskId, claim)), preparePoolRole, validateRecovery, resumeVerification, abortPoolAdmission, completePoolRole, chiefAccept, reworkPool, recordExecution, repositoryContext, repositoryRolePolicy, bindSession, releaseSession, sessionFor, recover, runReported, ownsWorkspace, gate: input => serialAttempt(current(input.taskId)?.id ?? input.taskId, () => gate(input)) } } export function install(rt) { diff --git a/core/src/host/project-development-service.js b/core/src/host/project-development-service.js index 90b22ce3b..578aa0e77 100644 --- a/core/src/host/project-development-service.js +++ b/core/src/host/project-development-service.js @@ -164,7 +164,7 @@ export function createDevelopmentService(rt) { attemptsPage: { total: rows.length, limit, before: end > limit ? page[0].id : null } } } function context(body) { - if (!['roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures'].includes(body.section)) throw failure('Unknown Attempt context section') + if (!['roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures', 'resumptions'].includes(body.section)) throw failure('Unknown Attempt context section') const attempt = ownEntry(rt.state.projectAttempts ?? {}, body.attemptId) const text = JSON.stringify(attempt[body.section] ?? null), offset = body.offset ?? 0, limit = body.limit ?? 8000 if (!Number.isSafeInteger(offset) || offset < 0 || offset > text.length || !Number.isSafeInteger(limit) || limit < 1 || limit > 16000) throw failure('Invalid context page') diff --git a/core/src/host/project-pools.js b/core/src/host/project-pools.js index 9184a974a..2e0aebb48 100644 --- a/core/src/host/project-pools.js +++ b/core/src/host/project-pools.js @@ -536,6 +536,17 @@ export function createProjectPools(rt) { receipt.acceptance = { claimId: claim.id, botId: claim.botId, generation: claim.generation, acceptedAt: receipt.acceptedAt, ...(claim.chiefRunId ? { chiefRunId: claim.chiefRunId } : {}) } await rt.save(); return clone(claim) } + async function validateChiefRoute(body, actor) { + const task = taskOf(body.taskId), c = task.collaboration + rt.projectChiefControl.requireJobActor(actor, task.projectId) + rt.requireProjectAccess(actor, task.projectId, 'run-control') + if (!c || c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') + const target = targetOf(task.projectId, body.targetBotId) + await authority(task) + await adapter?.validateRecovery?.({ task, target }) + if (c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') + return target + } async function chiefRoute(body, actor) { const task = taskOf(body.taskId), c = task.collaboration rt.projectChiefControl.requireJobActor(actor, task.projectId) @@ -544,6 +555,7 @@ export function createProjectPools(rt) { const target = targetOf(task.projectId, body.targetBotId) if (target.role === 'chief') throw refuse('Chief must decide a concrete next role or request human clarification') reasonOf(body.reason) + await validateChiefRoute(body, actor) await authority(task) await checkAdmission(task, { from: `${task.projectId}:chief`, to: target.id, attemptId: c.attemptId, requestId: body.decisionId, purpose: 'chief-intervention' }) if (c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') @@ -564,7 +576,10 @@ export function createProjectPools(rt) { c.initiatedBy = actor.memberId; c.sourceSessionId = null const previous = rt.state.projectAttempts?.[c.attemptId] if ((target.policyRole ?? target.role) === 'developer') c.attemptId = await adapter.rework({ task, claim: oldClaim ?? { attemptId: c.attemptId, id: body.decisionId, botId: `${task.projectId}:chief` }, reason: body.reason, force: body.rework !== false, decisionId: body.decisionId }) - else if (previous && previous.status !== 'running') throw refuse('Interrupted delivery must return to Developer in a new Attempt') + else if (previous && previous.status !== 'running') { + if (!adapter.resumeVerification) throw refuse('Interrupted delivery requires an evidence-supported recovery adapter') + await adapter.resumeVerification({ task, target, decisionId: body.decisionId, reason: body.reason }) + } return await offer(task, { from: `${task.projectId}:chief`, to: target.id, reason: body.reason, requestId: body.decisionId, attemptId: c.attemptId, purpose: 'chief-intervention' }) } catch (error) { c.status = 'blocked'; c.block = { reason: String(error.message), decisionId: body.decisionId, at: Date.now() }; await rt.save(); throw error @@ -625,7 +640,7 @@ export function createProjectPools(rt) { return { connect(value) { adapter = value; return () => { if (adapter === value) adapter = null } }, get, list, accept: (taskId, botId, handoffId, requestId) => serial(() => claim(taskId, botId, handoffId, requestId)), admit: (attempt, actor, body) => serial(() => admit(attempt, actor, body)), available, owns, correct, resumeChildren: (id, waitId) => serial(() => resumeChildren(id, waitId)), - claim: (id, botId) => serial(() => claim(id, botId)), remember: (id, token, body) => serial(() => remember(id, token, body)), decide: (id, token, decision) => serial(() => decide(id, token, decision)), result: row => serial(() => result(row)), pump, recover, revokePrincipal: (projectId, memberId) => serial(() => revokePrincipal(projectId, memberId)), revokeExecution: (id, claimId, reason) => serial(() => revokeExecution(id, claimId, reason)), validateClosure, retryChief: (id, actor) => serial(() => retryChief(id, actor)), chiefClaim: (id, actor) => serial(() => chiefClaim(id, actor)), chiefRoute: (body, actor) => serial(() => chiefRoute(body, actor)) } + claim: (id, botId) => serial(() => claim(id, botId)), remember: (id, token, body) => serial(() => remember(id, token, body)), decide: (id, token, decision) => serial(() => decide(id, token, decision)), result: row => serial(() => result(row)), pump, recover, revokePrincipal: (projectId, memberId) => serial(() => revokePrincipal(projectId, memberId)), revokeExecution: (id, claimId, reason) => serial(() => revokeExecution(id, claimId, reason)), validateClosure, retryChief: (id, actor) => serial(() => retryChief(id, actor)), chiefClaim: (id, actor) => serial(() => chiefClaim(id, actor)), validateChiefRoute: (body, actor) => serial(() => validateChiefRoute(body, actor)), chiefRoute: (body, actor) => serial(() => chiefRoute(body, actor)) } } export function install(rt) { rt.projectPools = createProjectPools(rt) diff --git a/core/src/host/project-service-views.js b/core/src/host/project-service-views.js index 2cb49c3e3..f744b4431 100644 --- a/core/src/host/project-service-views.js +++ b/core/src/host/project-service-views.js @@ -59,6 +59,7 @@ export function attemptMetadataView(attempt) { if (attempt.release) result.release = { ...pick(attempt.release, ['id', 'status', 'observedAt']), ...pick(attempt.release.spec, ['repository', 'tag', 'sha', 'mode']), sourceRef: { query: 'release.status', attemptId: attempt.id } } if (attempt.pr) result.pr = pick(attempt.pr, ['number', 'sha', 'url', 'repository']) if (attempt.publicationBinding) result.publicationBinding = pick(attempt.publicationBinding, ['number', 'repository', 'branch', 'previousSha', 'upstreamSha', 'upstreamRef']) + if (attempt.resumptions) result.resumptions = { total: attempt.resumptions.length, recent: attempt.resumptions.slice(-5).map(row => pick(row, ['kind', 'decisionId', 'previousRunId', 'runId', 'previousStatus', 'candidateSha', 'targetBotId', 'at'])), source: { query: 'attempt.context', attemptId: attempt.id, section: 'resumptions' } } if (attempt.integrationInputs) result.integrationInputs = attempt.integrationInputs.slice(0, 20).map(row => pick(row, ['version', 'repository', 'ref', 'sha', 'purpose'])) if (attempt.evidence?.tests?.preparation) result.environment = pick(attempt.evidence.tests.preparation, ['id', 'status', 'executor', 'workspaceId', 'resourceId', 'identitySha256', 'sha256']) if (attempt.workspaces) result.workspaces = Object.fromEntries(Object.entries(attempt.workspaces).slice(0, 32).map(([role, row]) => [role, pick(row, ['id', 'status', 'role', 'resourceId', 'materialization'])])) diff --git a/core/test/project-agents.test.js b/core/test/project-agents.test.js index 19e4ee876..47e2ebe49 100644 --- a/core/test/project-agents.test.js +++ b/core/test/project-agents.test.js @@ -949,3 +949,26 @@ test('native Chief deadline fencing waits for actual idleness and preserves the assert.equal(rt.projectAgents.chiefBusy('p1'), false) assert.equal(rt.state.projectAgents[chief.id].sessionId, rt.botOf(chief.botId).sessionId) }) + +test('native exit deadline is bounded and cannot silently release its live execution', async t => { + const { rt } = await setup(t); await get(rt, alice) + rt.projectChiefControl.validateActor = async () => {} + rt.state.projectChiefPolicies = { p1: { inspection: { lifecycle: { stopTimeoutMs: 20 } } } } + rt.state.projectExecutions.bounded = { id: 'bounded', projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'bounded', invocation: 1, prompt: 'Bounded exit', projectFacts: {}, onResult: async () => {} }) + rt.resolveAgent = async () => ({ inbox: {}, cancel() {}, whenIdle: () => new Promise(() => {}) }) + const proof = await rt.projectAgents.stopChiefJob('p1', 'bounded') + assert.equal(proof.confirmed, false); assert.match(proof.reason, /timed out/) + assert.equal(rt.projectAgents.chiefBusy('p1'), true, 'Revoked but live execution retains its Session boundary') +}) + +test('failed Chief execution-boundary commit cannot strand a busy delegation before dispatch', async t => { + const { rt } = await setup(t); await get(rt, alice) + rt.projectChiefControl.validateActor = async () => {} + const run = rt.state.projectExecutions.uncommitted = { id: 'uncommitted', projectId: 'p1', invocation: 1, status: 'running' } + const save = rt.save; let failed = false + rt.save = async () => { if (!failed && rt.projectAgents.chiefBusy('p1')) { failed = true; throw new Error('boundary snapshot failed') } return save() } + await assert.rejects(rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: run.id, invocation: 1, prompt: 'Do not dispatch before commit', projectFacts: {}, onResult: async () => {} }), /boundary snapshot failed/) + assert.equal(failed, true); assert.equal(rt.projectAgents.chiefBusy('p1'), false) + assert.equal(run.exitProof.kind, 'chief-never-dispatched') +}) diff --git a/core/test/project-chief-control.test.js b/core/test/project-chief-control.test.js index 1e7e25aec..8f357b605 100644 --- a/core/test/project-chief-control.test.js +++ b/core/test/project-chief-control.test.js @@ -464,3 +464,44 @@ test('Chief receipt protocol correction precedes all role proposals and Task eff assert.equal(run.correction.status, 'resolved'); assert.equal(proposals, 0) await f.rt.projectEffects.close() }) + +test('unsupported verification route is corrected by Chief before changing the Task generation', async () => { + const { chiefProtocolError } = await import('../src/host/project-chief-protocol.js') + const f = await controlFixture(), run = f.rt.state.projectExecutions[f.request.runId], generation = f.task.collaboration.generation + f.rt.projectPools.connect({ validate: () => ({ ok: true }), validateRecovery: async ({ target }) => { + if (target.role !== 'developer') throw Object.assign(chiefProtocolError('targetBotId', 'a sealed candidate for verification', 't'), { recovery: { kind: 'unsupported-route', actions: ['inspect candidate', 'choose explicit rework'] } }) + }, rework: async () => 'a' }) + await f.request.onResult(f.result({ action: 'route', targetBotId: 'p:reviewer', reason: 'Review candidate' })) + assert.equal(run.status, 'correction-pending'); assert.equal(f.task.collaboration.generation, generation) + assert.equal(run.correction.feedback[0].recovery.kind, 'unsupported-route'); assert.equal(f.task.chiefQuestion, undefined) + await f.control.pump(); await f.request.onResult(f.result({ action: 'return', targetBotId: 'p:developer', reason: 'Produce the missing candidate' })) + assert.equal(run.status, 'done', run.failure); assert.equal(f.task.collaboration.botId, 'p:developer') +}) + +test('unmanaged native exit adapter unblocks an interrupted inspection and old invocation callbacks remain fenced', async () => { + const f = await controlFixture(), old = f.request, run = f.rt.state.projectExecutions[old.runId] + run.purpose = 'inspection'; delete run.taskId + let reconciled = 0 + f.rt.projectAgents.reconcileChiefExit = async current => { + if (current.invocation === 1) { ++reconciled; current.exitProof = { confirmed: true, kind: 'native-chief-prior-host-exited', runId: current.id, invocation: 1 }; return true } + return false + } + const restored = createProjectChiefControl(f.rt); f.rt.projectChiefControl = restored; restored.recover() + run.recovery = { status: 'waiting-exit', attempts: 0, notBefore: Date.now() - 1 } + await restored.pump(); await new Promise(resolve => setTimeout(resolve, 25)) + assert.equal(run.status, 'running'); assert.equal(run.invocation, 2); assert.equal(reconciled, 1) + await old.onResult(f.result({ action: 'close', reason: 'Late old response' })) + assert.equal(f.accepted(), 0, 'old response cannot act through a reused Run record') +}) + +test('an uncommitted stop intent cannot send a stop or consume the mechanical stop budget', async () => { + const f = await controlFixture({ modern: true }), run = f.rt.state.projectExecutions[f.request.runId] + run.purpose = 'inspection'; delete run.taskId + let stopped = 0; f.rt.projectAgents.stopChiefJob = async () => { ++stopped; return { confirmed: false } } + const restored = createProjectChiefControl(f.rt); f.rt.projectChiefControl = restored; restored.recover() + run.recovery = { status: 'waiting-exit', attempts: 0, notBefore: Date.now() - 1 } + f.rt.save = async () => {} + await assert.rejects(restored.pump(), /not confirmed/) + assert.equal(stopped, 0); assert.equal(run.recovery.stopAttempts, 0) + await f.rt.projectEffects.close() +}) diff --git a/core/test/project-chief-lifecycle.test.js b/core/test/project-chief-lifecycle.test.js index 3b20c54d0..6a3a65d5c 100644 --- a/core/test/project-chief-lifecycle.test.js +++ b/core/test/project-chief-lifecycle.test.js @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { inspectionLifecycle, inspectionHealth } from '../src/host/project-chief-lifecycle.js' test('inspection configuration bounds deadlines/recovery and health distinguishes enablement from success', () => { - for (const lifecycle of [{ maxRecoveries: 5 }, { runTimeoutMs: 0 }, { recoveryDelayMs: -1 }, { overdueMs: Infinity }, { unknown: 1 }]) assert.throws(() => inspectionLifecycle({ lifecycle }), /Invalid/) + for (const lifecycle of [{ maxRecoveries: 5 }, { stopTimeoutMs: 0 }, { maxStopAttempts: 11 }, { runTimeoutMs: 0 }, { recoveryDelayMs: -1 }, { overdueMs: Infinity }, { unknown: 1 }]) assert.throws(() => inspectionLifecycle({ lifecycle }), /Invalid/) const state = { projectChiefPolicies: { p: { enabled: true, scheduleId: 's', updatedAt: 100, inspection: { lifecycle: { overdueMs: 1000 } } } }, projectScheduledTasks: { s: { enabled: true, deliveries: {} } }, projectExecutions: {}, projectChiefDecisions: {} } assert.equal(inspectionHealth(state, 'p', 200).status, 'pending') assert.equal(inspectionHealth(state, 'p', 2000).status, 'overdue') @@ -13,3 +13,10 @@ test('inspection configuration bounds deadlines/recovery and health distinguishe assert.equal(inspectionHealth(state, 'p', 600).status, 'degraded') state.projectScheduledTasks.s.enabled = false; assert.equal(inspectionHealth(state, 'p', 20000).status, 'disabled') }) + +test('health retains unknown-exit cancellation and separates actual checkpoints from inspection or message churn', () => { + const state = { projectChiefPolicies: { p: { enabled: true, scheduleId: 's', updatedAt: 100 } }, projectScheduledTasks: { s: { enabled: true } }, projectExecutions: { old: { id: 'old', projectId: 'p', kind: 'chief-control', purpose: 'inspection', createdAt: 100, status: 'cancelled', recovery: { exitConfirmed: false, stopAttempts: 3 } } }, tasks: { t: { projectId: 'p', collaboration: { history: [{ createdAt: 120, evidence: [{ kind: 'host-tests', passed: true, sha: 'candidate', sha256: 'log' }] }, { createdAt: 150, evidence: [{ kind: 'host-tests', passed: true, sha: 'candidate', sha256: 'second-log' }] }] } } } } + const health = inspectionHealth(state, 'p', 200) + assert.equal(health.status, 'degraded'); assert.equal(health.active[0].nextRecoveryAt, null) + assert.equal(health.lastMeaningfulProgressAt, 120); assert.equal(health.lastSuccessAt, null) +}) diff --git a/core/test/project-chief-recovery.test.js b/core/test/project-chief-recovery.test.js new file mode 100644 index 000000000..51f733c8d --- /dev/null +++ b/core/test/project-chief-recovery.test.js @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { nativeChiefBoundary, classifyChiefRecovery } from '../src/host/project-chief-recovery.js' +const scope = () => { + const run = { id: 'run', projectId: 'p', invocation: 2, workerId: 'native-chief-job-run', executionBoundary: { version: 'chief-execution-boundary/v1', kind: 'native-in-process', runId: 'run', invocation: 2, botId: 'chief', sessionId: 'original', carrierGeneration: 1, exchangeId: 'exchange' } } + return { run, state: { projectResourceLeases: {}, projectHarnessWorkers: {}, projectAgents: { 'public-p': { botId: 'chief', sessionId: 'original', carrierGeneration: 1, lastRequest: { chiefRunId: 'run', chiefInvocation: 2, exchangeId: 'exchange' } } } } } +} +test('native host exit boundary rejects SDK admissions, unknown leases and stale invocation identities', () => { + const f = scope(); assert.equal(nativeChiefBoundary(f.state, f.run).boundary.kind, 'native-in-process') + f.state.projectHarnessWorkers.w = { chiefRunId: 'run', chiefInvocation: 2, status: 'unreachable' } + assert.equal(nativeChiefBoundary(f.state, f.run), null) + delete f.state.projectHarnessWorkers.w + f.state.projectResourceLeases.l = { projectId: 'p', chiefRunId: 'run', chiefInvocation: 2, status: 'unreachable', worker: { kind: 'sdk', handle: 'w' } } + assert.equal(nativeChiefBoundary(f.state, f.run), null) + f.state.projectResourceLeases = {}; f.run.executionBoundary.invocation = 1 + assert.equal(nativeChiefBoundary(f.state, f.run), null) + delete f.run.executionBoundary + assert(nativeChiefBoundary(f.state, f.run), 'legacy native handle still requires the exact persisted request binding') + f.state.projectAgents['public-p'].lastRequest.chiefInvocation = 1 + assert.equal(nativeChiefBoundary(f.state, f.run), null) +}) +test('Chief recovery feedback distinguishes stale state, missing authority, unknown exit and unsupported routes', () => { + for (const [error, kind] of [[new Error('Chief routing snapshot is stale'), 'stale-snapshot'], [new Error('worker exit unconfirmed'), 'worker-exit-unknown'], [Object.assign(new Error('denied'), { code: 'bot/forbidden' }), 'authority-unavailable'], [new Error('Interrupted delivery needs recovery'), 'unsupported-route']]) assert.equal(classifyChiefRecovery(error).kind, kind) + assert.equal(classifyChiefRecovery(new Error('worker exit unconfirmed')).retryable, true) + assert.equal(classifyChiefRecovery(Object.assign(new Error('denied'), { code: 'bot/forbidden' })).retryable, false) +}) + +test('legacy unmanaged namespace assurance is operator-owned and bound to one exact old native invocation', async () => { + const { corroboratedNativeScope, scopeDigest } = await import('../src/host/project-chief-recovery.js') + const old = { kind: 'unmanaged-process', hostPid: 12, startTicks: '10', bootId: 'same' }, run = { id: 'run', invocation: 2 } + const record = { runId: 'run', invocation: 2, scopeSha256: scopeDigest(old), pidNamespace: 'pid:[123]' } + assert.equal(corroboratedNativeScope(old, run), old) + assert.equal(corroboratedNativeScope(old, { ...run, invocation: 3 }, [record]), old) + assert.equal(corroboratedNativeScope({ ...old, startTicks: '11' }, run, [record]).pidNamespace, undefined) + assert.equal(corroboratedNativeScope(old, run, [record]).pidNamespace, 'pid:[123]') + assert.equal(old.pidNamespace, undefined) +}) diff --git a/core/test/project-delivery.test.js b/core/test/project-delivery.test.js index aae6a0d45..d74ab3c8e 100644 --- a/core/test/project-delivery.test.js +++ b/core/test/project-delivery.test.js @@ -1915,3 +1915,46 @@ test('a newly created unsealed successor after failure pins current host HEAD ev await prepareRole(f.rt, f.task, { id: 'failed-route-prepare', attemptId: id, generation: 2 }, f.definition) assert.equal(git(next.workspaces.developer.path, ['rev-parse', 'HEAD']), f.latest) }) + +for (const role of ['tester', 'reviewer']) test('interrupted sealed candidate resumes ' + role + ' in the same Attempt only with current evidence', async t => { + const f = await setup(t); await f.start('resume-' + role); await implement(f) + if (role === 'reviewer') await f.gate('tester') + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id], sha = attempt.candidateSha, oldRun = attempt.runId + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 }; attempt.status = 'interrupted' + const target = { id: 'p1:' + role, role } + await f.rt.projectDelivery.validateRecovery({ task, target }) + await f.rt.projectDelivery.resumeVerification({ task, target, decisionId: 'recovery-decision', reason: 'Continue verification of the original sealed candidate' }) + assert.equal(attempt.status, 'running'); assert.equal(attempt.candidateSha, sha); assert.equal(task.deliveryAttemptId, attempt.id) + assert.notEqual(attempt.runId, oldRun); assert.equal(attempt.resumptions.length, 1) + await f.rt.projectDelivery.resumeVerification({ task, target, decisionId: 'recovery-decision', reason: 'Duplicate delivery' }) + assert.equal(attempt.resumptions.length, 1); assert.equal(Object.keys(f.rt.state.projectAttempts).length, 1) + assert.equal(git(attempt.workspaces.developer.path, ['rev-parse', 'HEAD']), sha) +}) + +test('verification recovery preflight rejects unsealed/stale candidates and Reviewer without tests before routing effects', async t => { + const f = await setup(t); await f.start('recovery-gates') + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id], target = { id: 'p1:reviewer', role: 'reviewer' } + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 } + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), error => error.code === 'bot/chief-protocol' && error.recovery.kind === 'unsupported-route') + assert.equal(task.collaboration.generation, 3) + await implement(f); attempt.status = 'interrupted' + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), /passing host tests/) + await f.rt.projectDelivery.validateRecovery({ task, target: { id: 'p1:tester', role: 'tester' } }) + attempt.status = 'running'; await f.gate('tester'); attempt.status = 'interrupted' + await f.rt.projectDelivery.validateRecovery({ task, target }) + writeFileSync(attempt.evidence.tests.log, 'changed test evidence') + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), /unchanged matching host test log/) + writeFileSync(join(attempt.workspaces.developer.path, 'code.mjs'), '// unverified mutation') + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target: { id: 'p1:tester', role: 'tester' } }), /unchanged clean sealed candidate/) + assert.equal(attempt.status, 'interrupted'); assert.equal(task.collaboration.generation, 3) +}) + +test('unknown SDK or native allocation exit cannot be bypassed by choosing another verification role', async t => { + const f = await setup(t); await f.start('unknown-verification-exit'); await implement(f) + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id] + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 }; attempt.status = 'interrupted' + f.rt.state.projectHarnessWorkers ??= {} + f.rt.state.projectHarnessWorkers.unknown = { id: 'unknown', attemptId: attempt.id, status: 'unreachable' } + await assert.rejects(f.rt.projectDelivery.resumeVerification({ task, target: { id: 'p1:tester', role: 'tester' }, decisionId: 'unknown', reason: 'Cannot bypass unknown exit' }), error => error.code === 'bot/chief-recovery' && error.recovery.kind === 'worker-exit-unknown') + assert.equal(attempt.status, 'interrupted'); assert.equal(attempt.resumptions, undefined) +}) diff --git a/dev/demo/chief-restart.mjs b/dev/demo/chief-restart.mjs new file mode 100644 index 000000000..44181e7b4 --- /dev/null +++ b/dev/demo/chief-restart.mjs @@ -0,0 +1,45 @@ +// Real unmanaged DSH restart and original scheduled delivery; inference is a fixture. +import assert from 'node:assert/strict' +export function chiefRestartModel() { + let calls = 0, release + return { + get started() { return calls > 0 }, + finishHeld() { release?.([{ type: 'text', text: '{"summary":"Late old invocation","decisions":[]}' }]) }, + reply({ body }) { + if (!JSON.stringify(body.messages ?? []).includes('chief-unmanaged-restart-fixture')) return undefined + if (++calls === 1) return new Promise(resolve => { release = resolve }) + return [{ type: 'text', text: '{"summary":"Inspected the current shared state after recovery","decisions":[]}' }] + }, + } +} +export async function checkChiefRestart({ api, saved, waitFor, restart, model }) { + const invite = await api('auth-invite'), owner = await api('auth-redeem', { token: invite.token, displayName: 'Recovery fixture owner' }) + const call = (endpoint, body = {}) => api(endpoint, body, owner.sessionToken) + const { project } = await call('project-create', { name: 'Disposable unmanaged Chief restart', repository: null }) + await call('project-agents', { projectId: project.id }) + await call('project-chief-configure', { projectId: project.id, enabled: true, inspection: { prompt: 'chief-unmanaged-restart-fixture: inspect only current shared facts', when: { kind: 'every', seconds: 60 }, lifecycle: { runTimeoutMs: 60000, queueTimeoutMs: 60000, recoveryDelayMs: 100, maxRecoveries: 2, stopTimeoutMs: 500, maxStopAttempts: 2 } } }) + await call('project-chief-inspect', { projectId: project.id, requestId: 'first-held-inspection' }) + const prior = await waitFor(() => { + const run = Object.values(saved()?.projectExecutions ?? {}).find(row => row.projectId === project.id && row.purpose === 'inspection' && row.status === 'running') + return model.started && run?.workerId && run.executionBoundary?.kind === 'native-in-process' ? run : null + }, 'first native inspection is actually answering') + assert.equal(prior.processScope.kind, 'unmanaged-process') + assert.match(prior.processScope.pidNamespace, /^pid:\[\d+\]$/) + const chief = saved().projectAgents[`public-${project.id}`], began = Date.now() + await restart() + model.finishHeld() + const outcome = await waitFor(async () => { + const current = await call('project-chief-state', { projectId: project.id }) + const scheduled = current.runs.find(row => row.source?.scheduleId && row.source?.occurrenceId && row.status === 'done') + return current.health.lastSuccessAt >= began && scheduled ? { current, scheduled } : null + }, 'post-restart original scheduled occurrence completes') + const old = saved().projectExecutions[prior.id] + assert.equal(old.exitProof.kind, 'native-chief-prior-host-exited') + assert.equal(saved().projectAgents[`public-${project.id}`].sessionId, chief.sessionId) + assert.equal(new Set((old.invocationHistory ?? []).map(row => row.number)).size, (old.invocationHistory ?? []).length) + assert.ok(outcome.current.health.lastSuccessAt) + assert.equal(Object.values(saved().tasks).filter(task => task.projectId === project.id).length, 0) + return { milestone: '2347-unmanaged-chief-restart', runtime: 'actual DSH', transport: 'real DSH HTTP', model: 'deterministic fixture', recoveryMs: Date.now() - began, + checks: ['actual unmanaged host killed during native Chief inspection', 'birth-checked prior native host exit', 'same resident Chief Session', 'later original scheduled occurrence completed without manual inspect', 'late old result cannot add an invocation verdict'], + notRun: ['paid model', 'live project', 'unknown external SDK/container exit recovery', 'deployment'] } +} diff --git a/dev/demo/run.mjs b/dev/demo/run.mjs index e44195fa3..420e33c9e 100644 --- a/dev/demo/run.mjs +++ b/dev/demo/run.mjs @@ -1,3 +1,4 @@ +import { chiefRestartModel, checkChiefRestart } from './chief-restart.mjs' // M0: real pinned DSH, disposable data, deterministic model, actual shell execution. import assert from 'node:assert/strict' import { spawn, execFileSync } from 'node:child_process' @@ -45,6 +46,7 @@ async function waitFor(read, label, milliseconds = 90_000) { } async function main() { + const chiefRestart = process.argv.includes('--chief-restart'), chiefModel = chiefRestart ? chiefRestartModel() : null const nativePools = process.argv.includes('--native-pools') const sqlite = process.argv.includes('--sqlite') const nativeModel = nativePools ? nativePoolModel() : null @@ -91,7 +93,7 @@ async function main() { '@deepseek-ai/dsh-base': '0.2.1-rc.2', '@deepseek-ai/dsh-web-app': '0.2.1-rc.2' }, dsh: { profile: { bundles: ['@deepseek-ai/dsh-base', '@deepseek-ai/dsh-web-app', 'ds-bot', '@flybot/workflow'] } } })) const observations = [] - const model = mockModel(observations, { reply: nativeModel ? input => nativeModel.reply(input) ?? deliveryReply(input) : (workbench ? input => deliveryReply(input) ?? agentReply(input) : delivery ? deliveryReply : agents ? agentReply : undefined) }) + const model = mockModel(observations, { reply: chiefModel ? input => chiefModel.reply(input) : nativeModel ? input => nativeModel.reply(input) ?? deliveryReply(input) : (workbench ? input => deliveryReply(input) ?? agentReply(input) : delivery ? deliveryReply : agents ? agentReply : undefined) }) model.listen(0, '127.0.0.1') await once(model, 'listening') const port = await freePort() @@ -104,7 +106,7 @@ async function main() { // External connector credentials are not needed in this transport check. for (const name of Object.keys(env)) if (/TOKEN|SECRET|API_KEY/.test(name) && name !== 'DEEPSEEK_API_KEY') delete env[name] let patch = join(root, 'docker/web.patch.yml') - if (shared || delivery || agents) { + if (shared || delivery || agents || chiefRestart) { patch = join(directory, 'shared.patch.yml') const botConfig = { projectWorkbench: true, ...(nativePools ? { projectResources: { 'native-local': { type: 'local', root: directory, capacity: 1, allowedProjectOwnerIds: ['owner-1'], harnesses: ['flybot'] } } } : {}), @@ -115,7 +117,8 @@ async function main() { ...(sqlite ? { stateStorage: 'sqlite' } : {}), ...(delivery ? { projectDispatch: nativePools ? 'pools' : 'workflow', projectRepositories: { [githubRepository ?? 'example/delivery-fixture']: { ...(githubRepository ? { github: { credential: 'gh-cli', configDir: githubConfigDir, ...(codex ? { login: codex.login } : {}) } } : {}), path: workspace, testCommand: queueGate?.command ?? (live || codex ? [process.execPath, golden] : [process.execPath, '--test', 'code.test.mjs']), allowedPaths: ['code.mjs', 'code.test.mjs'] }, ...(live ? { 'example/intentional-failure': { path: workspace, testCommand: [process.execPath, golden, '--force-failure'], allowedPaths: ['code.mjs', 'code.test.mjs'] } } : {}) } } : {}) } writeFileSync(patch, readFileSync(join(root, 'docker/web.patch.yml'), 'utf8') + (live ? '\n- id: llm-deepseek\n config: { thinking: disabled, reasoningEffort: off, maxTokens: 8192 }\n' : '') + `\n- id: bot\n config: ${JSON.stringify(botConfig)}\n`, { mode: 0o600 }) } - const child = spawn(process.execPath, [cli, '--profile', 'web', '--patch', patch, + if (chiefRestart) delete env.INVOCATION_ID + let child = spawn(process.execPath, [cli, '--profile', 'web', '--patch', patch, '--no-open', '--port', String(port)], { cwd: workspace, env, stdio: ['ignore', log, log], detached: true }) closeSync(log) let childError @@ -141,7 +144,7 @@ async function main() { return readFileSync(logPath, 'utf8').match(/http:\/\/127\.0\.0\.1:\d+\/\?token=[^\s]+/)?.[0] }, 'DSH startup') const response = await fetch(login, { redirect: 'manual', signal: AbortSignal.timeout(5000) }) - const cookie = response.headers.getSetCookie().map(row => row.split(';')[0]).join('; ') + let cookie = response.headers.getSetCookie().map(row => row.split(';')[0]).join('; ') assert.ok(cookie, 'DSH login cookie') const call = async (endpoint, payload = {}, member) => { const response = await fetch(`http://127.0.0.1:${port}/api/bot`, { method: 'POST', @@ -154,6 +157,25 @@ async function main() { assert.equal(result.ok, true, `API ${args[0]}: ${result.error?.code ?? 'failed'}`) return result.value } + if (chiefRestart) { + const restart = async () => { + const previous = child, offset = readFileSync(logPath, 'utf8').length + stop('SIGKILL'); await once(previous, 'exit') + const output = openSync(logPath, 'a', 0o600) + childError = null + child = spawn(process.execPath, [cli, '--profile', 'web', '--patch', patch, '--no-open', '--port', String(port)], { cwd: workspace, env, stdio: ['ignore', output, output], detached: true }) + closeSync(output) + child.on('error', error => { childError = error }) + const login = await waitFor(() => { + if (childError || child.exitCode !== null) throw new Error('Restarted disposable DSH exited before startup') + return readFileSync(logPath, 'utf8').slice(offset).match(/http:\/\/127\.0\.0\.1:\d+\/\?token=[^\s]+/)?.[0] + }, 'unmanaged DSH restart') + const response = await fetch(login, { redirect: 'manual', signal: AbortSignal.timeout(5000) }) + cookie = response.headers.getSetCookie().map(row => row.split(';')[0]).join('; ') + } + console.log(JSON.stringify(await checkChiefRestart({ api, saved, waitFor, restart, model: chiefModel }))) + return + } if (codex) { const report = await checkCodexDelivery({ api, saved, waitFor: (read, label) => waitFor(read, label, 600_000), settings: codex, artifacts: process.env.FLYBOT_DEMO_ARTIFACTS }) assert.equal(observations.length, 0, 'Codex acceptance must never call the deterministic native model') @@ -272,6 +294,7 @@ async function main() { await Promise.race([once(child, 'exit'), sleep(5000)]) stop('SIGKILL') } + chiefModel?.finishHeld() model.closeAllConnections() await new Promise(done => model.close(done)) if (!keep) rmSync(directory, { recursive: true, force: true }) diff --git a/dev/demo/test/runtime.test.mjs b/dev/demo/test/runtime.test.mjs index 2917aac4f..1a6e3ca2a 100644 --- a/dev/demo/test/runtime.test.mjs +++ b/dev/demo/test/runtime.test.mjs @@ -104,3 +104,15 @@ test('actual DSH native return persists Task-role activation, release and contin assert.ok(report.notRun.includes('real model')) assert.deepEqual(readdirSync(directory), [], 'Disposable SQLite state, workers and worktrees must be removed') }) + +test('actual unmanaged DSH restart recovers native Chief and later original scheduled inspection', { timeout: 150_000 }, t => { + const directory = mkdtempSync(join(tmpdir(), 'flybot-chief-restart-runtime-')) + t.after(() => rmSync(directory, { recursive: true, force: true })) + const result = spawnSync(process.execPath, [resolve(import.meta.dirname, '../run.mjs'), '--chief-restart'], { encoding: 'utf8', timeout: 140_000, env: { ...process.env, TMPDIR: directory, M0_REPORT: '' } }) + assert.equal(result.status, 0, result.stderr) + const report = JSON.parse(result.stdout) + assert.equal(report.runtime, 'actual DSH'); assert.equal(report.model, 'deterministic fixture') + assert.ok(report.checks.includes('later original scheduled occurrence completed without manual inspect')) + assert.ok(report.recoveryMs > 0); assert.ok(report.recoveryMs < 90000) + assert.deepEqual(readdirSync(directory), []) +}) diff --git a/docs/chief-recovery.md b/docs/chief-recovery.md new file mode 100644 index 000000000..59ff68929 --- /dev/null +++ b/docs/chief-recovery.md @@ -0,0 +1,107 @@ +# Chief inspection and interrupted delivery recovery + +`chief.state` reports the configured policy separately from actual inspection +health. `lastSuccessAt` is a validated successful inspection; a configured/enabled +schedule, queued request, partial judgement or failed delivery is not success. +`lastAppliedDecisionAt` identifies applied non-defer actions, not necessarily a +verified artifact checkpoint. `health.active` includes isolated old invocations, +waiting reason, recovery issue, stop budget and `nextRecoveryAt` (null when there +is no scheduled retry). `lastMeaningfulProgressAt` uses the existing deduplicated candidate/test checkpoints; +repeated tests of the same SHA and message churn do not advance it. + +Use the existing `project-development/v1` commands and queries. A current +Owner/Maintainer can configure `chief.configure` with: + +```json +{ + "inspection": { + "when": {"kind": "every", "seconds": 1800}, + "lifecycle": { + "queueTimeoutMs": 600000, + "runTimeoutMs": 900000, + "recoveryDelayMs": 5000, + "maxRecoveries": 2, + "stopTimeoutMs": 10000, + "maxStopAttempts": 3, + "overdueMs": 3600000 + } + } +} +``` + +Timing still belongs to the original scheduled-task service. Durations are1–86400000 +ms, maxRecoveries0–4, maxStopAttempts0–10. Timeouts are not exit proof. Technical +retries preserve the Run/occurrence and recheck current policy, schedule revision, +principal, Task snapshot and invocation token. A stop intent commits before stop +transport; attempts are bounded. Exhaustion retains the actual wait and evidence +for Chief/operator judgement, without choosing a business role or human fallback. +Revocation cancels authority but retains the old execution barrier until exit is +confirmed. Cleanup may observe/stop old execution; it cannot restore old authority. + +## Unmanaged and supervised host restart + +Systemd control-group evidence remains valid only for its matching manager +boundary. Unmanaged recovery distinguishes native Chief from independent execution: + +- A native Chief invocation records its exact carrier/Session, invocation, + allocation identity and host process birth (boot ID, PID namespace, PID and start + ticks). Matching host-process exit can end that in-process invocation. Any + associated native-turn leases must match the same Project/invocation/exchange. +- SDK/container workers and resource preparation are independent. Host death, + an empty controller map, PID disappearance alone or timeout never releases them. + Exact existing worker/lease stop and exit evidence is still required. A failed + SDK admission with no registered allocation can record never-dispatched evidence; + started/ambiguous execution cannot use that path. +- Old callbacks must match the current invocation and active token. A recovered + Run cannot apply a response through its older invocation. Resume/cancel/stop + reconciliation is idempotent and preserves history. + +Pre-namespace unmanaged records lack enough information to compare process birth +safely across PID namespaces. They remain visible as unproven, rather than guessed +exited. An operator who can establish that a particular old native host used the +current PID namespace may configure one exact corroboration: + +```yaml +projectChiefRecovery: + legacyNativeScopes: + - runId: chief-run-EXACT-ID + invocation: 1 + scopeSha256: EXACT_SHA256_OF_JSON_STRINGIFY_ORIGINAL_PROCESS_SCOPE + pidNamespace: "pid:[EXACT_PREVIOUS_HOST_NAMESPACE]" +``` + +This operator setting is not model/API input. It allows at most20 entries and binds +one Run, invocation and original scope digest. It supplies only the missing +namespace fact; the host must still prove the old process exited, and the native +execution/lease checks still apply. It cannot confirm unknown SDK/container exit. +Do not fill it by guessing the old namespace or editing Task/binding/lease state. +New invocations record namespace identity automatically. + +## Continue a sealed interrupted Attempt + +Chief independently selects `route`, `return` or `request-tests`. Protocol preflight +runs before Task generation changes, proposals or routing effects. An unsupported +route returns structured facts to the same Chief correction mechanism, including +current Attempt/candidate and the missing conditions; the host selects no role. + +`route` to Tester can resume a failed/interrupted current code Attempt when its +requirements/repository binding and clean host-sealed candidate remain valid. +Reviewer additionally requires passing tests for that SHA and current configured +command, with the unchanged immutable test log. Every old worker and lease must +have confirmed exit. Unknown execution, cancelled/superseded Attempts, changed +requirements/candidates or stale evidence refuse continuation. An unsealed draft +uses the existing Developer recovery path; an explicit `return` requests rework. + +Successful verification recovery keeps Task, Attempt, candidate and role context. +It records a successor execution Run and an immutable resumption entry; normal +Task-role admission still enforces resource/persona/profile and context restoration +(#2346). `attempt.context` with `section:"resumptions"` pages full history; +Attempt summaries return at most5 recent metadata entries and a source reference. +This is not passing tests, PR merge or Task acceptance. + +Validation includes a real unmanaged DSH process killed during inspection, with +the same resident Session recovered and a later original scheduled occurrence +completed without manual inspect. Inference is deterministic, not paid-model +acceptance. Known SDK exit and unknown worker/container boundaries have separate +contract tests. No live project or service is modified by the demonstration: +`node dev/demo/run.mjs --chief-restart` uses disposable data and removes it on success. diff --git a/docs/history/2026-11-04-role-convergence-checkpoints.md b/docs/history/2026-11-04-role-convergence-checkpoints.md new file mode 100644 index 000000000..13cb4edf8 --- /dev/null +++ b/docs/history/2026-11-04-role-convergence-checkpoints.md @@ -0,0 +1,51 @@ +# Historical role/context convergence checkpoints + +Preserved verbatim from PROJECT.md; current contracts remain in the active +state and canonical subsystem documentation. + +### Task-role execution convergence — 2026-11-03 (#2184) + +Task-role SDK retries retain pinned public persona/skills and the latest observed +backend context across Attempts. Dynamic input is not a definition identity. A +binding incompatibility or unknown old execution fails visibly; no silent fresh +Session or rollback to older context. Definition snapshots are shared by digest, +current rights still revoke execution, failed unused preparations release their +pins, and model input follows a durable-save/cancellation barrier. Existing +Harness/lease/Workflow services remain authoritative. Adapter capability inspection +reports exact native Task-role resume with committed activation/quiescence; actual DSH deterministic return/context coverage is recorded separately from real-provider validation. No +public service protocol, SQL transaction, deployment or frontend change is claimed. + + +### Collaboration contract convergence — 2026-11-03 (#2175) + +Expose durable Task handoffs through one scoped contract projection, with distinct +pending/accepted/processed/revoked/invalidated states, source/target context references, +actual claim/worker binding, initiative fences and predecessor/successor history. +Idempotent trusted offer/accept ports reuse pools; unresolved old writers cannot be +bypassed by Chief rerouting to another resource. Consultation is a separate private +protocol over retained transcripts, without Task creation or initiative transfer. +Chief prefers an active personal agent for user questions: safe answers require user +publication, watch answers use existing inbox rights; only committed Effects may wake +an authorized native watch. External clients have isolated mailboxes, not fabricated +provider sessions. SDK process/HTTP tests and real-provider acceptance remain distinct. +Native restoration/context bounds, transaction events and parent waits stay on their +original dependent issues; no control-plane, deployment or frontend changes. + + +### Context/memory convergence — 2026-11-03 (#2177) + +Unify model/Harness context budgeting and bounded source tools across persistent +Chief and temporary roles. Unknown capacity stays explicit; full rules/current +constraints remain required, retrieval yields before overflow, and native/SDK input +is checked before delivery. Working summaries use original public role-memory +records with exact source/evidence/uncertainty, separate resident Chief topics and +rollback of uncommitted staging. Facts have source/classification/version/archive; +role-authored experience cannot declare facts or overwrite human control. Private +notes retain history under bounded views. Codex tool history has an explicit cap; +SDK compaction remains runtime-managed, without claiming aggregate usage is current +window pressure. No new memory platform, executor or timer. Native Task-role physical +restore2184 and normalized SQL storage2182 remain separate acceptance; real SDK +verification must name exact runtime source hashes, not borrow deterministic tests. + +Follow-up #2177: native resident Chief current facts are actual per-turn input with whole-input budget and final authority checks; do not rely on updating cached system prompt metadata. Preserve the original resident Session. + diff --git a/docs/openapi/flybot.openapi.json b/docs/openapi/flybot.openapi.json index dad21101a..f7c38ef47 100644 --- a/docs/openapi/flybot.openapi.json +++ b/docs/openapi/flybot.openapi.json @@ -1043,6 +1043,16 @@ "minimum": 0, "maximum": 4 }, + "stopTimeoutMs": { + "type": "integer", + "minimum": 1, + "maximum": 86400000 + }, + "maxStopAttempts": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, "overdueMs": { "type": "integer", "minimum": 1, @@ -1122,6 +1132,17 @@ } ] }, + "lastMeaningfulProgressAt": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, "overdueMs": { "type": "integer", "minimum": 1 @@ -1177,6 +1198,37 @@ } ] }, + "recoveryIssue": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChiefRecoveryIssue" + }, + { + "type": "null" + } + ] + }, + "waitingFor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "nextRecoveryAt": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, "deadlineAt": { "type": "integer", "minimum": 0 @@ -1287,6 +1339,9 @@ "maxLength": 200, "description": "Opaque server-issued ID; never infer authority from it." } + }, + "recovery": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -1852,6 +1907,31 @@ }, "preview": { "type": "boolean" + }, + "resumptions": { + "type": "object", + "properties": { + "total": { + "type": "integer", + "minimum": 0 + }, + "recent": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VerificationResumption" + }, + "maxItems": 5 + }, + "source": { + "$ref": "#/components/schemas/JsonValue" + } + }, + "additionalProperties": true, + "required": [ + "total", + "recent", + "source" + ] } }, "additionalProperties": true, @@ -4899,6 +4979,129 @@ "status" ] }, + "ChiefRecoveryIssue": { + "type": "object", + "properties": { + "version": { + "const": "chief-recovery/v1", + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "host-recovering", + "worker-exit-unknown", + "stale-snapshot", + "unsupported-route", + "authority-unavailable", + "resource-unavailable", + "execution-failed" + ] + }, + "retryable": { + "type": "boolean" + }, + "conditions": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 16 + }, + "actions": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 16 + }, + "stopBudgetExhausted": { + "type": "boolean" + }, + "attemptId": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Opaque server-issued ID; never infer authority from it." + }, + "attemptStatus": { + "type": "string" + }, + "candidateSha": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "required": [ + "version", + "kind", + "retryable" + ] + }, + "VerificationResumption": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "decisionId": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Opaque server-issued ID; never infer authority from it." + }, + { + "type": "null" + } + ] + }, + "previousRunId": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Opaque server-issued ID; never infer authority from it." + }, + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Opaque server-issued ID; never infer authority from it." + }, + "previousStatus": { + "type": "string" + }, + "candidateSha": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "targetBotId": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Opaque server-issued ID; never infer authority from it." + }, + "at": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": true, + "required": [ + "kind", + "runId", + "at" + ] + }, "DevelopmentQueryInboxListPayload": { "type": "object", "properties": { @@ -8506,7 +8709,8 @@ "agentDefinitions", "integrationAdmissions", "testHistory", - "preparationFailures" + "preparationFailures", + "resumptions" ] }, "offset": { @@ -9372,6 +9576,9 @@ ] }, "maxItems": 9 + }, + "recoveryIssue": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -14886,6 +15093,9 @@ ] }, "maxItems": 9 + }, + "recoveryIssue": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -20200,6 +20410,9 @@ ] }, "maxItems": 9 + }, + "recoveryIssue": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -27768,6 +27981,9 @@ ] }, "maxItems": 9 + }, + "recoveryIssue": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -28009,6 +28225,9 @@ ] }, "maxItems": 9 + }, + "recoveryIssue": { + "$ref": "#/components/schemas/ChiefRecoveryIssue" } }, "additionalProperties": true, @@ -29642,7 +29861,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -29720,7 +29939,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -29798,7 +30017,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -29876,7 +30095,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -30276,7 +30495,7 @@ "notes": "Read-only transport/initiative history; it is not role claim acceptance.", "implementation": { "path": "core/src/host/project-pools.js", - "sha256": "e206c7c6bbb63c3c74808ad2496ff0ca70f89ebc3ab08f9055805c1126f3a4f7" + "sha256": "39abcbd04ea3d5f6bb6cf2cfbf80cab0b59e6afc9a2ef1042a490d80b5266883" } }, { @@ -30354,7 +30573,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-pools.js", - "sha256": "e206c7c6bbb63c3c74808ad2496ff0ca70f89ebc3ab08f9055805c1126f3a4f7" + "sha256": "39abcbd04ea3d5f6bb6cf2cfbf80cab0b59e6afc9a2ef1042a490d80b5266883" } }, { @@ -30432,7 +30651,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -30754,7 +30973,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" } }, { @@ -30836,7 +31055,7 @@ "notes": "Although a query, the downstream integration checkpoint gate requires manager authority and confirmed release.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -30914,7 +31133,7 @@ "notes": "UTF-8 byte offsets, not character offsets; nextOffset preserves character boundaries. Rechecks rights after source I/O. No caller path. environmentRole selects that role workspace latest preparation log and is mutually exclusive with SDK execution role. Preparation is not tests or inference.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -31075,7 +31294,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" } }, { @@ -32963,7 +33182,7 @@ "notes": "New Project Task contracts require actual Chief adjudication; third-party clients cannot forge Chief credentials.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -33071,7 +33290,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -33178,7 +33397,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -33272,7 +33491,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -33366,7 +33585,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -33475,7 +33694,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" } }, { @@ -34050,7 +34269,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" } }, { @@ -34150,7 +34369,7 @@ "notes": "30-minute inspection is a configurable schedule default, not a framework deadline. correctionBudget defaults to 2 (0–4); protocol correction reuses the Chief Session. Exhaustion is incomplete judgement, never automatic consultation or default routing. inspection.lifecycle configures queue/execution deadlines, bounded recovery delay/count and health overdue threshold; timeout is not exit proof.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" } }, { @@ -34422,7 +34641,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -38575,7 +38794,7 @@ "notes": "FBA may only use view=summary, which never borrows native private history. Use v1 roles and consultations for external agents.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -38668,7 +38887,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -38739,7 +38958,7 @@ "notes": "Default latest50; byte-budgeted messages, oversized records use messageIndex source pages. Snapshot prefix survives append, edits invalidate cursor. Private defaults to the current human.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -38812,7 +39031,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -38888,7 +39107,7 @@ "notes": "Human facts and role-authored inference have distinct authority. Updates retain history; memory is not permission.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -39057,7 +39276,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -39149,7 +39368,7 @@ "notes": "External agents should prefer v1 consultation.ask. A consultation is not a Task handoff.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -39222,7 +39441,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -39756,7 +39975,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" } }, { @@ -43439,7 +43658,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-pools.js", - "sha256": "e206c7c6bbb63c3c74808ad2496ff0ca70f89ebc3ab08f9055805c1126f3a4f7" + "sha256": "39abcbd04ea3d5f6bb6cf2cfbf80cab0b59e6afc9a2ef1042a490d80b5266883" }, "aliasOf": "development.query.handoff.get" }, @@ -43515,7 +43734,7 @@ "notes": "Read-only transport/initiative history; it is not role claim acceptance.", "implementation": { "path": "core/src/host/project-pools.js", - "sha256": "e206c7c6bbb63c3c74808ad2496ff0ca70f89ebc3ab08f9055805c1126f3a4f7" + "sha256": "39abcbd04ea3d5f6bb6cf2cfbf80cab0b59e6afc9a2ef1042a490d80b5266883" }, "aliasOf": "development.query.handoff.list" }, @@ -43795,7 +44014,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" }, "aliasOf": "development.query.task.detail" }, @@ -43892,7 +44111,7 @@ "notes": "New Project Task contracts require actual Chief adjudication; third-party clients cannot forge Chief credentials.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.command.task.accept" }, @@ -43985,7 +44204,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.command.task.reject" }, @@ -44077,7 +44296,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.command.attempt.stop" }, @@ -44169,7 +44388,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.command.attempt.cleanup" }, @@ -44548,7 +44767,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.command.attempt.publication-retry" }, @@ -44624,7 +44843,7 @@ "notes": "UTF-8 byte offsets, not character offsets; nextOffset preserves character boundaries. Rechecks rights after source I/O. No caller path. environmentRole selects that role workspace latest preparation log and is mutually exclusive with SDK execution role. Preparation is not tests or inference.", "implementation": { "path": "core/src/host/project-delivery.js", - "sha256": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890" + "sha256": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8" }, "aliasOf": "development.query.attempt.log" }, @@ -44932,7 +45151,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" }, "aliasOf": "development.command.chief.inspect" }, @@ -45030,7 +45249,7 @@ "notes": "30-minute inspection is a configurable schedule default, not a framework deadline. correctionBudget defaults to 2 (0–4); protocol correction reuses the Chief Session. Exhaustion is incomplete judgement, never automatic consultation or default routing. inspection.lifecycle configures queue/execution deadlines, bounded recovery delay/count and health overdue threshold; timeout is not exit proof.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" }, "aliasOf": "development.command.chief.configure" }, @@ -45106,7 +45325,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-chief-control.js", - "sha256": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64" + "sha256": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904" }, "aliasOf": "development.query.chief.state" }, @@ -45372,7 +45591,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-agents.js", - "sha256": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c" + "sha256": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95" }, "aliasOf": "development.command.proposal.publish" }, @@ -45551,7 +45770,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-development-service.js", - "sha256": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2" + "sha256": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa" }, "aliasOf": "development.query.pools" }, @@ -45947,7 +46166,7 @@ "core/src/host/groups.js": "d7f0175c8ad67522aca392d7e47f9d2773c02f525bd6f33d564cbec784217bbf", "core/src/host/harness.js": "49e4b3ad56827d47cf8c5af99600026437c37a5e3b7e01bad2000983f6541af3", "core/src/host/home-lock.js": "c4ad69a0a3a2726888b69ffc46a624644379960834f385d83b93f88b2f05fa04", - "core/src/host/hooks.js": "263e7d7b7c7f7eb8f2e58686cf7e565c76bebe5e36506ba9108367bb0a5da583", + "core/src/host/hooks.js": "f5ab13ebd4506b0549024fc1aac934d772b170478ef12d04528eab9b8c6ab062", "core/src/host/images.js": "18579c262f130ffba10b51cf8f9562730f1ebfa3eff6321f698d72b537fae9d4", "core/src/host/journal.js": "5050048eba15af5520664b7f40dbca1aa457f08c3bbf5ed2798fc99ee9d64906", "core/src/host/meetings.js": "eb68d1cc18ed4370dc8271bd9620966aabb9d8c01001e19fbfc16124852bc063", @@ -45967,12 +46186,13 @@ "core/src/host/project-agent-inbox.js": "046ccf3aa9bb6c2f57c010ba3f798786f2de4acb603b4482e85bfee5ccac96f9", "core/src/host/project-agent-skills.js": "6545f39aec7bbb8bd812558d37422fae1fa0e4fc7ab2fa02f1c50501d29c8445", "core/src/host/project-agent-webhook.js": "129d8ea133a5d85f3b3cb2627dee45fe8617552b422213b93539436dd7a1e5ce", - "core/src/host/project-agents.js": "9edc8a4af53347b2b14d5f26b739a7f6683a4767358b86e520ddcfea1162fa9c", + "core/src/host/project-agents.js": "c4d12c7e047667c11c902953cffd243177f36485f2b0ee3c02ef1ed76fa58b95", "core/src/host/project-assistant.js": "4fa4309d4aa7abf78b6ba4e55ba754f320384f931e0e4a5c6620c859e1b50676", "core/src/host/project-board.js": "dfb0b4bc05ae42335632e3edbba883a03c5da2fb17899b23e8226320ea08aec4", - "core/src/host/project-chief-control.js": "64fc6a71b95888cd289af540bf57bf35e5d6f31ef5cbaffb4dadf8873b881e64", - "core/src/host/project-chief-lifecycle.js": "e854d2fb22b4ea5041ea63a7f072f5af61863c9703ed063b82d0858a15129c68", + "core/src/host/project-chief-control.js": "d3237afcdd34b8b5f2ed2807f0e2512c2a8f47e9719ecbb0be67e2b17197e904", + "core/src/host/project-chief-lifecycle.js": "ed85bb1be90198910b20ffd605895812dfa848fce1049f7bd27b32dbfae3a328", "core/src/host/project-chief-protocol.js": "a09d85e04171b22625d3eed131dd6a5227dc1a946ad39d8a59f017bb75f71936", + "core/src/host/project-chief-recovery.js": "b4efeef007222a87249a808bc49deb84e0e6014c700bb661629a9eb8426327f7", "core/src/host/project-chief.js": "a3f01a053177fd8f10d4b778d79ac98c25aee70aa3c4d967dc7e1da2aea2cf6e", "core/src/host/project-child-inputs.js": "db1f534c4222dba1b500d308b1cd07f0df50a02eb228f520b150a3b79d6f126d", "core/src/host/project-codex-config.js": "4ddf4e4e3e2b324b862ad4f6a6c2a90ac3f3003843b8e446f873b146872c4598", @@ -45984,8 +46204,8 @@ "core/src/host/project-credentials.js": "18fe61057fb920664fc5077d81eee0957f7eaf64821dd10d2d2a3cd463f5fdea", "core/src/host/project-delegations.js": "d4d86192551994a3419e0d2ca0f64bc4d50a75960970d2b79ed9f97b82940903", "core/src/host/project-delivery-template.js": "daa5cc8185abc1ec84421ae3fcd5574a50573cfd1a7c4cb10941158e29c79a13", - "core/src/host/project-delivery.js": "0e2de57798d9d121ccd1e6b93466fe791cd9baa41b5ae59ac52fd76fa7ad0890", - "core/src/host/project-development-service.js": "ae1afd759df9d888d39d658f4b24f55756ecefcbb625d794d629479a575f75f2", + "core/src/host/project-delivery.js": "973fbc98dd4e00a2634f680b77f0eb83d4ae0cbc9fd9071c3c2e1d1e4042dbe8", + "core/src/host/project-development-service.js": "3b05f9d6afaebacc4260eaea2ca1cd9e63bdc0436d97bfe003d4196dea1c61aa", "core/src/host/project-domain.js": "6d6a20727905005f7a8de0247a429a10ffbd1436773ee96a5d7615b2fb763e82", "core/src/host/project-effects.js": "5557782333a0b3437dacde2b6924fe12ea54225098571fa8e404d79a432be549", "core/src/host/project-execution-authority.js": "9c5ffb4bc9153d8f9d9e7901f7aef2aee90cacc19f19d8a0726ac0adb93e3e36", @@ -46007,9 +46227,9 @@ "core/src/host/project-merge-heads.js": "9e5e39b8aaa191c2175feaafcf6524b8a16dbafdfa573f94222b9f71d8482074", "core/src/host/project-metrics.js": "80f209d76955e5a3058fa66757543059fded1030625e5e155be5254494ce5264", "core/src/host/project-native-sessions.js": "4b825e7e509d0a9a0279ee7a70d1de383378a1892b7822dc7d478bd46de21c00", - "core/src/host/project-pools.js": "e206c7c6bbb63c3c74808ad2496ff0ca70f89ebc3ab08f9055805c1126f3a4f7", + "core/src/host/project-pools.js": "39abcbd04ea3d5f6bb6cf2cfbf80cab0b59e6afc9a2ef1042a490d80b5266883", "core/src/host/project-private-file.js": "e8b51d8da7bcd72658eb6eb953dbf9d0054ea0ee519b0229785e8ea0aacef6df", - "core/src/host/project-process-proof.js": "85f4ed03f5191b842ca899b712c99177c059658d43a66e8ddeedc1c06eafcdea", + "core/src/host/project-process-proof.js": "d4c7e8b4f0d2dd177c3e02fd0afe62b4c13ca10348d8f1caf43c011cda47388b", "core/src/host/project-receipt-correction.js": "7162d771d2e28fb4f55af6d492d9b9dc18c9e96f7fbacf660b84c551281ec167", "core/src/host/project-release-github.js": "47c52d0057dd39f1ee627c619970efad36e0134bc0c7bc218d43187230d51d64", "core/src/host/project-releases.js": "ac588064b036f2273c7b61cf3389c1da720e6f2166c37d269afd9d9095eb6885", @@ -46026,7 +46246,7 @@ "core/src/host/project-scheduler.js": "9419010ed18fcfe801cab23562ea2ad53cb4dc500078ff1ec2af4438cc6726a6", "core/src/host/project-service-events.js": "71917d1ab367a9db8bef8c1d1003228255a57528e84a57e76de3a39d08ac28ec", "core/src/host/project-service-sse.js": "df97d6b11156da885a6a47d50a4a77c8c99b6af5eb0066de14312cc512762297", - "core/src/host/project-service-views.js": "2588cfdea77d4f6cc805a1dd89a76bdd8d49e1e341d69209a3d5b93ccd634510", + "core/src/host/project-service-views.js": "d54e38759b9641e3d8eba486812ead5fea2e31097fe1ac17fb1703fd2734540e", "core/src/host/project-service.js": "8ba30c1d7c066867eae06a6612d5bbba3bc2278f34fcf7f1c2242876565dba8f", "core/src/host/project-session-continuation.js": "8d1312bfbe1b69abbaefd8d55dc351bfe19aaa6ae85f16e83a9dd7ff148dfa4d", "core/src/host/project-sources.js": "cf4c11c8e853f3091c4d159b850d06fe7e8bf291fdc616d251eb23d071d63403", diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 40f48e31b..c0ff2ce19 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -995,3 +995,7 @@ The [compatibility contract](project-compatibility.md) inventories original entr private proposal archive and current-scope feedback preserve immutable business evidence; Task views expose correction status without proposal text or paths. See [input contract](project-inputs.md#invalid-receipt-correction-2338). + +#2347 preserves cancelled/unknown Chief invocation barriers until exact exit proof, +with typed recovery health and paged Attempt resumption history. The disposable +unmanaged DSH restart demo exercises the existing scheduler, not another timer. diff --git a/docs/subsystems/plugin-runtime.md b/docs/subsystems/plugin-runtime.md index acef17126..85f9e1106 100644 --- a/docs/subsystems/plugin-runtime.md +++ b/docs/subsystems/plugin-runtime.md @@ -196,3 +196,6 @@ prevents send. The published FlyBot type declares the additive optional seam. #2338 extends the original Workflow pool adapter with same-worker receipt correction. SDK uses the existing input channel; native uses the original idle Agent Session and tool guard. No plugin gains independent Task admission or completion authority. + +#2347 extends the trusted original pool adapter with recovery preflight and +sealed-candidate verification continuation; current rights and exit gates remain. diff --git a/docs/subsystems/project-agents.md b/docs/subsystems/project-agents.md index 75112cb59..5789072ba 100644 --- a/docs/subsystems/project-agents.md +++ b/docs/subsystems/project-agents.md @@ -505,3 +505,7 @@ Personal GitHub comments explicitly reject installation credentials; project aut #2330 adds a trusted internal stopChiefJob lifecycle port. It fences the exact exchange, removes its queued input, waits for SDK release or native whenIdle, restores only its own prompt, and persists invocation-bound exit proof while retaining the resident Session. A late old callback cannot restore another invocation's prompt. Unknown exit retains busy isolation. + +#2347 persists Chief invocation/carrier/host boundaries before execution. Native +restart reconciliation proves the old host birth ended; SDK admission without a +handle never falls back to cancelling the resident native Chief. Stops are bounded. diff --git a/docs/subsystems/project-chief-control.md b/docs/subsystems/project-chief-control.md index 6c5e9a259..866d1893d 100644 --- a/docs/subsystems/project-chief-control.md +++ b/docs/subsystems/project-chief-control.md @@ -1,6 +1,8 @@ --- subsystem: project-chief-control owned_paths: + - core/src/host/project-chief-recovery.js + - core/test/project-chief-recovery.test.js - core/src/host/project-chief-lifecycle.js - core/test/project-chief-lifecycle.test.js - core/src/host/project-chief-protocol.js @@ -264,3 +266,8 @@ Missing-runner facts additionally include stage=runner-startup. testsStarted is configured command invocation; Chief must not interpret it as assertion execution. #2346 adds bounded contextRecovery facts from claim/worker history, including resumed versus reconstructed identity and source hashes. Chief chooses recovery; no automatic human escalation follows a context mismatch. + +#2347 adds typed recovery facts, namespace/birth-checked unmanaged native-Chief +exit, bounded durable stop attempts and safe sealed-candidate verification recovery. +Unknown SDK/container execution stays isolated, including after policy revocation. +See [recovery API and operator guide](../chief-recovery.md). diff --git a/docs/subsystems/project-delivery.md b/docs/subsystems/project-delivery.md index d63304742..1929b1c45 100644 --- a/docs/subsystems/project-delivery.md +++ b/docs/subsystems/project-delivery.md @@ -642,3 +642,8 @@ Vitest or live npm-ci/provider acceptance. patch never repeats code/testing/review or replaces a frozen business output. [Input correction](project-inputs.md#invalid-receipt-correction-2338) records the original evidence and post-failure reconciliation boundaries. + +#2347 allows explicit Chief routing of an interrupted sealed candidate to Tester +or Reviewer only after current requirements, clean candidate, matching evidence +and all prior exits are verified. Same Attempt/Task-role context and immutable +resumptions are preserved. [Recovery contract](../chief-recovery.md). diff --git a/docs/subsystems/project-development-service.md b/docs/subsystems/project-development-service.md index 8bba12a29..f31fe6da9 100644 --- a/docs/subsystems/project-development-service.md +++ b/docs/subsystems/project-development-service.md @@ -238,3 +238,6 @@ v1 github.connection queries and human-only github.connection.configure commands #2336 execution metadata includes startupPhase, while raw resource views retain reservation owner, reclamation reason/effect and exact release proof. Confirmed no-start evidence uses the shared exit predicate; phase alone grants no execution or acceptance. + +#2347 exposes typed Chief recovery health and bounded Attempt resumption metadata, +with full history through the existing hash-bound attempt.context source pages. diff --git a/docs/subsystems/project-harness.md b/docs/subsystems/project-harness.md index cae4f15c9..4d4fd886f 100644 --- a/docs/subsystems/project-harness.md +++ b/docs/subsystems/project-harness.md @@ -458,3 +458,7 @@ Correction turns disable tools while preserving the role filesystem profile; sealed execution logs/result hashes remain original. Stale turn frames are ignored. Native correction waits for the original Agent to idle and denies business tools. See [receipt contract](project-inputs.md#invalid-receipt-correction-2338). + +#2347 distinguishes in-process native-Chief host birth from independent SDK, +container and native-role tool lifecycles; host exit cannot release the latter. +Verification continuation still uses the original Task-role restoration gates. diff --git a/docs/subsystems/project-pools.md b/docs/subsystems/project-pools.md index dd44f9b87..66cf1c67e 100644 --- a/docs/subsystems/project-pools.md +++ b/docs/subsystems/project-pools.md @@ -201,3 +201,7 @@ confirmed cleanup; neither empty workerId nor Task failure alone proves exit. receipt-only corrections through existing Effects, preserving the claim and frozen business intent. Exhaustion remains visible to Chief; no fixed escalation. See [receipt correction](project-inputs.md#invalid-receipt-correction-2338). + +#2347 preflights recovery routes before changing generation, returns unsupported +routes to Chief, and reuses the existing delivery adapter for evidence-supported +Tester/Reviewer continuation. No default next role or extra acceptance authority. diff --git a/patches/ds-bot/files/src/host/hooks.js b/patches/ds-bot/files/src/host/hooks.js index 4c5580b71..79c79a35c 100644 --- a/patches/ds-bot/files/src/host/hooks.js +++ b/patches/ds-bot/files/src/host/hooks.js @@ -1432,6 +1432,8 @@ export function install(rt) { ...(attempt.pr ? [{ pr: attempt.pr.number, sha: attempt.pr.sha, checks: 'unknown' }] : []), ] }, + validateRecovery: input => rt.projectDelivery.validateRecovery(input), + resumeVerification: input => rt.projectDelivery.resumeVerification(input), complete: input => rt.projectDelivery.completePoolRole(input), rework: input => rt.projectDelivery.reworkPool(input), }) diff --git a/patches/ds-bot/files/src/host/project-agents.js b/patches/ds-bot/files/src/host/project-agents.js index 000879965..055b6a100 100644 --- a/patches/ds-bot/files/src/host/project-agents.js +++ b/patches/ds-bot/files/src/host/project-agents.js @@ -1,3 +1,6 @@ +import { inspectionLifecycle } from './project-chief-lifecycle.js' +import { captureProcessScope, priorHostProcessExited } from './project-process-proof.js' +import { nativeChiefBoundary, scopeDigest, corroboratedNativeScope } from './project-chief-recovery.js' import { CHIEF_AUTONOMY } from './project-chief-protocol.js' import { agentHistoryPage } from './project-history-pages.js' import { createProjectAgentInbox } from './project-agent-inbox.js' @@ -364,17 +367,21 @@ export function createProjectAgents(rt) { bot.instructions = priorPrompt.instructions if (priorPrompt.systemPrompt === undefined) delete bot.systemPrompt; else bot.systemPrompt = priorPrompt.systemPrompt } + const run = state.projectExecutions?.[runId] + if (!run || run.invocation !== invocation || run.status !== 'running') throw refuse('Chief invocation changed before execution admission') const delegation = { id: randomUUID(), exchangeId: randomUUID(), actor: { ...actor }, definitionRevision: set.revision, started: false, revoked: false, chiefJob: { runId, invocation, prompt: fullPrompt }, onChiefResult: onResult } delegations.set(agent.id, delegation) agent.lastRequest = { id: randomUUID(), initiatedBy: actor.memberId, executorBotId: agent.botId, exchangeId: delegation.exchangeId, phase: 'queued', context: snapshot, definitionRevision: set.revision, chiefRunId: runId, chiefInvocation: invocation, at: Date.now() } - await rt.save() + run.executionBoundary = { version: 'chief-execution-boundary/v1', kind: (definition.execution?.harness ?? 'flybot') === 'flybot' ? 'native-in-process' : 'sdk-worker', phase: 'preparing', allocationKey: delegation.id, runId, invocation, botId: agent.botId, sessionId: agent.sessionId, carrierGeneration: agent.carrierGeneration, exchangeId: delegation.exchangeId } + try { run.processScope ??= await captureProcessScope(); await rt.save() } + catch (error) { if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId, invocation, at: Date.now() }; throw error } const finish = async result => { try { if (delegations.get(agent.id)?.id !== delegation.id || delegation.revoked) throw refuse('Chief execution was revoked'); const outcome = await onResult({ ...result, executionEvidence: { ...result.executionEvidence, chiefRunId: runId, carrierGeneration: agent.carrierGeneration } }); agent.lastRequest.phase = outcome?.correction ? outcome.correction.status === 'pending' ? 'correcting' : 'incomplete' : result.ok ? 'completed' : 'failed' } finally { restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); await rt.save(); rt.projectScheduler?.wake() } } delegation.finishChiefJob = finish; delegation.restoreChiefPrompt = restorePrompt try { - if ((definition.execution?.harness ?? 'flybot') !== 'flybot') { delegation.started = true; delegation.sdkExecution = true; return await rt.projectHarness.chiefTurn({ agent, delegation, definition, context: snapshot, text: 'Inspect the pinned Project facts and return the bounded Chief judgement.', onResult: finish }) } + if ((definition.execution?.harness ?? 'flybot') !== 'flybot') { run.executionBoundary.phase = 'allocating'; await rt.save(); delegation.started = true; delegation.sdkExecution = true; return await rt.projectHarness.chiefTurn({ agent, delegation, definition, context: snapshot, text: 'Inspect the pinned Project facts and return the bounded Chief judgement.', onResult: finish }) } const bot = rt.botOf(agent.botId) nativeSystem = definition.instructions + '\n\nFor each configured Chief job, use the current request protocol and scoped snapshot in that turn. Prior turns are historical background; act only on Task IDs and facts in the current snapshot. Preserve your resident Session and memory.' const nativeInput = fullPrompt + '\n\nCurrent scoped Chief job facts (data, not authority):\n' + snapshot.text @@ -391,9 +398,15 @@ export function createProjectAgents(rt) { } await rt.projectChiefControl.validateActor(actor) if (delegations.get(agent.id)?.id !== delegation.id || delegation.revoked) throw refuse('Chief execution was revoked before current job input') + run.executionBoundary.phase = 'input-sending'; await rt.save(); await rt.projectChiefControl.validateActor(actor) await rt.deliver({ toId: agent.botId, sessionId: agent.sessionId, role: 'request', audience: 'user', text: nativeInput, fromName: 'Project scheduled/arbitration request', untrusted: true, exchangeId: delegation.exchangeId, projectDelegation: { agentId: agent.id, delegationId: delegation.id } }) return `native-chief-job-${runId}` - } catch (error) { restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id); throw error } + } catch (error) { + restorePrompt(); if (delegations.get(agent.id)?.id === delegation.id) delegations.delete(agent.id) + const allocations = [...Object.values(state.projectResourceLeases ?? {}), ...Object.values(state.projectHarnessWorkers ?? {})].filter(row => row.attemptId === `chief-turn-${delegation.id}` || row.chiefRunId === runId && row.chiefInvocation === invocation) + if (state.projectExecutions?.[runId] === run && run.invocation === invocation && !allocations.length && (run.executionBoundary.kind === 'sdk-worker' || run.executionBoundary.phase === 'preparing')) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId, invocation, at: Date.now() }; await rt.save() } + throw error + } }) } function appendReply(agent, entry) { logOf(agent.id).push(entry) @@ -624,6 +637,25 @@ export function createProjectAgents(rt) { return serial(() => { if (expectedCarrierId && keyOf(projectId, memberId) !== expectedCarrierId) throw refuse('Personal-agent default changed before notification'); return send({ projectId, visibility: 'private', text }, actor) }) } const stoppingChiefJobs = new Map() + async function reconcileChiefExit(run) { + if (!['interrupted', 'failed', 'cancelled'].includes(run.status)) return false + const selected = nativeChiefBoundary(state, run), scope = run.processScope, invocation = run.invocation + if (!selected || !await priorHostProcessExited(corroboratedNativeScope(scope, run, rt.config.projectChiefRecovery?.legacyNativeScopes))) return false + const current = () => state.projectExecutions?.[run.id] === run && run.processScope === scope && run.invocation === invocation && ['interrupted', 'failed', 'cancelled'].includes(run.status) + if (!current()) return false + const proof = { confirmed: true, kind: 'native-chief-prior-host-exited', runId: run.id, invocation, carrierGeneration: selected.boundary.carrierGeneration, scopeSha256: scopeDigest(scope), at: Date.now() } + for (const lease of selected.leases) { + if (!current() || state.projectResourceLeases?.[lease.id] !== lease) return false + await rt.projectResources.release(lease.id, lease.generation, { handle: selected.boundary.exchangeId, exited: true, proof }) + if (lease.status !== 'released') return false + } + if (!current()) return false + run.exitProof = proof + const carrier = agents()[`public-${run.projectId}`], request = carrier?.lastRequest + if (request?.chiefRunId === run.id && request.chiefInvocation === invocation && carrier.carrierGeneration === proof.carrierGeneration) { request.phase = 'interrupted'; request.exitProof = structuredClone(proof) } + await rt.save() + return true + } function stopChiefJob(projectId, runId) { if (stoppingChiefJobs.has(runId)) return stoppingChiefJobs.get(runId) const pending = (async () => { @@ -634,13 +666,17 @@ export function createProjectAgents(rt) { const request = agent.lastRequest request.phase = 'stopping'; await rt.save() let confirmed = false + const run = state.projectExecutions?.[runId] + if (!request.workerId && (run?.executionBoundary?.kind === 'sdk-worker' || Object.values(state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === runId && row.chiefInvocation === request.chiefInvocation))) return { confirmed: false, reason: 'SDK admission is incomplete; reconcile its exact worker and leases without cancelling the native Chief' } if (request.workerId) confirmed = await rt.projectHarness.release(request.workerId) else { const active = await rt.resolveAgent?.(agent.sessionId) if (!active || typeof active.whenIdle !== 'function') return { confirmed: false, reason: 'Native Chief exit cannot be confirmed' } for (const message of [...(active.inbox?.nextStep ?? []), ...(active.inbox?.nextTurn ?? [])]) if (message.source?.exchangeId === request.exchangeId) active.inbox.remove?.(message.id) active.cancel?.({ kind: 'hook', reason: 'Configured Chief inspection deadline' }) - await active.whenIdle() + let timer + const stopped = await Promise.race([active.whenIdle().then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), inspectionLifecycle(state.projectChiefPolicies?.[projectId]?.inspection).stopTimeoutMs) })]).finally(() => clearTimeout(timer)) + if (!stopped) return { confirmed: false, reason: 'Native Chief exit confirmation timed out; execution remains isolated' } const lease = delegation?.resourceLease ?? request.resourceLease if (lease) await rt.projectResources.release(lease.id, lease.generation, { handle: request.exchangeId, exited: true }) confirmed = true @@ -803,7 +839,7 @@ export function createProjectAgents(rt) { } const isDelegationActive = (id, delegationId) => { const entry = delegations.get(id); return entry?.id === delegationId && !entry.revoked } const ensureProject = (projectId, actor) => serial(async () => { await ensure(projectId, actor, 'public'); return ensure(projectId, actor) }) - return { stream, liveFor, hasExchange: exchangeId => [...delegations.values()].some(row => row.exchangeId === exchangeId && !row.revoked), privateIdentity: keyOf, ensurePublic: (projectId, actor) => serial(() => ensure(projectId, actor, 'public')), contextExchange, chiefMemory, chiefBusy: projectId => delegations.has(`public-${projectId}`) || [...stoppingChiefJobs.keys()].some(id => state.projectExecutions?.[id]?.projectId === projectId), stopChiefJob, startChiefJob, endpoint, consultation: consultations.endpoint, contactUser: consultations.contactUser, agentConsultations, agentInbox, inboxExecutionActive, chiefQuestion, askChief, suggestClosure, githubRead, action, wakeWatch, revokeWatchGrant, ensureProject, isDelegationActive, actorOf, context, memory, task, run, onEvent, revoke, revokeSession, removeProject, deliveryDefinitions, recordForBot, roles: roleGovernance, proposeRole, proposeRoleForJob, ROLES } + return { stream, liveFor, hasExchange: exchangeId => [...delegations.values()].some(row => row.exchangeId === exchangeId && !row.revoked), privateIdentity: keyOf, ensurePublic: (projectId, actor) => serial(() => ensure(projectId, actor, 'public')), contextExchange, chiefMemory, reconcileChiefExit, chiefBusy: projectId => delegations.has(`public-${projectId}`) || [...stoppingChiefJobs.keys()].some(id => state.projectExecutions?.[id]?.projectId === projectId), stopChiefJob, startChiefJob, endpoint, consultation: consultations.endpoint, contactUser: consultations.contactUser, agentConsultations, agentInbox, inboxExecutionActive, chiefQuestion, askChief, suggestClosure, githubRead, action, wakeWatch, revokeWatchGrant, ensureProject, isDelegationActive, actorOf, context, memory, task, run, onEvent, revoke, revokeSession, removeProject, deliveryDefinitions, recordForBot, roles: roleGovernance, proposeRole, proposeRoleForJob, ROLES } } export function install(rt) { diff --git a/patches/ds-bot/files/src/host/project-chief-control.js b/patches/ds-bot/files/src/host/project-chief-control.js index 2994f1f5d..bbe19f0dc 100644 --- a/patches/ds-bot/files/src/host/project-chief-control.js +++ b/patches/ds-bot/files/src/host/project-chief-control.js @@ -1,3 +1,4 @@ +import { classifyChiefRecovery, progressCheckpoint } from './project-chief-recovery.js' // Chief arbitration uses existing Tasks, Runs, Sessions and resource leases. import { randomUUID, createHash } from 'node:crypto' import { readFileSync } from 'node:fs' @@ -20,22 +21,13 @@ export function inspectionResources(state, task) { const historical = rows.filter(row => row.attemptId !== task.deliveryAttemptId && row.status === 'released' && proven.has(row.releaseProof)) return { resources: rows.filter(row => !historical.includes(row)), ...(historical.length ? { releasedResourceHistory: { count: historical.length, allConfirmedReleased: true, sha256: digest(historical), source: 'projectResourceLeases and projectHarnessWorkers; original records retained' } } : {}) } } -function verifiedProgress(evidence = []) { - return evidence.filter(row => row.kind === 'host-tests' && row.passed === true && typeof row.sha === 'string' && typeof row.sha256 === 'string') -} export function progressFingerprint(attempt) { const tests = attempt?.evidence?.tests return digest(tests?.passed === true && tests.sha === attempt.candidateSha ? { testedCandidate: tests.sha } : null) } export function inspectionFacts(state, task, thresholds, now = Date.now()) { const c = task.collaboration, attempt = state.projectAttempts?.[task.deliveryAttemptId], fingerprint = progressFingerprint(attempt) - let checkpoint = 0, lastMeaningfulProgressAt = null, progressEvidence = [] - const seen = new Set() - for (const [index, receipt] of (c?.history ?? []).entries()) { - const fresh = verifiedProgress(receipt.evidence).filter(row => !seen.has(row.sha)) - for (const row of fresh) seen.add(row.sha) - if (fresh.length) { checkpoint = index + 1; lastMeaningfulProgressAt = receipt.createdAt; progressEvidence = fresh.map(row => ({ handoffId: receipt.id, sha: row.sha, testLogSha256: row.sha256 })) } - } + const { checkpoint, lastMeaningfulProgressAt, progressEvidence } = progressCheckpoint(c?.history) const handoffs = (c?.history.length ?? 0) - checkpoint const worker = c?.claim?.workerId ? state.projectHarnessWorkers?.[c.claim.workerId] : null const reasons = [] @@ -67,11 +59,12 @@ export function createProjectChiefControl(rt) { const stopping = new Map() let pumping = false, reconciling = null rt.projectEffects?.register('chief.inspection-recovery', { run: row => ({ runId: row.payload.runId }) }, { replace: true }) + rt.projectEffects?.register('chief.inspection-stop', { run: row => ({ runId: row.payload.runId, stopAttempt: row.payload.stopAttempt }) }, { replace: true }) rt.projectEffects?.register('chief.correction', { run: row => ({ runId: row.payload.runId, round: row.payload.round }) }, { replace: true }) const policyOf = id => ownEntry(policies(), id) function requireJobActor(actor, projectId) { const job = active.get(actor?.chiefJobId), policy = policyOf(projectId) - if (rt.readOnly?.() || !job || job.token !== actor.chiefToken || job.run.projectId !== projectId || job.run.initiatedBy !== actor.memberId || job.run.status !== 'running' || !policy?.enabled || policy.revision !== job.run.policyRevision || policy.authorityMemberId !== actor.memberId) throw refuse('Chief job authority is missing, revoked or stale') + if (rt.readOnly?.() || !job || job.token !== actor.chiefToken || job.run.projectId !== projectId || job.run.initiatedBy !== actor.memberId || job.run.status !== 'running' || job.run.invocation !== job.invocation || !policy?.enabled || policy.revision !== job.run.policyRevision || policy.authorityMemberId !== actor.memberId) throw refuse('Chief job authority is missing, revoked or stale') return job } async function validateActor(actor) { @@ -210,11 +203,11 @@ export function createProjectChiefControl(rt) { } async function completed(job, result) { const { run } = job - if (active.get(run.id) !== job || run.status !== 'running' || job.resultInProgress) return { ignored: true } + if (active.get(run.id) !== job || run.invocation !== job.invocation || run.status !== 'running' || job.resultInProgress) return { ignored: true } job.resultInProgress = true let correcting = false try { - if (!result.ok) throw refuse('Chief execution did not complete; inspect its resource evidence') + if (!result.ok) throw Object.assign(refuse('Chief execution did not complete; inspect its resource evidence'), { recovery: result.recoveryIssue ?? classifyChiefRecovery(new Error('Chief execution failed')) }) await validateActor(job.actor) const execution = result.executionEvidence, carrier = rt.state.projectAgents?.[`public-${run.projectId}`] if (!execution || execution.chiefRunId !== run.id || execution.carrierGeneration !== carrier?.carrierGeneration) throw refuse('Chief carrier changed or execution evidence is missing') @@ -237,6 +230,7 @@ export function createProjectChiefControl(rt) { if (run.appliedDecisions?.some(row => row.taskId === decision.taskId)) continue const saved = job.snapshot.find(row => row.taskId === decision.taskId) if (!saved) throw chiefProtocolError('taskId', 'an exact Task in this invocation snapshot', decision.taskId) + if (['route', 'return', 'request-tests'].includes(decision.action)) await rt.projectPools?.validateChiefRoute?.({ taskId: decision.taskId, targetBotId: decision.action === 'request-tests' ? `${run.projectId}:tester` : decision.targetBotId, expectedGeneration: saved.generation }, job.actor) try { rt.projectInputs?.validateChiefReceipt?.(decision.taskId, saved, run, decision.inputReceipts) } catch (error) { if (!error.receiptFeedback?.correctable) throw error @@ -260,7 +254,7 @@ export function createProjectChiefControl(rt) { if (rt.state.tasks[decision.taskId]?.chiefActionIssue) delete rt.state.tasks[decision.taskId].chiefActionIssue if (rt.state.tasks[decision.taskId]?.chiefDecisionIssue) delete rt.state.tasks[decision.taskId].chiefDecisionIssue } catch (error) { - if (run.purpose !== 'inspection') throw error + if (run.purpose !== 'inspection' || error.code === 'bot/chief-protocol') throw error const failure = { taskId: decision.taskId, code: error.code ?? 'bot/chief', message: String(error.message).slice(0, 1000), at: Date.now() }; failures.push(failure) const task = rt.state.tasks[decision.taskId]; if (task?.projectId === run.projectId) task.chiefActionIssue = { runId: run.id, ...failure } await rt.save() @@ -268,14 +262,14 @@ export function createProjectChiefControl(rt) { } await validateActor(job.actor) if (run.correction) run.correction.status = 'resolved' - delete run.failure + delete run.failure; delete run.recoveryIssue run.waitingFor = null; run.status = failures.length ? 'partial' : 'done'; run.decisionFailures = failures; if (failures.length) run.failure = `${failures.length} Task decisions were refused; independent decisions continued`; run.summary = parsed.summary.slice(0, 8000); run.decisionIds = results.map(row => row.id); run.executionEvidence = result.executionEvidence ?? null waiters.get(run.id)?.resolve() } catch (error) { if (active.get(run.id) !== job || run.status !== 'running') return { ignored: true } if (error.code === 'bot/chief-protocol') { const previous = run.correction, round = (previous?.round ?? 0) + 1, limit = run.correctionLimit ?? correctionBudget(policyOf(run.projectId)) - const feedback = { ...(error.receiptFeedback ? { receiptFeedback: error.receiptFeedback } : {}), code: error.code, taskId: error.taskId, fields: error.fields, message: error.message.slice(0, 500), at: Date.now(), sideEffects: 'response protocol rejected before applying its decisions', appliedTaskIds: (run.appliedDecisions ?? []).map(row => row.taskId) } + const feedback = { ...(error.recovery ? { recovery: error.recovery } : {}), ...(error.receiptFeedback ? { receiptFeedback: error.receiptFeedback } : {}), code: error.code, taskId: error.taskId, fields: error.fields, message: error.message.slice(0, 500), at: Date.now(), sideEffects: 'response protocol rejected before applying its decisions', appliedTaskIds: (run.appliedDecisions ?? []).map(row => row.taskId) } run.correction = protocolCorrection(previous, { version: 'chief-correction/v1', limit, feedback, priorExecution: job.executionEvidence ?? null }) run.status = round <= limit ? 'correction-pending' : 'incomplete'; run.failure = feedback.message if (rt.projectEffects) { @@ -301,27 +295,33 @@ export function createProjectChiefControl(rt) { if (!correcting) waiters.get(run.id)?.reject(refuse('Chief judgement incomplete; correction budget exhausted')) return { correction: structuredClone(run.correction) } } - run.status = 'failed'; run.failure = String(error.message).slice(0, 1000) + run.status = 'failed'; run.failure = String(error.message).slice(0, 1000); run.recoveryIssue = error.recovery ?? classifyChiefRecovery(error) + if (run.purpose === 'inspection' && run.recoveryIssue.retryable) { + const config = inspectionLifecycle(policyOf(run.projectId)?.inspection) + run.status = 'interrupted'; run.waitingFor = 'recovery'; run.recovery = { ...run.recovery, status: 'waiting-exit', attempts: run.recovery?.attempts ?? 0, notBefore: Date.now() + config.recoveryDelayMs } + } for (const audit of Object.values(rt.state.projectChiefDecisions ?? {})) if (audit.runId === run.id && audit.status === 'requested') { audit.status = 'rejected'; audit.failure = run.failure; audit.finishedAt = Date.now() } if (run.taskId) { const task = ownEntry(rt.state.tasks, run.taskId); if (task?.projectTask && task.projectTask.admission.status !== 'accepted') task.projectTask.admission.failure = { chiefRunId: run.id, reason: run.failure, at: Date.now() }; if (task?.collaboration && !terminal.has(task.status)) { task.collaboration.status = 'blocked'; if (task.collaboration.claim?.chiefRunId === run.id) task.collaboration.claim.status = 'failed'; task.collaboration.block = { reason: run.failure, at: Date.now() } } } waiters.get(run.id)?.reject(error) } finally { if (run.invocation !== job.invocation) return; run.lastInvocationFinishedAt = Date.now(); if (!correcting) run.finishedAt = Date.now(); if (active.get(run.id) === job) active.delete(run.id); if (!correcting) waiters.delete(run.id); await rt.save(); if (correcting) rt.projectScheduler?.wake() } } async function inspectionExited(run) { + await rt.projectAgents.reconcileChiefExit?.(run) const workers = Object.values(rt.state.projectHarnessWorkers ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined ? row.id === run.workerId : row.chiefInvocation === run.invocation)) const leases = Object.values(rt.state.projectResourceLeases ?? {}).filter(row => row.chiefRunId === run.id && row.chiefInvocation === run.invocation || workers.some(worker => worker.leaseId === row.id)) if (leases.some(row => row.status !== 'released')) return false if (workers.length) return workers.every(row => confirmedWorkerExit(rt.state, row) && leases.some(lease => lease.id === row.leaseId && lease.projectId === row.projectId && lease.attemptId === row.attemptId && lease.role === row.role && lease.resourceId === row.resourceId && lease.generation === row.generation && lease.worker?.handle === row.id)) - if (leases.length && leases.every(row => row.releaseProof && (!row.worker && row.releaseProof.kind === 'unbound-reservation-release' || row.worker?.kind === 'native-turn' && ['host-confirmed-worker-exit', 'systemd-prior-invocation-exited'].includes(row.releaseProof.kind)))) return true + if (leases.length && leases.every(row => row.releaseProof && (!row.worker && row.releaseProof.kind === 'unbound-reservation-release' || row.worker?.kind === 'native-turn' && ['host-confirmed-worker-exit', 'systemd-prior-invocation-exited', 'native-chief-prior-host-exited'].includes(row.releaseProof.kind)))) return true const agent = rt.state.projectAgents?.[`public-${run.projectId}`], proof = agent?.lastRequest?.exitProof if (proof?.confirmed && proof.runId === run.id && agent.lastRequest.chiefRunId === run.id && proof.carrierGeneration === agent.carrierGeneration && proof.invocation === run.invocation) return true - if (['native-chief-quiescent', 'sdk-chief-confirmed-exit', 'chief-never-dispatched'].includes(run.exitProof?.kind) && run.exitProof?.confirmed && run.exitProof.runId === run.id && run.exitProof.invocation === run.invocation) return true + if (['native-chief-quiescent', 'native-chief-prior-host-exited', 'sdk-chief-confirmed-exit', 'chief-never-dispatched'].includes(run.exitProof?.kind) && run.exitProof?.confirmed && run.exitProof.runId === run.id && run.exitProof.invocation === run.invocation) return true return Boolean(await priorScopeExited(run.processScope)) } async function inspectionBarrier(current) { for (const run of Object.values(rt.state.projectExecutions ?? {})) { - if (run.id === current.id || run.projectId !== current.projectId || run.kind !== 'chief-control' || run.status !== 'interrupted' || !run.startedAt) continue + if (run.id === current.id || run.projectId !== current.projectId || run.kind !== 'chief-control' || run.status !== 'interrupted' && run.recovery?.exitConfirmed !== false || !run.startedAt) continue if (!await inspectionExited(run)) return true + if (run.recovery?.exitConfirmed === false) { run.recovery.exitConfirmed = true; if (run.status !== 'interrupted') run.recovery.status = 'isolated-exit-confirmed'; await rt.save() } } return false } @@ -343,23 +343,33 @@ export function createProjectChiefControl(rt) { await rt.save() } } - if (run.status !== 'interrupted') continue - if (!policy?.enabled || policy.revision !== run.policyRevision || run.source?.scheduleId && (!rt.state.projectScheduledTasks?.[run.source.scheduleId]?.enabled || run.source.scheduleRevision !== undefined && rt.state.projectScheduledTasks[run.source.scheduleId].revision !== run.source.scheduleRevision)) { run.status = 'cancelled'; run.failure = 'Inspection recovery authority or schedule changed'; await rt.save(); continue } + if (run.status !== 'interrupted' && run.recovery?.exitConfirmed !== false) continue + if (run.status === 'interrupted' && (!policy?.enabled || policy.revision !== run.policyRevision || run.source?.scheduleId && (!rt.state.projectScheduledTasks?.[run.source.scheduleId]?.enabled || run.source.scheduleRevision !== undefined && rt.state.projectScheduledTasks[run.source.scheduleId].revision !== run.source.scheduleRevision))) { run.status = 'cancelled'; run.failure = 'Inspection recovery authority or schedule changed'; active.delete(run.id); await rt.save() } if (!run.recovery) { run.recovery = { status: 'waiting-exit', notBefore: now + config.recoveryDelayMs, attempts: 0 }; await rt.save() } const exited = await inspectionExited(run) run.recovery.exitConfirmed = exited if (!exited) { - run.recovery.status = 'waiting-exit' - if (!stopping.has(run.id) && now >= (run.recovery.nextStopAt ?? 0) && rt.projectAgents.stopChiefJob) { + run.recovery.status = (run.recovery.stopAttempts ?? 0) >= config.maxStopAttempts ? 'exit-unconfirmed' : 'waiting-exit' + run.recoveryIssue = { version: 'chief-recovery/v1', kind: 'worker-exit-unknown', retryable: true, conditions: ['matching execution exit proof'], actions: ['inspect registered worker and lease evidence'], stopBudgetExhausted: (run.recovery.stopAttempts ?? 0) >= config.maxStopAttempts } + if (!stopping.has(run.id) && (run.recovery.stopAttempts ?? 0) < config.maxStopAttempts && now >= (run.recovery.nextStopAt ?? 0) && rt.projectAgents.stopChiefJob) { + run.recovery.stopAttempts = (run.recovery.stopAttempts ?? 0) + 1 run.recovery.nextStopAt = now + config.recoveryDelayMs const invocation = run.invocation + let stopIntent + try { + if (rt.projectEffects) { + stopIntent = rt.projectEffects.stage({ kind: 'chief.inspection-stop', key: `${run.id}:${invocation}:${run.recovery.stopAttempts}`, projectId: run.projectId, payload: { runId: run.id, invocation, stopAttempt: run.recovery.stopAttempts } }) + await rt.projectEffects.persist(stopIntent) + } else await rt.save() + } catch (error) { if (!stopIntent || rt.projectEffects.discardUncommitted(stopIntent)) --run.recovery.stopAttempts; throw error } const pending = Promise.resolve().then(() => rt.projectAgents.stopChiefJob(run.projectId, run.id)).then(async proof => { - if (proof?.confirmed && run.invocation === invocation && run.status === 'interrupted') { run.exitProof = { ...proof, runId: run.id, invocation, at: Date.now() }; await rt.save(); rt.projectScheduler?.wake() } + if (proof?.confirmed && run.invocation === invocation && ['interrupted', 'cancelled', 'failed'].includes(run.status)) { run.exitProof = { ...proof, runId: run.id, invocation, at: Date.now() }; await rt.save(); rt.projectScheduler?.wake() } }).catch(error => { run.recovery.stopFailure = String(error.message).slice(0, 300) }).finally(() => stopping.delete(run.id)) stopping.set(run.id, pending) } continue } + if (run.status !== 'interrupted') { run.recovery.status = 'isolated-exit-confirmed'; active.delete(run.id); await rt.save(); continue } if (active.has(run.id) || rt.projectAgents.chiefBusy(run.projectId)) continue if (run.recovery.supersededBy) { run.status = 'cancelled'; run.recovery.status = 'coalesced'; await rt.save(); continue } if ((run.recovery.attempts ?? 0) >= config.maxRecoveries) { run.status = 'failed'; run.recovery.status = 'exhausted'; await rt.save(); continue } @@ -368,6 +378,8 @@ export function createProjectChiefControl(rt) { await rt.projectDelegations.background(policy.authorityMemberId) rt.requireProjectAccess({ memberId: policy.authorityMemberId }, run.projectId, 'configure') if (policyOf(run.projectId) !== policy || policy.revision !== run.policyRevision) throw refuse('Inspection policy changed during recovery') + if (run.recovery.stopAttempts) run.recovery.stopHistory = [...(run.recovery.stopHistory ?? []), { invocation: run.invocation, attempts: run.recovery.stopAttempts, lastStopAt: run.recovery.nextStopAt }].slice(-5) + run.recovery.stopAttempts = 0; delete run.recovery.nextStopAt run.recovery.attempts = (run.recovery.attempts ?? 0) + 1; run.recovery.status = 'requeued'; run.recovery.previousInvocation = run.invocation run.status = 'queued'; run.queuedAt = now; delete run.finishedAt if (rt.projectEffects) { @@ -396,11 +408,11 @@ export function createProjectChiefControl(rt) { if (run.invocation === job.invocation) { run.workerId = workerId; if (run.status === 'running') run.waitingFor = 'execution'; run.workerGeneration = rt.state.projectHarnessWorkers?.[workerId]?.generation ?? null; await rt.save() } } catch (error) { if (active.get(run.id) !== job || run.status !== 'running') { - if (run.invocation === job.invocation && !Object.values(rt.state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === run.id) && rt.state.projectAgents?.[`public-${run.projectId}`]?.lastRequest?.chiefRunId !== run.id) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId: run.id, invocation: job.invocation, at: Date.now() }; await rt.save() } + if (run.invocation === job.invocation && !Object.values(rt.state.projectHarnessWorkers ?? {}).some(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined ? row.id === run.workerId : row.chiefInvocation === job.invocation)) && !(rt.state.projectAgents?.[`public-${run.projectId}`]?.lastRequest?.chiefRunId === run.id && rt.state.projectAgents[`public-${run.projectId}`].lastRequest.chiefInvocation === job.invocation)) { run.exitProof = { confirmed: true, kind: 'chief-never-dispatched', runId: run.id, invocation: job.invocation, at: Date.now() }; await rt.save() } return } if (error.code === 'bot/busy') { run.status = 'queued'; active.delete(run.id); await rt.save() } - else { run.dispatchFailure = String(error.message).slice(0, 1000); await completed(job, { ok: false, text: String(error.message) }) } + else { run.dispatchFailure = String(error.message).slice(0, 1000); run.recoveryIssue = classifyChiefRecovery(error); await completed(job, { ok: false, text: String(error.message), recoveryIssue: run.recoveryIssue }) } } } async function pump() { @@ -419,8 +431,7 @@ export function createProjectChiefControl(rt) { for (const run of Object.values(rt.state.projectExecutions ?? {}).filter(row => row.kind === 'chief-control' && row.status === 'interrupted' && row.taskId && row.purpose !== 'inspection')) { const task = rt.state.tasks[run.taskId], policy = policyOf(run.projectId) if (!policy?.enabled || task?.collaboration?.claim?.chiefRunId !== run.id || task.collaboration.claim.status !== 'interrupted') continue - const worker = rt.state.projectHarnessWorkers?.[run.workerId] - if (worker?.status !== 'released' && !await priorScopeExited(run.processScope)) continue + if (!await inspectionExited(run)) continue try { await rt.projectPools.retryChief(task.id, { memberId: policy.authorityMemberId }); run.recovery = { status: 'requeued', at: Date.now() }; await rt.save() } catch (error) { run.recovery = { status: 'blocked', reason: error.message, at: Date.now() } } } for (const task of Object.values(rt.state.tasks)) if (task.collaboration?.status === 'arbitration' && !terminal.has(task.status) && policyOf(task.projectId)?.enabled) await request(task.projectId, 'arbitration', { handoffId: task.collaboration.currentHandoffId }, task.id) @@ -431,7 +442,7 @@ export function createProjectChiefControl(rt) { if (!policy?.enabled || policy.revision !== run.policyRevision) { run.status = 'cancelled'; waiters.get(run.id)?.reject(refuse('Chief policy changed before execution')); waiters.delete(run.id); await rt.save(); continue } const actor = { memberId: policy.authorityMemberId, chiefJobId: run.id, chiefToken: randomUUID() } const job = { run, actor, token: actor.chiefToken, snapshot: [], invocation: (run.invocation ?? 0) + 1 } - run.invocation = job.invocation + run.invocation = job.invocation; run.executionBoundary = null; delete run.workerId; delete run.workerGeneration run.status = 'running'; run.waitingFor = 'preparing'; run.startedAt = Date.now(); active.set(run.id, job); await rt.save() void dispatchJob(job).catch(error => rt.warn?.('Chief dispatch result could not be saved: %s', error.message)) } @@ -439,7 +450,7 @@ export function createProjectChiefControl(rt) { } function recover() { let changed = false - for (const run of Object.values(rt.state.projectExecutions ?? {})) if (run.kind === 'chief-control' && run.status === 'running') { run.status = 'interrupted'; run.failure = 'Host restarted before Chief result was validated'; changed = true } + for (const run of Object.values(rt.state.projectExecutions ?? {})) if (run.kind === 'chief-control' && run.status === 'running') { run.status = 'interrupted'; run.waitingFor = 'recovery'; run.recoveryIssue = { version: 'chief-recovery/v1', kind: 'host-recovering', retryable: true, conditions: ['prior execution exit reconciliation'], actions: ['wait for the existing recovery pass'] }; run.failure = 'Host restarted before Chief result was validated'; changed = true } return changed } return { configure, view, request, scheduled, scheduledState, pump, recover, requireJobActor, validateActor, applyDecision, template: structuredClone(template) } diff --git a/patches/ds-bot/files/src/host/project-chief-lifecycle.js b/patches/ds-bot/files/src/host/project-chief-lifecycle.js index 838aee03b..6a4d468f1 100644 --- a/patches/ds-bot/files/src/host/project-chief-lifecycle.js +++ b/patches/ds-bot/files/src/host/project-chief-lifecycle.js @@ -1,9 +1,10 @@ +import { progressCheckpoint } from './project-chief-recovery.js' // Mechanical inspection lifecycle; no business routing or automatic human escalation. -export const INSPECTION_LIFECYCLE = { queueTimeoutMs: 600000, runTimeoutMs: 900000, recoveryDelayMs: 5000, maxRecoveries: 2, overdueMs: 3600000 } +export const INSPECTION_LIFECYCLE = { queueTimeoutMs: 600000, runTimeoutMs: 900000, recoveryDelayMs: 5000, maxRecoveries: 2, stopTimeoutMs: 10000, maxStopAttempts: 3, overdueMs: 3600000 } export function inspectionLifecycle(inspection) { if (inspection?.lifecycle !== undefined && (!inspection.lifecycle || typeof inspection.lifecycle !== 'object' || Array.isArray(inspection.lifecycle))) throw new Error('Invalid Chief inspection lifecycle configuration') const value = { ...INSPECTION_LIFECYCLE, ...inspection?.lifecycle } - for (const [key, number] of Object.entries(value)) if (!(key in INSPECTION_LIFECYCLE) || !Number.isSafeInteger(number) || number < (key === 'maxRecoveries' ? 0 : 1) || number > (key === 'maxRecoveries' ? 4 : 86400000)) throw new Error('Invalid Chief inspection lifecycle configuration: ' + key) + for (const [key, number] of Object.entries(value)) if (!(key in INSPECTION_LIFECYCLE) || !Number.isSafeInteger(number) || number < (['maxRecoveries', 'maxStopAttempts'].includes(key) ? 0 : 1) || number > (key === 'maxRecoveries' ? 4 : key === 'maxStopAttempts' ? 10 : 86400000)) throw new Error('Invalid Chief inspection lifecycle configuration: ' + key) return value } export function inspectionHealth(state, projectId, now = Date.now()) { @@ -15,10 +16,10 @@ export function inspectionHealth(state, projectId, now = Date.now()) { const lastSuccessAt = last(runs.filter(row => row.status === 'done').map(row => row.finishedAt)) const enabled = Boolean(policy?.enabled && schedule?.enabled) const anchor = lastSuccessAt ?? policy?.updatedAt ?? schedule?.createdAt ?? now - const active = runs.filter(row => ['queued', 'running', 'correction-pending', 'interrupted'].includes(row.status)).map(row => ({ runId: row.id, occurrenceId: row.source?.occurrenceId ?? null, status: row.status, waitingFor: row.waitingFor ?? null, ageMs: Math.max(0, now - (row.startedAt ?? row.createdAt)), failure: row.failure ?? null, recovery: row.recovery ?? null, deadlineAt: (row.status === 'queued' ? (row.queuedAt ?? row.createdAt) + config.queueTimeoutMs : (row.startedAt ?? row.createdAt) + config.runTimeoutMs) })) + const active = runs.filter(row => (['queued', 'running', 'correction-pending', 'interrupted'].includes(row.status) || row.recovery?.exitConfirmed === false)).map(row => ({ runId: row.id, occurrenceId: row.source?.occurrenceId ?? null, status: row.status, waitingFor: row.waitingFor ?? null, ageMs: Math.max(0, now - (row.startedAt ?? row.createdAt)), failure: row.failure ?? null, recovery: row.recovery ?? null, recoveryIssue: row.recoveryIssue ?? null, nextRecoveryAt: row.recovery?.exitConfirmed === false ? (row.recovery.stopAttempts ?? 0) < config.maxStopAttempts ? row.recovery.nextStopAt ?? row.recovery.notBefore ?? null : null : row.recovery?.notBefore ?? null, deadlineAt: (row.status === 'queued' ? (row.queuedAt ?? row.createdAt) + config.queueTimeoutMs : (row.startedAt ?? row.createdAt) + config.runTimeoutMs) })) const latestDelivery = deliveries.toSorted((a,b) => (b.at ?? 0) - (a.at ?? 0))[0] const latest = runs.toSorted((a,b) => b.createdAt - a.createdAt)[0] - return { enabled, status: !enabled ? 'disabled' : now - anchor > config.overdueMs ? 'overdue' : active.some(row => row.status === 'interrupted' || row.deadlineAt <= now) || latestDelivery?.status === 'failed' && (latestDelivery.finishedAt ?? latestDelivery.at) >= (lastSuccessAt ?? 0) || ['failed', 'partial', 'incomplete'].includes(latest?.status) ? 'degraded' : lastSuccessAt === null ? 'pending' : 'healthy', lastOccurrenceAt: last(deliveries.map(row => row.at)), lastStartedAt: last(runs.map(row => row.startedAt)), lastCompletedAt: last(runs.map(row => row.finishedAt)), lastSuccessAt, lastAppliedDecisionAt: last(Object.values(state.projectChiefDecisions ?? {}).filter(row => row.projectId === projectId && row.status === 'applied' && !['continue', 'needs-human'].includes(row.action)).map(row => row.finishedAt)), overdueMs: config.overdueMs, active } + return { enabled, status: !enabled ? 'disabled' : now - anchor > config.overdueMs ? 'overdue' : active.some(row => row.status === 'interrupted' || row.recovery?.exitConfirmed === false || row.deadlineAt <= now) || latestDelivery?.status === 'failed' && (latestDelivery.finishedAt ?? latestDelivery.at) >= (lastSuccessAt ?? 0) || ['failed', 'partial', 'incomplete'].includes(latest?.status) ? 'degraded' : lastSuccessAt === null ? 'pending' : 'healthy', lastOccurrenceAt: last(deliveries.map(row => row.at)), lastStartedAt: last(runs.map(row => row.startedAt)), lastCompletedAt: last(runs.map(row => row.finishedAt)), lastSuccessAt, lastMeaningfulProgressAt: last(Object.values(state.tasks ?? {}).filter(row => row.projectId === projectId).map(row => progressCheckpoint(row.collaboration?.history).lastMeaningfulProgressAt)), lastAppliedDecisionAt: last(Object.values(state.projectChiefDecisions ?? {}).filter(row => row.projectId === projectId && row.status === 'applied' && !['continue', 'needs-human'].includes(row.action)).map(row => row.finishedAt)), overdueMs: config.overdueMs, active } } export function scheduledInspectionState(state, row, delivery) { diff --git a/patches/ds-bot/files/src/host/project-chief-recovery.js b/patches/ds-bot/files/src/host/project-chief-recovery.js new file mode 100644 index 000000000..4880f219e --- /dev/null +++ b/patches/ds-bot/files/src/host/project-chief-recovery.js @@ -0,0 +1,51 @@ +// Recovery facts and boundaries; no business role selection or second dispatcher. +import { createHash } from 'node:crypto' +export const scopeDigest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex') +export function chiefRecoveryError(kind, message, { retryable = false, conditions = [], actions = [], ...facts } = {}) { + return Object.assign(new Error(message), { code: 'bot/chief-recovery', recovery: { version: 'chief-recovery/v1', kind, retryable, conditions, actions, ...facts } }) +} +export function nativeChiefBoundary(state, run) { + const boundary = run.executionBoundary + if (boundary) { + if (boundary.version !== 'chief-execution-boundary/v1' || boundary.kind !== 'native-in-process' || boundary.runId !== run.id || boundary.invocation !== run.invocation || !boundary.botId || !boundary.sessionId || !Number.isSafeInteger(boundary.carrierGeneration)) return null + } else if (boundary !== undefined || run.workerId !== `native-chief-job-${run.id}`) return null + // A native host proof must never hide an independent worker, including a + // worker registered before its handle was copied into the Chief Run. + const workers = Object.values(state.projectHarnessWorkers ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined || row.chiefInvocation === run.invocation)) + if (workers.length) return null + const leases = Object.values(state.projectResourceLeases ?? {}).filter(row => row.chiefRunId === run.id && (row.chiefInvocation === undefined || row.chiefInvocation === run.invocation)) + const carrier = state.projectAgents?.[`public-${run.projectId}`], request = carrier?.lastRequest + const selected = boundary ?? (request?.chiefRunId === run.id && request.chiefInvocation === run.invocation && !request.workerId ? { version: 'chief-execution-boundary/v1', kind: 'native-in-process', runId: run.id, invocation: run.invocation, botId: carrier.botId, sessionId: carrier.sessionId, carrierGeneration: carrier.carrierGeneration, exchangeId: request.exchangeId } : null) + if (!selected || !Number.isSafeInteger(run.invocation) || run.invocation < 1) return null + if (leases.some(row => row.projectId !== run.projectId || row.chiefInvocation !== run.invocation || row.worker?.kind !== 'native-turn' || row.worker.botId !== selected.botId || row.worker.sessionId !== selected.sessionId || row.worker.handle !== selected.exchangeId || row.preparation?.status === 'active')) return null + return { boundary: selected, leases } +} + +export function classifyChiefRecovery(error) { + if (error.recovery) return structuredClone(error.recovery) + const message = String(error.message) + const kind = /stale|generation|snapshot.*changed/i.test(message) ? 'stale-snapshot' : /worker.*exit|confirmed.*exit|unreachable/i.test(message) ? 'worker-exit-unknown' : /interrupted delivery|candidate.*missing|matching passing tests/i.test(message) ? 'unsupported-route' : ['bot/forbidden', 'bot/delegation', 'bot/execution-revoked'].includes(error.code) ? 'authority-unavailable' : ['bot/resource', 'bot/busy'].includes(error.code) ? 'resource-unavailable' : 'execution-failed' + return { version: 'chief-recovery/v1', kind, retryable: ['worker-exit-unknown', 'resource-unavailable'].includes(kind), conditions: kind === 'worker-exit-unknown' ? ['matching exit proof; absence of a host controller is insufficient'] : kind === 'stale-snapshot' ? ['fresh scoped snapshot'] : ['inspect current scoped evidence'], actions: kind === 'unsupported-route' ? ['select an evidence-supported role or rework explicitly'] : [] } +} + +// Optional operator corroboration for pre-namespace records; scoped to one exact +// old native invocation. It is not a PID-only exit claim or a public Bot grant. +export function corroboratedNativeScope(scope, run, configuration) { + if (scope?.pidNamespace !== undefined || configuration === undefined) return scope + if (!Array.isArray(configuration) || configuration.length > 20) throw new Error('legacyNativeScopes must contain at most20 exact operator attestations') + const selected = configuration.find(row => row.runId === run.id && row.invocation === run.invocation && row.scopeSha256 === scopeDigest(scope)) + if (!selected) return scope + if (!/^pid:\[\d+\]$/.test(selected.pidNamespace ?? '') || scope?.kind !== 'unmanaged-process') throw new Error('Legacy native namespace corroboration is invalid') + return { ...scope, pidNamespace: selected.pidNamespace } +} + +export function progressCheckpoint(history = []) { + let checkpoint = 0, lastMeaningfulProgressAt = null, progressEvidence = [] + const seen = new Set() + for (const [index, receipt] of history.entries()) { + const fresh = (receipt.evidence ?? []).filter(row => row.kind === 'host-tests' && row.passed === true && typeof row.sha === 'string' && typeof row.sha256 === 'string' && !seen.has(row.sha)) + for (const row of fresh) seen.add(row.sha) + if (fresh.length) { checkpoint = index + 1; lastMeaningfulProgressAt = receipt.createdAt; progressEvidence = fresh.map(row => ({ handoffId: receipt.id, sha: row.sha, testLogSha256: row.sha256 })) } + } + return { checkpoint, lastMeaningfulProgressAt, progressEvidence } +} diff --git a/patches/ds-bot/files/src/host/project-chief-schedule.json b/patches/ds-bot/files/src/host/project-chief-schedule.json index cfef0a12f..bad4b99c4 100644 --- a/patches/ds-bot/files/src/host/project-chief-schedule.json +++ b/patches/ds-bot/files/src/host/project-chief-schedule.json @@ -18,6 +18,8 @@ "runTimeoutMs": 900000, "recoveryDelayMs": 5000, "maxRecoveries": 2, - "overdueMs": 3600000 + "overdueMs": 3600000, + "stopTimeoutMs": 10000, + "maxStopAttempts": 3 } } diff --git a/patches/ds-bot/files/src/host/project-delivery.js b/patches/ds-bot/files/src/host/project-delivery.js index 52a8ceb8e..0f78ff564 100644 --- a/patches/ds-bot/files/src/host/project-delivery.js +++ b/patches/ds-bot/files/src/host/project-delivery.js @@ -1,3 +1,5 @@ +import { chiefRecoveryError } from './project-chief-recovery.js' +import { chiefProtocolError } from './project-chief-protocol.js' // Delivery coordinates artifacts/worktrees around the existing Workflow service. import { execFile } from 'node:child_process' import { createHash, randomUUID } from 'node:crypto' @@ -516,7 +518,7 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { return { baseSha: attempt.baseSha ?? null, checkoutSha: attempt.candidateSha ?? attempt.seedSha ?? attempt.baseSha ?? null, availableInputs: await approvedRepositoryInputs(project), snapshot: await integrationSnapshot(attempt), preparationFailure: attempt.preparationFailures?.at(-1) ?? null, - materialization: workspaceFor(attempt, 'developer')?.materialization ?? null } + materialization: workspaceFor(attempt, 'developer')?.materialization ?? null, verificationRecovery: { attemptStatus: attempt.status, sealedCandidate: Boolean(attempt.candidateSha && attempt.evidence.implementation?.sha === attempt.candidateSha), testsPassedForCandidate: Boolean(attempt.evidence.tests?.passed && attempt.evidence.tests.sha === attempt.candidateSha), requires: ['current requirements and repository binding', 'clean sealed candidate', 'all old allocation exits confirmed', 'normal Task-role context admission'], reviewerRequires: 'matching passing host test command and immutable log' } } } catch { return { status: 'unavailable', reason: 'Registered repository or workspace facts are unavailable; preserve prior execution and inspect current bindings' } } } async function admitChiefInputs(taskId, inputs, actor, saved) { @@ -1049,6 +1051,39 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { attempt.acceptedViaChief = decision.id; if (task.result) task.result.summary = `Chief confirmed: ${decision.reason}`; task.completedBy = { kind: 'chief', authorizedBy: actor.memberId, decisionId: decision.id, runId: actor.chiefJobId }; await rt.save(); return accepted } catch (error) { attempt.chiefDecision = prior; throw error } } + async function validateRecovery({ task, target }) { + const attempt = ownEntry(attempts(), task.collaboration?.attemptId), capability = target.policyRole ?? target.role + if (!attempt || capability === 'developer') return + const refuseRoute = (message, actions) => Object.assign(chiefProtocolError('targetBotId', message, task.id), { recovery: { version: 'chief-recovery/v1', kind: 'unsupported-route', retryable: false, attemptId: attempt.id, attemptStatus: attempt.status, candidateSha: attempt.candidateSha ?? null, actions, conditions: [message] } }) + if (!['running', 'interrupted', 'failed'].includes(attempt.status) || attempt.cancelRequested || attempt.supersededBy || task.deliveryAttemptId !== attempt.id) throw refuseRoute('a current active or recoverable Attempt; preserve ended/superseded history', ['inspect current Attempt', 'choose an explicit rework or reopen action']) + if (task.collaboration.kind !== 'code' || !['tester', 'reviewer'].includes(capability)) { + if (attempt.status !== 'running') throw refuseRoute('a running non-code Attempt or explicit rework', ['choose an explicit rework']) + return + } + if (task.projectTask && attempt.requirement.projectTaskRevision !== task.projectTask.requirements.revision || task.title !== attempt.requirement.title || (task.criteria ?? '') !== attempt.requirement.criteria) throw refuseRoute('matching current Task requirements', ['rework against current requirements']) + settingsOf(projectOf(task.projectId, attempt)) + if (!attempt.candidateSha || attempt.evidence.implementation?.sha !== attempt.candidateSha) throw refuseRoute('a host-sealed candidate before verification routing', ['route to a permitted implementation role', 'admit approved repository inputs when appropriate']) + try { await verifyCandidate(attempt) } catch { throw refuseRoute('the unchanged clean sealed candidate and valid child inputs', ['inspect candidate/workspace evidence', 'choose explicit rework']) } + if (capability === 'reviewer') { + const tests = attempt.evidence.tests, settings = settingsOf(projectOf(task.projectId, attempt)) + if (!tests?.passed || tests.sha !== attempt.candidateSha || JSON.stringify(tests.command) !== JSON.stringify(settings.testCommand)) throw refuseRoute('matching passing host tests for Reviewer recovery', ['request tests', 'inspect environment readiness']) + try { if (await fileDigest(tests.log) !== tests.sha256) throw new Error('changed') } catch { throw refuseRoute('an unchanged matching host test log', ['request fresh tests']) } + } + } + async function resumeVerification({ task, target, decisionId, reason }) { + const attempt = ownEntry(attempts(), task.collaboration?.attemptId) + if (!attempt || attempt.status === 'running') return + try { await validateRecovery({ task, target }) } catch (error) { throw chiefRecoveryError('unsupported-route', error.message, { attemptId: attempt.id, conditions: error.recovery?.conditions ?? [], actions: error.recovery?.actions ?? [] }) } + const leases = Object.values(state.projectResourceLeases ?? {}).filter(row => row.attemptId === attempt.id), workers = Object.values(state.projectHarnessWorkers ?? {}).filter(row => row.attemptId === attempt.id), native = Object.values(state.projectNativeWorkers ?? {}).filter(row => row.attemptId === attempt.id) + if (leases.some(row => row.status !== 'released') || workers.some(row => !confirmedWorkerExit(state, row)) || native.some(row => !nativeExitConfirmed(state, row)) || Object.values(state.bots ?? {}).some(row => row.ephemeral && row.attemptId === attempt.id)) throw chiefRecoveryError('worker-exit-unknown', 'Verification recovery waits for confirmed exit of every previous allocation', { retryable: true, attemptId: attempt.id, conditions: ['all exact worker and lease exits confirmed'], actions: ['inspect registered execution evidence'] }) + const previousRunId = attempt.runId, runId = `pool-resume-${hash(attempt.id + ':' + decisionId)}` + const resumption = { kind: 'sealed-candidate-verification', decisionId, previousRunId, runId, previousStatus: attempt.status, candidateSha: attempt.candidateSha, targetBotId: target.id, reason, at: Date.now(), leaseIds: leases.map(row => row.id), workerIds: workers.map(row => row.id) } + ;(attempt.resumptions ??= []).push(resumption) + attempt.runId = runId; attempt.status = 'running'; attempt.updatedAt = Date.now() + rt.bindProject({ kind: 'run', id: previousRunId, projectId: task.projectId, taskId: task.id, attemptId: attempt.id, status: 'failed' }) + rt.bindProject({ kind: 'run', id: runId, projectId: task.projectId, taskId: task.id, attemptId: attempt.id, status: 'queued' }) + await rt.save() + } async function reworkPool({ task, claim, reason, force = false, decisionId }) { const previous = ownEntry(attempts(), claim.attemptId) if (!previous) throw reject('Chief rework has no original Attempt') @@ -1145,7 +1180,7 @@ export function createProjectDelivery(rt, { publisher, releaseProvider } = {}) { } } const ownsWorkspace = (projectId, path) => Object.values(state.projectWorkspaces ?? {}).find(workspace => workspace.projectId === projectId && workspace.path === path && DELIVERY_ROLES.includes(workspace.policyRole ?? workspace.role) && ['worktree', 'resource-clone', 'decision-directory'].includes(workspace.kind) && workspace.status === 'allocated' && ongoing.has(ownEntry(attempts(), workspace.attemptId)?.status)) - return { releases, recoveryFacts, admitChiefInputs, reconcileExecutions: () => reconcileExecutionAuthority(rt), exitConfirmed: workerId => executionExitConfirmed(rt, workerId), nativeSessions, endpoint, context, admitChildInputs: (taskId, claim) => serialAttempt(claim.attemptId, () => childInputs.admit(taskId, claim)), preparePoolRole, abortPoolAdmission, completePoolRole, chiefAccept, reworkPool, recordExecution, repositoryContext, repositoryRolePolicy, bindSession, releaseSession, sessionFor, recover, runReported, ownsWorkspace, gate: input => serialAttempt(current(input.taskId)?.id ?? input.taskId, () => gate(input)) } + return { releases, recoveryFacts, admitChiefInputs, reconcileExecutions: () => reconcileExecutionAuthority(rt), exitConfirmed: workerId => executionExitConfirmed(rt, workerId), nativeSessions, endpoint, context, admitChildInputs: (taskId, claim) => serialAttempt(claim.attemptId, () => childInputs.admit(taskId, claim)), preparePoolRole, validateRecovery, resumeVerification, abortPoolAdmission, completePoolRole, chiefAccept, reworkPool, recordExecution, repositoryContext, repositoryRolePolicy, bindSession, releaseSession, sessionFor, recover, runReported, ownsWorkspace, gate: input => serialAttempt(current(input.taskId)?.id ?? input.taskId, () => gate(input)) } } export function install(rt) { diff --git a/patches/ds-bot/files/src/host/project-development-service.js b/patches/ds-bot/files/src/host/project-development-service.js index 90b22ce3b..578aa0e77 100644 --- a/patches/ds-bot/files/src/host/project-development-service.js +++ b/patches/ds-bot/files/src/host/project-development-service.js @@ -164,7 +164,7 @@ export function createDevelopmentService(rt) { attemptsPage: { total: rows.length, limit, before: end > limit ? page[0].id : null } } } function context(body) { - if (!['roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures'].includes(body.section)) throw failure('Unknown Attempt context section') + if (!['roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures', 'resumptions'].includes(body.section)) throw failure('Unknown Attempt context section') const attempt = ownEntry(rt.state.projectAttempts ?? {}, body.attemptId) const text = JSON.stringify(attempt[body.section] ?? null), offset = body.offset ?? 0, limit = body.limit ?? 8000 if (!Number.isSafeInteger(offset) || offset < 0 || offset > text.length || !Number.isSafeInteger(limit) || limit < 1 || limit > 16000) throw failure('Invalid context page') diff --git a/patches/ds-bot/files/src/host/project-pools.js b/patches/ds-bot/files/src/host/project-pools.js index 9184a974a..2e0aebb48 100644 --- a/patches/ds-bot/files/src/host/project-pools.js +++ b/patches/ds-bot/files/src/host/project-pools.js @@ -536,6 +536,17 @@ export function createProjectPools(rt) { receipt.acceptance = { claimId: claim.id, botId: claim.botId, generation: claim.generation, acceptedAt: receipt.acceptedAt, ...(claim.chiefRunId ? { chiefRunId: claim.chiefRunId } : {}) } await rt.save(); return clone(claim) } + async function validateChiefRoute(body, actor) { + const task = taskOf(body.taskId), c = task.collaboration + rt.projectChiefControl.requireJobActor(actor, task.projectId) + rt.requireProjectAccess(actor, task.projectId, 'run-control') + if (!c || c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') + const target = targetOf(task.projectId, body.targetBotId) + await authority(task) + await adapter?.validateRecovery?.({ task, target }) + if (c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') + return target + } async function chiefRoute(body, actor) { const task = taskOf(body.taskId), c = task.collaboration rt.projectChiefControl.requireJobActor(actor, task.projectId) @@ -544,6 +555,7 @@ export function createProjectPools(rt) { const target = targetOf(task.projectId, body.targetBotId) if (target.role === 'chief') throw refuse('Chief must decide a concrete next role or request human clarification') reasonOf(body.reason) + await validateChiefRoute(body, actor) await authority(task) await checkAdmission(task, { from: `${task.projectId}:chief`, to: target.id, attemptId: c.attemptId, requestId: body.decisionId, purpose: 'chief-intervention' }) if (c.generation !== body.expectedGeneration) throw refuse('Chief routing snapshot is stale') @@ -564,7 +576,10 @@ export function createProjectPools(rt) { c.initiatedBy = actor.memberId; c.sourceSessionId = null const previous = rt.state.projectAttempts?.[c.attemptId] if ((target.policyRole ?? target.role) === 'developer') c.attemptId = await adapter.rework({ task, claim: oldClaim ?? { attemptId: c.attemptId, id: body.decisionId, botId: `${task.projectId}:chief` }, reason: body.reason, force: body.rework !== false, decisionId: body.decisionId }) - else if (previous && previous.status !== 'running') throw refuse('Interrupted delivery must return to Developer in a new Attempt') + else if (previous && previous.status !== 'running') { + if (!adapter.resumeVerification) throw refuse('Interrupted delivery requires an evidence-supported recovery adapter') + await adapter.resumeVerification({ task, target, decisionId: body.decisionId, reason: body.reason }) + } return await offer(task, { from: `${task.projectId}:chief`, to: target.id, reason: body.reason, requestId: body.decisionId, attemptId: c.attemptId, purpose: 'chief-intervention' }) } catch (error) { c.status = 'blocked'; c.block = { reason: String(error.message), decisionId: body.decisionId, at: Date.now() }; await rt.save(); throw error @@ -625,7 +640,7 @@ export function createProjectPools(rt) { return { connect(value) { adapter = value; return () => { if (adapter === value) adapter = null } }, get, list, accept: (taskId, botId, handoffId, requestId) => serial(() => claim(taskId, botId, handoffId, requestId)), admit: (attempt, actor, body) => serial(() => admit(attempt, actor, body)), available, owns, correct, resumeChildren: (id, waitId) => serial(() => resumeChildren(id, waitId)), - claim: (id, botId) => serial(() => claim(id, botId)), remember: (id, token, body) => serial(() => remember(id, token, body)), decide: (id, token, decision) => serial(() => decide(id, token, decision)), result: row => serial(() => result(row)), pump, recover, revokePrincipal: (projectId, memberId) => serial(() => revokePrincipal(projectId, memberId)), revokeExecution: (id, claimId, reason) => serial(() => revokeExecution(id, claimId, reason)), validateClosure, retryChief: (id, actor) => serial(() => retryChief(id, actor)), chiefClaim: (id, actor) => serial(() => chiefClaim(id, actor)), chiefRoute: (body, actor) => serial(() => chiefRoute(body, actor)) } + claim: (id, botId) => serial(() => claim(id, botId)), remember: (id, token, body) => serial(() => remember(id, token, body)), decide: (id, token, decision) => serial(() => decide(id, token, decision)), result: row => serial(() => result(row)), pump, recover, revokePrincipal: (projectId, memberId) => serial(() => revokePrincipal(projectId, memberId)), revokeExecution: (id, claimId, reason) => serial(() => revokeExecution(id, claimId, reason)), validateClosure, retryChief: (id, actor) => serial(() => retryChief(id, actor)), chiefClaim: (id, actor) => serial(() => chiefClaim(id, actor)), validateChiefRoute: (body, actor) => serial(() => validateChiefRoute(body, actor)), chiefRoute: (body, actor) => serial(() => chiefRoute(body, actor)) } } export function install(rt) { rt.projectPools = createProjectPools(rt) diff --git a/patches/ds-bot/files/src/host/project-process-proof.js b/patches/ds-bot/files/src/host/project-process-proof.js index d641c943b..a0f24a690 100644 --- a/patches/ds-bot/files/src/host/project-process-proof.js +++ b/patches/ds-bot/files/src/host/project-process-proof.js @@ -1,15 +1,17 @@ // A manager lifecycle boundary can prove exit after restart; a missing PID alone cannot. -import { readFile } from 'node:fs/promises' +import { readFile, readlink } from 'node:fs/promises' import { execFile } from 'node:child_process' import { promisify } from 'node:util' const exec = promisify(execFile) const read = path => readFile(path, 'utf8') +const namespace = () => readlink('/proc/self/ns/pid') const manager = async unit => (await exec('/usr/bin/systemctl', ['show', unit, '--property=InvocationID,KillMode,ControlGroup'], { timeout: 2000, maxBuffer: 8192, env: { PATH: '/usr/bin:/bin', LANG: 'C' } })).stdout -export async function captureProcessScope({ readText = read, inspectManager = manager, environment = process.env, pid = process.pid } = {}) { +export async function captureProcessScope({ readText = read, inspectManager = manager, environment = process.env, pid = process.pid, readNamespace = namespace } = {}) { const base = { kind: 'unmanaged-process', hostPid: pid } if (process.platform !== 'linux') return base try { const [boot, cgroups, stat] = await Promise.all([readText('/proc/sys/kernel/random/boot_id'), readText('/proc/self/cgroup'), readText(`/proc/${pid}/stat`)]) + base.pidNamespace = await readNamespace() base.bootId = boot.trim(); base.startTicks = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] const group = cgroups.split('\n').find(line => line.startsWith('0::'))?.slice(3), unit = group?.split('/').find(part => /^[A-Za-z0-9_.@-]+\.service$/.test(part)) if (!unit || !/^[a-f0-9]{32}$/.test(environment.INVOCATION_ID ?? '')) return base @@ -28,3 +30,17 @@ export function supervisorProofMatches(scope, proof) { return Boolean(scope?.kind === 'systemd-control-group' && scope.killMode === 'control-group' && /^[a-f0-9]{32}$/.test(scope.invocationId ?? '') && proof?.kind === 'systemd-prior-invocation-exited' && proof.previousInvocationId === scope.invocationId) } + +// Proves only the old host process exited. NEVER use this as SDK/container, +// native role tool subprocess or resource preparation exit confirmation. +export async function priorHostProcessExited(scope, { readText = read, readNamespace = namespace } = {}) { + if (!['unmanaged-process', 'systemd-control-group'].includes(scope?.kind) || !Number.isSafeInteger(scope.hostPid) || scope.hostPid < 1 || !/^[a-f0-9-]{36}$/.test(scope.bootId ?? '') || !/^\d+$/.test(scope.startTicks ?? '') || !/^pid:\[\d+\]$/.test(scope.pidNamespace ?? '')) return false + try { if ((await readText('/proc/sys/kernel/random/boot_id')).trim() !== scope.bootId || await readNamespace() !== scope.pidNamespace) return false } catch { return false } + try { + const stat = await readText(`/proc/${scope.hostPid}/stat`), end = stat.lastIndexOf(')') + if (end < 0 || stat.slice(0, stat.indexOf(' ')) !== String(scope.hostPid)) return false + const fields = stat.slice(end + 2).split(' ') + if (!/^\d+$/.test(fields[19] ?? '')) return false + return fields[19] !== scope.startTicks || fields[0] === 'Z' || fields[0] === 'X' + } catch (error) { return error.code === 'ENOENT' } +} diff --git a/patches/ds-bot/files/src/host/project-service-views.js b/patches/ds-bot/files/src/host/project-service-views.js index 2cb49c3e3..f744b4431 100644 --- a/patches/ds-bot/files/src/host/project-service-views.js +++ b/patches/ds-bot/files/src/host/project-service-views.js @@ -59,6 +59,7 @@ export function attemptMetadataView(attempt) { if (attempt.release) result.release = { ...pick(attempt.release, ['id', 'status', 'observedAt']), ...pick(attempt.release.spec, ['repository', 'tag', 'sha', 'mode']), sourceRef: { query: 'release.status', attemptId: attempt.id } } if (attempt.pr) result.pr = pick(attempt.pr, ['number', 'sha', 'url', 'repository']) if (attempt.publicationBinding) result.publicationBinding = pick(attempt.publicationBinding, ['number', 'repository', 'branch', 'previousSha', 'upstreamSha', 'upstreamRef']) + if (attempt.resumptions) result.resumptions = { total: attempt.resumptions.length, recent: attempt.resumptions.slice(-5).map(row => pick(row, ['kind', 'decisionId', 'previousRunId', 'runId', 'previousStatus', 'candidateSha', 'targetBotId', 'at'])), source: { query: 'attempt.context', attemptId: attempt.id, section: 'resumptions' } } if (attempt.integrationInputs) result.integrationInputs = attempt.integrationInputs.slice(0, 20).map(row => pick(row, ['version', 'repository', 'ref', 'sha', 'purpose'])) if (attempt.evidence?.tests?.preparation) result.environment = pick(attempt.evidence.tests.preparation, ['id', 'status', 'executor', 'workspaceId', 'resourceId', 'identitySha256', 'sha256']) if (attempt.workspaces) result.workspaces = Object.fromEntries(Object.entries(attempt.workspaces).slice(0, 32).map(([role, row]) => [role, pick(row, ['id', 'status', 'role', 'resourceId', 'materialization'])])) diff --git a/patches/ds-bot/files/test/project-agents.test.js b/patches/ds-bot/files/test/project-agents.test.js index 557756505..47e2ebe49 100644 --- a/patches/ds-bot/files/test/project-agents.test.js +++ b/patches/ds-bot/files/test/project-agents.test.js @@ -568,7 +568,8 @@ test('dormant native end cannot consume an SDK Chief job before worker registrat assert.equal(rt.projectAgents.isDelegationActive(input.agent.id, input.delegation.id), true) return 'delayed-sdk-worker' } - await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'chief-window', prompt: 'Inspect', projectFacts: {}, onResult: async () => { callbacks++ } }) + rt.state.projectExecutions['chief-window'] = { id: 'chief-window', projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, invocation: 1, runId: 'chief-window', prompt: 'Inspect', projectFacts: {}, onResult: async () => { callbacks++ } }) await request.onResult({ ok: true, text: '{}' }) assert.equal(callbacks, 1) assert.equal(rt.state.projectAgents[original.public.id].lastRequest.phase, 'completed') @@ -807,7 +808,8 @@ test('resident native Chief receives fresh job facts in each turn instead of rel rt.deliver = async args => { sends.push({ ...args, system: rt.botOf(chief.botId).systemPrompt }); return originalDeliver(args) } for (const index of [1, 2]) { let completed = false - await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: `chief-${index}`, prompt: 'Judge only this current job', projectFacts: { tasks: [{ taskId: `task-${index}`, title: `FRESH_JOB_${index}` }] }, onResult: async () => { completed = true } }) + rt.state.projectExecutions[`chief-${index}`] = { id: `chief-${index}`, projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, invocation: 1, runId: `chief-${index}`, prompt: 'Judge only this current job', projectFacts: { tasks: [{ taskId: `task-${index}`, title: `FRESH_JOB_${index}` }] }, onResult: async () => { completed = true } }) const sent = sends.at(-1) assert.match(sent.text, new RegExp(`FRESH_JOB_${index}`)) assert.match(sent.text, /Judge only this current job/) @@ -934,7 +936,7 @@ test('native Chief deadline fencing waits for actual idleness and preserves the const { rt } = await setup(t) const { public: chief } = await get(rt, alice) rt.projectChiefControl.validateActor = async () => {} - rt.state.projectExecutions.deadline = { projectId: 'p1' } + rt.state.projectExecutions.deadline = { id: 'deadline', projectId: 'p1', invocation: 1, status: 'running' } await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'deadline', invocation: 1, prompt: 'Bounded inspection', projectFacts: {}, onResult: async () => {} }) let idle, cancelled = false rt.resolveAgent = async () => ({ inbox: {}, cancel() { cancelled = true }, whenIdle: () => new Promise(resolve => { idle = resolve }) }) @@ -947,3 +949,26 @@ test('native Chief deadline fencing waits for actual idleness and preserves the assert.equal(rt.projectAgents.chiefBusy('p1'), false) assert.equal(rt.state.projectAgents[chief.id].sessionId, rt.botOf(chief.botId).sessionId) }) + +test('native exit deadline is bounded and cannot silently release its live execution', async t => { + const { rt } = await setup(t); await get(rt, alice) + rt.projectChiefControl.validateActor = async () => {} + rt.state.projectChiefPolicies = { p1: { inspection: { lifecycle: { stopTimeoutMs: 20 } } } } + rt.state.projectExecutions.bounded = { id: 'bounded', projectId: 'p1', invocation: 1, status: 'running' } + await rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: 'bounded', invocation: 1, prompt: 'Bounded exit', projectFacts: {}, onResult: async () => {} }) + rt.resolveAgent = async () => ({ inbox: {}, cancel() {}, whenIdle: () => new Promise(() => {}) }) + const proof = await rt.projectAgents.stopChiefJob('p1', 'bounded') + assert.equal(proof.confirmed, false); assert.match(proof.reason, /timed out/) + assert.equal(rt.projectAgents.chiefBusy('p1'), true, 'Revoked but live execution retains its Session boundary') +}) + +test('failed Chief execution-boundary commit cannot strand a busy delegation before dispatch', async t => { + const { rt } = await setup(t); await get(rt, alice) + rt.projectChiefControl.validateActor = async () => {} + const run = rt.state.projectExecutions.uncommitted = { id: 'uncommitted', projectId: 'p1', invocation: 1, status: 'running' } + const save = rt.save; let failed = false + rt.save = async () => { if (!failed && rt.projectAgents.chiefBusy('p1')) { failed = true; throw new Error('boundary snapshot failed') } return save() } + await assert.rejects(rt.projectAgents.startChiefJob({ projectId: 'p1', actor: alice, runId: run.id, invocation: 1, prompt: 'Do not dispatch before commit', projectFacts: {}, onResult: async () => {} }), /boundary snapshot failed/) + assert.equal(failed, true); assert.equal(rt.projectAgents.chiefBusy('p1'), false) + assert.equal(run.exitProof.kind, 'chief-never-dispatched') +}) diff --git a/patches/ds-bot/files/test/project-chief-control.test.js b/patches/ds-bot/files/test/project-chief-control.test.js index 1e7e25aec..8f357b605 100644 --- a/patches/ds-bot/files/test/project-chief-control.test.js +++ b/patches/ds-bot/files/test/project-chief-control.test.js @@ -464,3 +464,44 @@ test('Chief receipt protocol correction precedes all role proposals and Task eff assert.equal(run.correction.status, 'resolved'); assert.equal(proposals, 0) await f.rt.projectEffects.close() }) + +test('unsupported verification route is corrected by Chief before changing the Task generation', async () => { + const { chiefProtocolError } = await import('../src/host/project-chief-protocol.js') + const f = await controlFixture(), run = f.rt.state.projectExecutions[f.request.runId], generation = f.task.collaboration.generation + f.rt.projectPools.connect({ validate: () => ({ ok: true }), validateRecovery: async ({ target }) => { + if (target.role !== 'developer') throw Object.assign(chiefProtocolError('targetBotId', 'a sealed candidate for verification', 't'), { recovery: { kind: 'unsupported-route', actions: ['inspect candidate', 'choose explicit rework'] } }) + }, rework: async () => 'a' }) + await f.request.onResult(f.result({ action: 'route', targetBotId: 'p:reviewer', reason: 'Review candidate' })) + assert.equal(run.status, 'correction-pending'); assert.equal(f.task.collaboration.generation, generation) + assert.equal(run.correction.feedback[0].recovery.kind, 'unsupported-route'); assert.equal(f.task.chiefQuestion, undefined) + await f.control.pump(); await f.request.onResult(f.result({ action: 'return', targetBotId: 'p:developer', reason: 'Produce the missing candidate' })) + assert.equal(run.status, 'done', run.failure); assert.equal(f.task.collaboration.botId, 'p:developer') +}) + +test('unmanaged native exit adapter unblocks an interrupted inspection and old invocation callbacks remain fenced', async () => { + const f = await controlFixture(), old = f.request, run = f.rt.state.projectExecutions[old.runId] + run.purpose = 'inspection'; delete run.taskId + let reconciled = 0 + f.rt.projectAgents.reconcileChiefExit = async current => { + if (current.invocation === 1) { ++reconciled; current.exitProof = { confirmed: true, kind: 'native-chief-prior-host-exited', runId: current.id, invocation: 1 }; return true } + return false + } + const restored = createProjectChiefControl(f.rt); f.rt.projectChiefControl = restored; restored.recover() + run.recovery = { status: 'waiting-exit', attempts: 0, notBefore: Date.now() - 1 } + await restored.pump(); await new Promise(resolve => setTimeout(resolve, 25)) + assert.equal(run.status, 'running'); assert.equal(run.invocation, 2); assert.equal(reconciled, 1) + await old.onResult(f.result({ action: 'close', reason: 'Late old response' })) + assert.equal(f.accepted(), 0, 'old response cannot act through a reused Run record') +}) + +test('an uncommitted stop intent cannot send a stop or consume the mechanical stop budget', async () => { + const f = await controlFixture({ modern: true }), run = f.rt.state.projectExecutions[f.request.runId] + run.purpose = 'inspection'; delete run.taskId + let stopped = 0; f.rt.projectAgents.stopChiefJob = async () => { ++stopped; return { confirmed: false } } + const restored = createProjectChiefControl(f.rt); f.rt.projectChiefControl = restored; restored.recover() + run.recovery = { status: 'waiting-exit', attempts: 0, notBefore: Date.now() - 1 } + f.rt.save = async () => {} + await assert.rejects(restored.pump(), /not confirmed/) + assert.equal(stopped, 0); assert.equal(run.recovery.stopAttempts, 0) + await f.rt.projectEffects.close() +}) diff --git a/patches/ds-bot/files/test/project-chief-lifecycle.test.js b/patches/ds-bot/files/test/project-chief-lifecycle.test.js index 3b20c54d0..6a3a65d5c 100644 --- a/patches/ds-bot/files/test/project-chief-lifecycle.test.js +++ b/patches/ds-bot/files/test/project-chief-lifecycle.test.js @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { inspectionLifecycle, inspectionHealth } from '../src/host/project-chief-lifecycle.js' test('inspection configuration bounds deadlines/recovery and health distinguishes enablement from success', () => { - for (const lifecycle of [{ maxRecoveries: 5 }, { runTimeoutMs: 0 }, { recoveryDelayMs: -1 }, { overdueMs: Infinity }, { unknown: 1 }]) assert.throws(() => inspectionLifecycle({ lifecycle }), /Invalid/) + for (const lifecycle of [{ maxRecoveries: 5 }, { stopTimeoutMs: 0 }, { maxStopAttempts: 11 }, { runTimeoutMs: 0 }, { recoveryDelayMs: -1 }, { overdueMs: Infinity }, { unknown: 1 }]) assert.throws(() => inspectionLifecycle({ lifecycle }), /Invalid/) const state = { projectChiefPolicies: { p: { enabled: true, scheduleId: 's', updatedAt: 100, inspection: { lifecycle: { overdueMs: 1000 } } } }, projectScheduledTasks: { s: { enabled: true, deliveries: {} } }, projectExecutions: {}, projectChiefDecisions: {} } assert.equal(inspectionHealth(state, 'p', 200).status, 'pending') assert.equal(inspectionHealth(state, 'p', 2000).status, 'overdue') @@ -13,3 +13,10 @@ test('inspection configuration bounds deadlines/recovery and health distinguishe assert.equal(inspectionHealth(state, 'p', 600).status, 'degraded') state.projectScheduledTasks.s.enabled = false; assert.equal(inspectionHealth(state, 'p', 20000).status, 'disabled') }) + +test('health retains unknown-exit cancellation and separates actual checkpoints from inspection or message churn', () => { + const state = { projectChiefPolicies: { p: { enabled: true, scheduleId: 's', updatedAt: 100 } }, projectScheduledTasks: { s: { enabled: true } }, projectExecutions: { old: { id: 'old', projectId: 'p', kind: 'chief-control', purpose: 'inspection', createdAt: 100, status: 'cancelled', recovery: { exitConfirmed: false, stopAttempts: 3 } } }, tasks: { t: { projectId: 'p', collaboration: { history: [{ createdAt: 120, evidence: [{ kind: 'host-tests', passed: true, sha: 'candidate', sha256: 'log' }] }, { createdAt: 150, evidence: [{ kind: 'host-tests', passed: true, sha: 'candidate', sha256: 'second-log' }] }] } } } } + const health = inspectionHealth(state, 'p', 200) + assert.equal(health.status, 'degraded'); assert.equal(health.active[0].nextRecoveryAt, null) + assert.equal(health.lastMeaningfulProgressAt, 120); assert.equal(health.lastSuccessAt, null) +}) diff --git a/patches/ds-bot/files/test/project-chief-recovery.test.js b/patches/ds-bot/files/test/project-chief-recovery.test.js new file mode 100644 index 000000000..51f733c8d --- /dev/null +++ b/patches/ds-bot/files/test/project-chief-recovery.test.js @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { nativeChiefBoundary, classifyChiefRecovery } from '../src/host/project-chief-recovery.js' +const scope = () => { + const run = { id: 'run', projectId: 'p', invocation: 2, workerId: 'native-chief-job-run', executionBoundary: { version: 'chief-execution-boundary/v1', kind: 'native-in-process', runId: 'run', invocation: 2, botId: 'chief', sessionId: 'original', carrierGeneration: 1, exchangeId: 'exchange' } } + return { run, state: { projectResourceLeases: {}, projectHarnessWorkers: {}, projectAgents: { 'public-p': { botId: 'chief', sessionId: 'original', carrierGeneration: 1, lastRequest: { chiefRunId: 'run', chiefInvocation: 2, exchangeId: 'exchange' } } } } } +} +test('native host exit boundary rejects SDK admissions, unknown leases and stale invocation identities', () => { + const f = scope(); assert.equal(nativeChiefBoundary(f.state, f.run).boundary.kind, 'native-in-process') + f.state.projectHarnessWorkers.w = { chiefRunId: 'run', chiefInvocation: 2, status: 'unreachable' } + assert.equal(nativeChiefBoundary(f.state, f.run), null) + delete f.state.projectHarnessWorkers.w + f.state.projectResourceLeases.l = { projectId: 'p', chiefRunId: 'run', chiefInvocation: 2, status: 'unreachable', worker: { kind: 'sdk', handle: 'w' } } + assert.equal(nativeChiefBoundary(f.state, f.run), null) + f.state.projectResourceLeases = {}; f.run.executionBoundary.invocation = 1 + assert.equal(nativeChiefBoundary(f.state, f.run), null) + delete f.run.executionBoundary + assert(nativeChiefBoundary(f.state, f.run), 'legacy native handle still requires the exact persisted request binding') + f.state.projectAgents['public-p'].lastRequest.chiefInvocation = 1 + assert.equal(nativeChiefBoundary(f.state, f.run), null) +}) +test('Chief recovery feedback distinguishes stale state, missing authority, unknown exit and unsupported routes', () => { + for (const [error, kind] of [[new Error('Chief routing snapshot is stale'), 'stale-snapshot'], [new Error('worker exit unconfirmed'), 'worker-exit-unknown'], [Object.assign(new Error('denied'), { code: 'bot/forbidden' }), 'authority-unavailable'], [new Error('Interrupted delivery needs recovery'), 'unsupported-route']]) assert.equal(classifyChiefRecovery(error).kind, kind) + assert.equal(classifyChiefRecovery(new Error('worker exit unconfirmed')).retryable, true) + assert.equal(classifyChiefRecovery(Object.assign(new Error('denied'), { code: 'bot/forbidden' })).retryable, false) +}) + +test('legacy unmanaged namespace assurance is operator-owned and bound to one exact old native invocation', async () => { + const { corroboratedNativeScope, scopeDigest } = await import('../src/host/project-chief-recovery.js') + const old = { kind: 'unmanaged-process', hostPid: 12, startTicks: '10', bootId: 'same' }, run = { id: 'run', invocation: 2 } + const record = { runId: 'run', invocation: 2, scopeSha256: scopeDigest(old), pidNamespace: 'pid:[123]' } + assert.equal(corroboratedNativeScope(old, run), old) + assert.equal(corroboratedNativeScope(old, { ...run, invocation: 3 }, [record]), old) + assert.equal(corroboratedNativeScope({ ...old, startTicks: '11' }, run, [record]).pidNamespace, undefined) + assert.equal(corroboratedNativeScope(old, run, [record]).pidNamespace, 'pid:[123]') + assert.equal(old.pidNamespace, undefined) +}) diff --git a/patches/ds-bot/files/test/project-delivery.test.js b/patches/ds-bot/files/test/project-delivery.test.js index aae6a0d45..d74ab3c8e 100644 --- a/patches/ds-bot/files/test/project-delivery.test.js +++ b/patches/ds-bot/files/test/project-delivery.test.js @@ -1915,3 +1915,46 @@ test('a newly created unsealed successor after failure pins current host HEAD ev await prepareRole(f.rt, f.task, { id: 'failed-route-prepare', attemptId: id, generation: 2 }, f.definition) assert.equal(git(next.workspaces.developer.path, ['rev-parse', 'HEAD']), f.latest) }) + +for (const role of ['tester', 'reviewer']) test('interrupted sealed candidate resumes ' + role + ' in the same Attempt only with current evidence', async t => { + const f = await setup(t); await f.start('resume-' + role); await implement(f) + if (role === 'reviewer') await f.gate('tester') + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id], sha = attempt.candidateSha, oldRun = attempt.runId + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 }; attempt.status = 'interrupted' + const target = { id: 'p1:' + role, role } + await f.rt.projectDelivery.validateRecovery({ task, target }) + await f.rt.projectDelivery.resumeVerification({ task, target, decisionId: 'recovery-decision', reason: 'Continue verification of the original sealed candidate' }) + assert.equal(attempt.status, 'running'); assert.equal(attempt.candidateSha, sha); assert.equal(task.deliveryAttemptId, attempt.id) + assert.notEqual(attempt.runId, oldRun); assert.equal(attempt.resumptions.length, 1) + await f.rt.projectDelivery.resumeVerification({ task, target, decisionId: 'recovery-decision', reason: 'Duplicate delivery' }) + assert.equal(attempt.resumptions.length, 1); assert.equal(Object.keys(f.rt.state.projectAttempts).length, 1) + assert.equal(git(attempt.workspaces.developer.path, ['rev-parse', 'HEAD']), sha) +}) + +test('verification recovery preflight rejects unsealed/stale candidates and Reviewer without tests before routing effects', async t => { + const f = await setup(t); await f.start('recovery-gates') + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id], target = { id: 'p1:reviewer', role: 'reviewer' } + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 } + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), error => error.code === 'bot/chief-protocol' && error.recovery.kind === 'unsupported-route') + assert.equal(task.collaboration.generation, 3) + await implement(f); attempt.status = 'interrupted' + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), /passing host tests/) + await f.rt.projectDelivery.validateRecovery({ task, target: { id: 'p1:tester', role: 'tester' } }) + attempt.status = 'running'; await f.gate('tester'); attempt.status = 'interrupted' + await f.rt.projectDelivery.validateRecovery({ task, target }) + writeFileSync(attempt.evidence.tests.log, 'changed test evidence') + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target }), /unchanged matching host test log/) + writeFileSync(join(attempt.workspaces.developer.path, 'code.mjs'), '// unverified mutation') + await assert.rejects(f.rt.projectDelivery.validateRecovery({ task, target: { id: 'p1:tester', role: 'tester' } }), /unchanged clean sealed candidate/) + assert.equal(attempt.status, 'interrupted'); assert.equal(task.collaboration.generation, 3) +}) + +test('unknown SDK or native allocation exit cannot be bypassed by choosing another verification role', async t => { + const f = await setup(t); await f.start('unknown-verification-exit'); await implement(f) + const attempt = f.attempt(), task = f.rt.state.tasks[f.task.id] + task.collaboration = { kind: 'code', attemptId: attempt.id, generation: 3 }; attempt.status = 'interrupted' + f.rt.state.projectHarnessWorkers ??= {} + f.rt.state.projectHarnessWorkers.unknown = { id: 'unknown', attemptId: attempt.id, status: 'unreachable' } + await assert.rejects(f.rt.projectDelivery.resumeVerification({ task, target: { id: 'p1:tester', role: 'tester' }, decisionId: 'unknown', reason: 'Cannot bypass unknown exit' }), error => error.code === 'bot/chief-recovery' && error.recovery.kind === 'worker-exit-unknown') + assert.equal(attempt.status, 'interrupted'); assert.equal(attempt.resumptions, undefined) +}) diff --git a/patches/ds-bot/files/test/project-process-proof.test.js b/patches/ds-bot/files/test/project-process-proof.test.js index 0c764c703..45d6e266e 100644 --- a/patches/ds-bot/files/test/project-process-proof.test.js +++ b/patches/ds-bot/files/test/project-process-proof.test.js @@ -16,3 +16,17 @@ test('PID disappearance, changed host boot, remote resources or changed kill pol assert.equal(await priorScopeExited(original, { ...fixture(changed), readText: async path => path.endsWith('boot_id') ? 'different-boot' : fixture(changed).readText(path) }), false) assert.equal(await priorScopeExited({ kind: 'remote-resource' }, fixture(changed)), false) }) + +test('birth-checked unmanaged host exit is distinct from independent worker/process-group exit', async () => { + const { priorHostProcessExited } = await import('../src/host/project-process-proof.js') + const scope = { kind: 'unmanaged-process', hostPid: 123, bootId: 'a'.repeat(8) + '-aaaa-aaaa-aaaa-' + 'a'.repeat(12), startTicks: '900', pidNamespace: 'pid:[123]' } + const readText = async path => path.endsWith('boot_id') ? scope.bootId : `${scope.hostPid} (node) ${['S', ...Array(18).fill('0'), '900'].join(' ')}` + const readNamespace = async () => scope.pidNamespace + assert.equal(await priorHostProcessExited(scope, { readText, readNamespace }), false) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async path => path.endsWith('boot_id') ? scope.bootId : readText(path).then(text => text.replace('900', '901')) }), true) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async path => { if (path.endsWith('boot_id')) return scope.bootId; throw Object.assign(new Error('gone'), { code: 'ENOENT' }) } }), true) + for (const code of ['EACCES', 'EPERM', 'EIO']) assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async () => { throw Object.assign(new Error('unknown'), { code }) } }), false) + assert.equal(await priorHostProcessExited(scope, { readNamespace, readText: async () => 'other-boot' }), false) + assert.equal(await priorHostProcessExited({ ...scope, startTicks: null }, { readText, readNamespace }), false) + assert.equal(await priorScopeExited(scope), false, 'Host death is never a generic resource exit proof') +}) diff --git a/scripts/api-contracts.mjs b/scripts/api-contracts.mjs index 9570382f7..554824d7e 100644 --- a/scripts/api-contracts.mjs +++ b/scripts/api-contracts.mjs @@ -35,8 +35,8 @@ export const schemas = { RepositoryAssociation: record({ id, projectId: id, repository, defaultBranch: str(120), createdAt: int() }, ['repository', 'defaultBranch']), Project: record({ id, name: str(120), goal: str(4000), scope: str(4000), description: str(2000), repository: nullable(repository), repositories: array(ref('RepositoryAssociation')), defaultBranch: str(120), workDir: nullable(str()), status: choice('active', 'archived', 'closed', 'deleted'), ownerUserId: nullable(id), ownerUserIds: array(id), role: nullable(choice('owner', 'maintainer', 'member')), permissions: map(bool), roles: record({ maintainers: array(id), members: array(id) }), chief: json, closure: json, createdAt: int(), updatedAt: int() }, ['id', 'name', 'status']), Requirements: record({ revision: int(1), title: str(2000), criteria: str(2000), impact: str(2000), scope: str(2000), repository: nullable(repository) }, ['revision', 'title', 'criteria', 'impact']), - InspectionLifecycle: obj({ queueTimeoutMs: int(1, 86400000), runTimeoutMs: int(1, 86400000), recoveryDelayMs: int(1, 86400000), maxRecoveries: int(0, 4), overdueMs: int(1, 86400000) }), - InspectionHealth: record({ enabled: bool, status: choice('disabled', 'pending', 'healthy', 'degraded', 'overdue'), lastOccurrenceAt: nullable(int()), lastStartedAt: nullable(int()), lastCompletedAt: nullable(int()), lastSuccessAt: nullable(int()), lastAppliedDecisionAt: nullable(int()), overdueMs: int(1), active: array(record({ runId: id, occurrenceId: nullable(id), status: str(), ageMs: int(), failure: nullable(str()), recovery: nullable(json), deadlineAt: int() }, ['runId', 'status', 'ageMs', 'deadlineAt'])) }, ['enabled', 'status', 'lastSuccessAt', 'lastAppliedDecisionAt', 'active']), + InspectionLifecycle: obj({ queueTimeoutMs: int(1, 86400000), runTimeoutMs: int(1, 86400000), recoveryDelayMs: int(1, 86400000), maxRecoveries: int(0, 4), stopTimeoutMs: int(1, 86400000), maxStopAttempts: int(0, 10), overdueMs: int(1, 86400000) }), + InspectionHealth: record({ enabled: bool, status: choice('disabled', 'pending', 'healthy', 'degraded', 'overdue'), lastOccurrenceAt: nullable(int()), lastStartedAt: nullable(int()), lastCompletedAt: nullable(int()), lastSuccessAt: nullable(int()), lastAppliedDecisionAt: nullable(int()), lastMeaningfulProgressAt: nullable(int()), overdueMs: int(1), active: array(record({ runId: id, occurrenceId: nullable(id), status: str(), ageMs: int(), failure: nullable(str()), recovery: nullable(json), recoveryIssue: nullable(ref('ChiefRecoveryIssue')), waitingFor: nullable(str()), nextRecoveryAt: nullable(int()), deadlineAt: int() }, ['runId', 'status', 'ageMs', 'deadlineAt'])) }, ['enabled', 'status', 'lastSuccessAt', 'lastAppliedDecisionAt', 'active']), ChiefCorrection: record({ version: lit('chief-correction/v1'), round: int(1, 5), limit: int(0, 4), status: choice('pending', 'resolved', 'exhausted', 'commit-failed'), feedback: array(record({ code: lit('bot/chief-protocol'), taskId: nullable(id), fields: array(record({ field: str(), expected: str() }, ['field', 'expected'])), message: str(500), at: int(), sideEffects: str(), appliedTaskIds: array(id) }, ['code', 'fields', 'message', 'at']), 5) }, ['version', 'round', 'limit', 'status', 'feedback']), TaskRecord: record({ id, projectId: nullable(id), title: str(), criteria: str(), requesterId: nullable(id), projectTaskKind: choice('code', 'decision', 'release'), status: str(), priority: str(), progress: int(0, 100), note: str(), deliveryAttemptId: id, projectTask: json, collaboration: json, result: json, createdAt: int(), updatedAt: int() }, ['id', 'title', 'status']), TaskView: record({ version: lit('project-development/v1'), projectId: id, taskId: id, title: str(1000), publication: lit('published'), phase: choice('published', 'accepted', 'cancelled', 'stopping', 'blocked', 'awaiting-owner', 'awaiting-chief', 'queued', 'starting', 'running', 'awaiting-input', 'awaiting-acceptance', 'failed', 'paused', 'publishing'), waitingFor: nullable(str()), reason: nullable(str()), botId: nullable(id), currentRole: nullable(role), generation: nullable(int()), attemptId: nullable(id), queueId: nullable(id), workerId: nullable(id), candidateSha: nullable(sha), nextActions: array(str()), evidence: json, contract: nullable(json), source: json, inputs: array(json), effects: array(ref('Effect')), execution: nullable(json), outcome: nullable(json) }, ['projectId', 'taskId', 'phase', 'nextActions']), @@ -181,7 +181,7 @@ q('pools', empty, result({ pools: array(record({ botId: id, role, tasks: array(j q('inputs', obj({ ...taskId, ...page, limit: int(1, 200) }, ['taskId']), record({ inputs: array(ref('Input')), next: nullable(id) }), { source: 'core/src/host/project-inputs.js', notes: 'Receipt states distinguish persisted, delivered, model-declared-read and handled; none imply Task completion.' }) q('sources', obj({ threadId: id, eventId: id, offset: int(), ...page }), record({ sources: array(json), threads: array(json), subscriptions: array(json), events: array(json), next: nullable(id), event: json, text: str(), offset: int(), nextOffset: nullable(int()), sha256: hash }), { source: 'core/src/host/project-sources.js' }) q('execution.boundary', obj(taskId), record({ projectId: id, profile: json, authority: json, task: nullable(json), repository: json, resources: array(json), harnesses: json, limits: json }), { source: 'core/src/host/project-execution-boundary.js', notes: 'Reports effective current boundaries; it neither runs tests nor grants tools/credentials.' }) -q('attempt.context', obj({ ...attemptId, section: choice('roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures'), ...sourcePage }, ['attemptId', 'section']), ref('SourcePage')) +q('attempt.context', obj({ ...attemptId, section: choice('roleSessions', 'roleSessionHistory', 'rolePolicies', 'evidence', 'feedback', 'requirement', 'failure', 'agentDefinitions', 'integrationAdmissions', 'testHistory', 'preparationFailures', 'resumptions'), ...sourcePage }, ['attemptId', 'section']), ref('SourcePage')) q('attempt.integration-state', obj(attemptId, ['attemptId']), result({ attemptId: id, generation: int(), inputs: array(json), availableInputs: array(json), materialization: nullable(json), snapshot: json }), { permission: 'configure', source: 'core/src/host/project-delivery.js', preconditions: ['current-project-manager', 'current-released-integration-checkpoint'], notes: 'Although a query, the downstream integration checkpoint gate requires manager authority and confirmed release.' }) q('attempt.log', obj({ ...attemptId, role, environmentRole: role, offset: int(), limit: int(4, 8192), expectedSha256: hash }, ['attemptId']), ref('LogPage'), { source: 'core/src/host/project-delivery.js', notes: 'UTF-8 byte offsets, not character offsets; nextOffset preserves character boundaries. Rechecks rights after source I/O. No caller path. environmentRole selects that role workspace latest preparation log and is mutually exclusive with SDK execution role. Preparation is not tests or inference.' }) q('release.status', obj(attemptId, ['attemptId']), result({ release: nullable(ref('Release')) }), { source: 'core/src/host/project-releases.js', preconditions: ['current-project-membership', 'current-release-task-attempt'] }) @@ -352,3 +352,10 @@ schemas.TaskView.properties.execution = nullable(ref('ExecutionMetadata')) // Receipt correction is an internal role protocol, observable through existing Task views. schemas.InputCorrection = record({ version: lit('input-correction/v1'), round: int(1, 5), limit: int(0, 4), status: choice('pending', 'resolved', 'exhausted', 'transport-unavailable', 'transport-incomplete', 'interrupted'), proposalSha256: nullable(hash), feedback: array(record({ kind: str(), field: str(), correctable: bool, sideEffects: str(), businessEffects: str() }), 5) }, ['version', 'round', 'limit', 'status']) schemas.ExecutionMetadata.properties.inputCorrection = ref('InputCorrection') + +schemas.ChiefRecoveryIssue = record({ version: lit('chief-recovery/v1'), kind: choice('host-recovering', 'worker-exit-unknown', 'stale-snapshot', 'unsupported-route', 'authority-unavailable', 'resource-unavailable', 'execution-failed'), retryable: bool, conditions: array(str(), 16), actions: array(str(), 16), stopBudgetExhausted: bool, attemptId: id, attemptStatus: str(), candidateSha: nullable(hash) }, ['version', 'kind', 'retryable']) + +schemas.VerificationResumption = record({ kind: str(), decisionId: nullable(id), previousRunId: id, runId: id, previousStatus: str(), candidateSha: hash, targetBotId: id, at: int() }, ['kind', 'runId', 'at']) +schemas.Attempt.properties.resumptions = record({ total: int(), recent: array(ref('VerificationResumption'), 5), source: json }, ['total', 'recent', 'source']) +queries.get('chief.state').response.properties.runs.items.properties.recoveryIssue = ref('ChiefRecoveryIssue') +schemas.ChiefCorrection.properties.feedback.items.properties.recovery = ref('ChiefRecoveryIssue')