From 5cfe6fab3a1c08da3d4efad41733ba751b8ed710 Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:12:06 +0800 Subject: [PATCH] workflow: a saved graph keeps the stop retry deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Stop whose Bot release fails freezes the run on purpose and leaves the step deadline of the node it was on as the only retry (#2453). Saving the graph re-syncs the timers of every live run, and that path cleared the deadline: the frozen run then held its maxConcurrent slot with nothing left to free it until the owner pressed Stop again. syncTimers keeps — and re-arms — the deadline for the one paused state a failed Stop writes, `paused` with `releasePending`, while a Pause the owner asked for still clears it and Resume still arms a fresh one. The workflow subsystem doc said a paused run holds no deadline and that Stop again is the only retry; both now name that exception. Docs-changed: docs/subsystems/workflow.md --- ...0-saved-graph-keeps-stop-deadline.fixed.md | 1 + docs/subsystems/workflow.md | 2 +- plugins/workflow/src/service.js | 10 +++- .../workflow/test/stop-retry-deadline.test.js | 59 +++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 changelog.d/p560-saved-graph-keeps-stop-deadline.fixed.md diff --git a/changelog.d/p560-saved-graph-keeps-stop-deadline.fixed.md b/changelog.d/p560-saved-graph-keeps-stop-deadline.fixed.md new file mode 100644 index 000000000..beee7757b --- /dev/null +++ b/changelog.d/p560-saved-graph-keeps-stop-deadline.fixed.md @@ -0,0 +1 @@ +A Workflow run frozen by a **Stop** whose Bot could not be released keeps the step deadline that retries it when the graph is saved, so an unchanged save from Settings → Workflow graphs no longer takes away the only path out of the lane: the deadline releases again, ends the run as `paused-expired` and starts the task queued behind it. Before this, the save cleared that timer and the run held its `maxConcurrent` slot with nothing left to free it until the owner pressed Stop again ([#2459](https://github.com/ThinkFlowLab/FlyBot/issues/2459)). diff --git a/docs/subsystems/workflow.md b/docs/subsystems/workflow.md index 8c3d848bb..8d9262476 100644 --- a/docs/subsystems/workflow.md +++ b/docs/subsystems/workflow.md @@ -57,7 +57,7 @@ Everything in `plugins/workflow/`: the graph rules, the run store, the matcher, 1. Friends' Bots are not nodes. A `remote` Bot is refused by create, define and the matcher. 2. The canvas and the run share one graph: nodes, edges and positions. A drag commits `workflow/graph`, but only while **Edit** is on — a factory preset is copied first (#1378). A wire or a reconnect that would list a pair the graph already carries commits nothing new, because `normalizeGraph` refuses a graph that lists one pair twice and then saves none of it: unguarded, the plane would show no edge, no change and only a save error (#2163). Status highlighting reads that same graph. Replacing edges does not recompute who is already running. The Settings canvas paints that graph on the same 960px design plane, scaled to the width the settings page gives it, so the canvas is not a second scroll container inside the page's own scroll; its pointer deltas are divided by the scale, and its wires keep a stronger stroke than a plain divider so an edge can be followed at that size (#1483). While another dsh holds the Bot team the host is read-only (`roster.readOnly`): the page still reads through `workflow/state`, keeps its controls disabled, and offers no **Edit**, **Add link** or wire port, so it cannot take a change that core would refuse with `bot/read-only` and then drop (#2187). The same window draws the sidebar's workflow list without **New**, the row's **Rename**/**Delete** or **Hand a task to a Workflow**: the list is read-only too, and an `roster.readOnly` that is not a string, or holds only whitespace, counts as writable — trimmed the way `core/src/client/workflows.js:570` trims it (#2200). The Stage a row opens carries the same lock: it prints the host's sentence under `role="status"` and draws no **Hand a task to a Workflow**, no **Re-match**, no **Pause**/**Resume**/**Stop**, no **Edit**/drag palette and no **Reset**, while the row click, **Progress**, **Flow** and viewing a run still work; a writable roster draws every control exactly as before (#2213). Mounting the list in a read-only window still reads `projects` so the Project filter fills, but it sends no `workflow/ingest` for the open cards: ingest is a claim/match write the host refuses, so a reader opening the list writes nothing; a writable window still ingests every open card (#2222). The read-only sentence can also land after the list has mounted — a cold window starts on the empty roster, `ready: false` and no `readOnly` — so the ingest step asks the value the latest render saw rather than the one the mount saw, and it skips the board read and stops between two cards once the window has turned read-only. The Settings page draws no read-only sentence of its own: the host's Settings header already prints one for every page it shows (`core/src/client/settings.js` `bt-read-only`, `role="status"`, `hostText(roster.readOnly)`), and a second copy inside the page said the same thing twice (#2202). -3. **A live run advances only when its ephemeral Bot replies, following the graph's edges in order.** That reply advances the run and may appear on the workflow stage log; it is not delivered into a persistent sidebar Bot conversation (for example Chief). The `workflow-step` send middleware denies those replies after `stepResult` runs. Nodes start only when their forward predecessors finish and edges reach them. **Early termination (#1537, #1538, #1546, #1570, #1571):** When a node completes **or fails**, the run ends immediately with status `done`, sets `run.earlyEnd` to the condition name, and does not advance to successors, take a success-path `loop`, or burn `onFail` recovery attempts. The two markers that report the task itself end it only on a structured signal (#1546): `needs-human` and `already-claimed` must be a declared OUTPUT whose value **equals** the marker, or a whole reply line that is exactly `EARLY-END ` in any case. A step that quotes an issue title, negates the label or reports it absent keeps running. `tip-unchanged` (#1570) and `open-pr` (#1571) accept that same structured signal — in a declared OUTPUT the value counts as a whole word, so a brief that names `data-open-project` is not `open-pr` (#1866) — and also honor their prose fallbacks (`tip unchanged` / `unchanged tip`, `open PR already` / `already has an open PR` / `has an open PR` / `found an open PR`), so a Bot that forgets the structured marker cannot soft-lock Claim↔Verify on the same tip. See `checkEarlyEnd` in Key files. Factory prompt guidance lives in `EARLY_END_GUIDANCE` (`plugins/workflow/src/template.js`) for Settings graphs that wire Triage / Claim / Verify: Verify must escalate once on an unchanged tip, not re-run the full suite. There is no scripted or seeded run path: `workflow/state` exposes only `live === true` runs — the running ones, plus the last run when it is still `live` and ended `failed` — a failed sibling stays on the stage beside whatever is running now, with `run.failure` (`{ code, at, nodeName?, text? }`) naming what ended it, so the room does not have to open Runs to learn that a run failed (#1493, #1494). The failed sibling is left out when a running run carries the same task id: that is a retry, and the live run keeps the row (#1507). History entries carry the run `id` and the same `failure`, so a Runs row shows when the run ended and why, and a row for a run already on the stage is not listed twice. A run does not create a schedule. `scheduleLabel` is display text. When the host can create Bots, starting a live run (`assign`, `run_workflow`, `ingest`) spawns one ephemeral Bot per node from that node's prompt, role, model and tools, sends it the step from the Main Bot, and removes every instance when the run finishes, fails for good, or is reset. A `failOnce` retry keeps them. They are hidden and left out of the sidebar. Loading the plugin does not spawn Bots for the default workflows. Plugin unload (`dispose`) and the load-time `reap` both end a live run still marked `running`: in-flight nodes are failed, the run is archived as `failed`, and any ephemeral Bot ids are released. A run already written as `running` with no ephemeral map (an older unload) is healed the same way on the next load (#1414). Loading is also the one moment a run the host left behind is noticed when its ephemeral map is still present: `reap` releases those instances and records the run as `failed`. Nothing resumes it. A run that already ended keeps its own status (#1388). `workflow/reset` stopping one live run ends it the same way before its Bots are released: nodes still `running` become `failed`, so the strip cannot keep painting a run that is over (#1424). A graph edit that drops a node that is still running ends that run the way Stop does: the node is failed, the run is archived `failed`, its node Bots are released, and the queue drains (#1549); an id whose release failed stays named on that archived run, and the next load's `reap` retries it, instead of the edit forgetting it (the #1616 contract). Every wait is bounded (#1530). A dispatch that does not land is not marked handed: the chat names the code and the attempt, and it is retried after a short backoff at most twice, never twice inside one hand-off, so a 429 is not hammered. When it still does not land, the step and the run fail with the reason written into the chat. Each running live step also holds a deadline — its workflow's `stepTimeoutMinutes`, `DEFAULT_STEP_TIMEOUT_MINUTES` (45 minutes) unless the record sets one, an integer from 1 to 120 given at create, through `define_workflow`, or through `workflow/step-timeout` (armed even when the host cannot spawn ephemeral Bots; #1530, #1580); when it expires the node and the run fail as `failed`, the run's ephemeral Bots are released, the chat and `run.failure` say the step's deadline was reached (`step deadline reached (45 min)`, the number its lane set), and `warn` names it once; the line never reads as a reply that did not arrive, because a step still working is killed by the same timer (#1868). The deadline is cleared as soon as the step moves, and by Pause, Stop, Reset, delete, `reap` and `dispose`; a paused run holds none, and Resume arms it again from that moment. A timer never outlives the run that started it. A reply that matches no running step — its run ended, or the node already moved — is reported through `warn` once per Bot and run instead of being dropped silently. A reply that says `could not answer your message` or `model call failed` fails that node and does not start its successor. A 429 or rate-limit mention in any other reply does not. Stop and Reset (`workflow/reset`) follow the same order as `dispose`: they release the run's Bots before the record forgets their ids, and an id whose release failed stays named on the still-live run, so the next Stop, Reset, dispose or load-time `reap` retries it (#1476). A run left that way is marked `paused` and `releasePending` and shows as Paused: it hands off no further node, `resume` is refused with an error, and Stop again is the retry. A whole-workflow Reset that could not release every Bot freezes every run it did not reset, because each of them has lost the ids whose release did go. The markers go away when the run ends: Stop again, `dispose`, `reap`, or its own completion. +3. **A live run advances only when its ephemeral Bot replies, following the graph's edges in order.** That reply advances the run and may appear on the workflow stage log; it is not delivered into a persistent sidebar Bot conversation (for example Chief). The `workflow-step` send middleware denies those replies after `stepResult` runs. Nodes start only when their forward predecessors finish and edges reach them. **Early termination (#1537, #1538, #1546, #1570, #1571):** When a node completes **or fails**, the run ends immediately with status `done`, sets `run.earlyEnd` to the condition name, and does not advance to successors, take a success-path `loop`, or burn `onFail` recovery attempts. The two markers that report the task itself end it only on a structured signal (#1546): `needs-human` and `already-claimed` must be a declared OUTPUT whose value **equals** the marker, or a whole reply line that is exactly `EARLY-END ` in any case. A step that quotes an issue title, negates the label or reports it absent keeps running. `tip-unchanged` (#1570) and `open-pr` (#1571) accept that same structured signal — in a declared OUTPUT the value counts as a whole word, so a brief that names `data-open-project` is not `open-pr` (#1866) — and also honor their prose fallbacks (`tip unchanged` / `unchanged tip`, `open PR already` / `already has an open PR` / `has an open PR` / `found an open PR`), so a Bot that forgets the structured marker cannot soft-lock Claim↔Verify on the same tip. See `checkEarlyEnd` in Key files. Factory prompt guidance lives in `EARLY_END_GUIDANCE` (`plugins/workflow/src/template.js`) for Settings graphs that wire Triage / Claim / Verify: Verify must escalate once on an unchanged tip, not re-run the full suite. There is no scripted or seeded run path: `workflow/state` exposes only `live === true` runs — the running ones, plus the last run when it is still `live` and ended `failed` — a failed sibling stays on the stage beside whatever is running now, with `run.failure` (`{ code, at, nodeName?, text? }`) naming what ended it, so the room does not have to open Runs to learn that a run failed (#1493, #1494). The failed sibling is left out when a running run carries the same task id: that is a retry, and the live run keeps the row (#1507). History entries carry the run `id` and the same `failure`, so a Runs row shows when the run ended and why, and a row for a run already on the stage is not listed twice. A run does not create a schedule. `scheduleLabel` is display text. When the host can create Bots, starting a live run (`assign`, `run_workflow`, `ingest`) spawns one ephemeral Bot per node from that node's prompt, role, model and tools, sends it the step from the Main Bot, and removes every instance when the run finishes, fails for good, or is reset. A `failOnce` retry keeps them. They are hidden and left out of the sidebar. Loading the plugin does not spawn Bots for the default workflows. Plugin unload (`dispose`) and the load-time `reap` both end a live run still marked `running`: in-flight nodes are failed, the run is archived as `failed`, and any ephemeral Bot ids are released. A run already written as `running` with no ephemeral map (an older unload) is healed the same way on the next load (#1414). Loading is also the one moment a run the host left behind is noticed when its ephemeral map is still present: `reap` releases those instances and records the run as `failed`. Nothing resumes it. A run that already ended keeps its own status (#1388). `workflow/reset` stopping one live run ends it the same way before its Bots are released: nodes still `running` become `failed`, so the strip cannot keep painting a run that is over (#1424). A graph edit that drops a node that is still running ends that run the way Stop does: the node is failed, the run is archived `failed`, its node Bots are released, and the queue drains (#1549); an id whose release failed stays named on that archived run, and the next load's `reap` retries it, instead of the edit forgetting it (the #1616 contract). Every wait is bounded (#1530). A dispatch that does not land is not marked handed: the chat names the code and the attempt, and it is retried after a short backoff at most twice, never twice inside one hand-off, so a 429 is not hammered. When it still does not land, the step and the run fail with the reason written into the chat. Each running live step also holds a deadline — its workflow's `stepTimeoutMinutes`, `DEFAULT_STEP_TIMEOUT_MINUTES` (45 minutes) unless the record sets one, an integer from 1 to 120 given at create, through `define_workflow`, or through `workflow/step-timeout` (armed even when the host cannot spawn ephemeral Bots; #1530, #1580); when it expires the node and the run fail as `failed`, the run's ephemeral Bots are released, the chat and `run.failure` say the step's deadline was reached (`step deadline reached (45 min)`, the number its lane set), and `warn` names it once; the line never reads as a reply that did not arrive, because a step still working is killed by the same timer (#1868). The deadline is cleared as soon as the step moves, and by Pause, Reset, delete, `reap`, `dispose` and a Stop that released every Bot; a run the owner paused holds none, and Resume arms it again from that moment. A timer never outlives the run that started it. A reply that matches no running step — its run ended, or the node already moved — is reported through `warn` once per Bot and run instead of being dropped silently. A reply that says `could not answer your message` or `model call failed` fails that node and does not start its successor. A 429 or rate-limit mention in any other reply does not. Stop and Reset (`workflow/reset`) follow the same order as `dispose`: they release the run's Bots before the record forgets their ids, and an id whose release failed stays named on the still-live run, so the next Stop, Reset, dispose or load-time `reap` retries it (#1476). A run left that way is marked `paused` and `releasePending` and shows as Paused: it hands off no further node, `resume` is refused with an error, and Stop again is the retry. It also keeps the deadline of the node it was on — the one paused state that holds one — and a graph save re-syncs timers without taking it away (a save that cleared it left the run holding its slot with nothing left to free it); the deadline releases again there, ends the run as `paused-expired` and frees the slot, re-arming itself while a Bot still will not go (#2453, P-560). A whole-workflow Reset that could not release every Bot freezes every run it did not reset, because each of them has lost the ids whose release did go. The markers go away when the run ends: Stop again, `dispose`, `reap`, or its own completion. 4. Each workflow keeps a FIFO queue and up to `maxConcurrent` live runs (default 2, an integer from 1 to 8 on the record, via `workflow/concurrency`, or in Settings → Workflow graphs). A task that does not fit is enqueued. When a run finishes, every workflow with a free slot is drained, oldest enqueue first. A new assign on a workflow that already has a queue waits behind it. A task does not replace a running run or release that run's Bots. Migrating an old store keeps only runs that were still `running` with ephemeral Bot ids; other legacy scripted rows are dropped from active state (finished tasks may remain in lane history). History records a finished or failed live run. Lowering the cap does not stop a run that has already started. Assigning `{ botId }` claims the card for that local Bot and does not enqueue. A remote Bot is refused. `activeCount` is `runningOf(lane)` — live runs with `live === true` and `status === 'running'`. The stage aggregate chip and the Runs tab (运行记录) must agree: every such run appears as its own running row and progress card with its own `status` (never a hardcoded Running), the chip count does not exceed that list, and a failed sibling appears with its time and reason. On the Runs list, the clock stays in a non-shrinking column with a gap before the status chip, and a Failed row paints the Failed chip once — a generic Failed with no host detail does not also draw `wf-hist-reason` (#1618). 5. A node declares up to eight inputs and eight outputs. A required input is an output of the same name on an ancestor, or `task`, `title`, or `brief` on a root. A start with no provider is refused, and Bots spawned for that attempt are released. On a Bot-backed run a successor stays queued until its required inputs are present. If every forward predecessor is done, failed or skipped and an input is still missing, that node fails and the run stops. The dispatch text carries `Input : `. The Bot reports `OUTPUT : `; a single declared output may be the reply body. A declared output a gate reads is required (§3): when a `when`/`else` or `until` edge reads it and the step's reply does not produce it, the step fails with `failure.text` `Missing OUTPUT ` instead of settling, so the gate never reads silence as "not this branch" and the run cannot close `done` on it; a delivered value that does not equal `eq` still completes the step and skips the untaken branch (#1773). Nodes start in edge order. 6. Each node is a real ephemeral Bot for that run, then it is removed. A permanent sidebar Bot (`botId`) is optional and is not required for the step to run. **File and shell rights are the same as a normal sidebar Bot's:** every node Bot gets the `danger-full-access` preset, and `EPHEMERAL_DENIED_TOOLS` still refuses the team-admin tools at the guard and approval layers (#1466, 29 names). **`node.tools` selects connectors only:** `spawn` passes the node's cleaned list to `fb.bot.create`, the host records it as `stepTools`, and `connector-access.js` allows `mcp__github__*` for `github` and `mcp__composio__*` for `composio` — a connector the node did not name is refused, and a node that names none reaches none. The list is also stored in `fields.tools` for the UI. A hidden run Bot cannot show an approval card, so the approval seam answers the same way the guard does. The host stores this on the run Bot as `stepTools` and `permission`. diff --git a/plugins/workflow/src/service.js b/plugins/workflow/src/service.js index 23325b722..30875c596 100644 --- a/plugins/workflow/src/service.js +++ b/plugins/workflow/src/service.js @@ -283,11 +283,17 @@ export function createWorkflowService({ // advance(); without this the step could sit unanswered forever (#1530). if (run?.live !== true) return const running = new Set(orderedIds(item, run.nodeStatus, 'running')) + // A Stop that could not release its Bots freezes the run on purpose (#1476), + // and the deadline of the node it was on is the only retry that run gets + // (#2453). Saving the graph re-syncs every live run, so that one state has to + // keep its deadline: without it the run holds its lane slot with nothing left + // to free it until the owner presses Stop again (P-560). + const stopRetry = run.status === 'running' && run.paused === true && run.releasePending === true for (const [key, entry] of [...timers]) { if (entry.runId !== run.id) continue - if (run.status !== 'running' || run.paused === true || !running.has(entry.nodeId)) clearTimerAt(key) + if (run.status !== 'running' || (run.paused === true && !stopRetry) || !running.has(entry.nodeId)) clearTimerAt(key) } - if (run.status !== 'running' || run.paused === true) return + if (run.status !== 'running' || (run.paused === true && !stopRetry)) return for (const nodeId of running) armDeadline(item, run, nodeId) } diff --git a/plugins/workflow/test/stop-retry-deadline.test.js b/plugins/workflow/test/stop-retry-deadline.test.js index 56fd2dd56..7e2dbbf55 100644 --- a/plugins/workflow/test/stop-retry-deadline.test.js +++ b/plugins/workflow/test/stop-retry-deadline.test.js @@ -179,6 +179,65 @@ test('the retry only happens at the deadline the run reached', async () => { assert.equal(after.history[0].failure?.code, 'paused-expired') }) +// Saving the graph is one of the ways a live run's timers are re-synced +// (`saveGraph` → `syncTimers`), and an unchanged save used to take the deadline +// away from a run a failed Stop had frozen: the run then had no retry left and +// kept its slot until the owner pressed Stop again. The save has to leave the +// deadline alone for that one state, while an owner's own Pause still clears it. +test('saving the graph keeps the deadline a failed Stop left as its only retry', async () => { + const releaseFails = new Set() + const { svc, released, timers, flow, run } = await stoppedButHeld({ releaseFails }) + + // The canvas saves the graph as it stands: the same nodes and edges the run is on. + const item = row(svc.state(), flow.id) + const saved = svc.saveGraph({ workflowId: flow.id, nodes: item.nodes, edges: item.edges }) + assert.equal(saved.ok, true, JSON.stringify(saved)) + assert.equal(row(svc.state(), flow.id).run?.paused, true, 'the save leaves the frozen run where it was') + assert.equal(timers.pending(), 1, 'the deadline the failed Stop left is still armed after a save') + + releaseFails.clear() + await svc.assign({ workflowId: flow.id, taskId: '#2605', title: 'Waiting behind it' }) + assert.equal(row(svc.state(), flow.id).queueDepth, 1, 'the frozen run still holds the only slot') + + await timers.tick(STEP_DEADLINE_MS) + + const after = row(svc.state(), flow.id) + assert.equal(after.run?.task?.id, '#2605', 'the next deadline still frees the slot and starts the parked task') + assert.equal(after.activeCount, 1) + assert.equal(after.queueDepth, 0) + assert.ok(released.includes(run.ephemeral.plan), 'the deadline retried the release that failed') + assert.equal(after.history[0].failure?.code, 'paused-expired', 'the frozen run ends paused-expired instead of holding the lane') + assert.equal(timers.pending(), 1, 'the run that started holds its own deadline and nothing else') + + // With nothing parked behind it, the same deadline frees the lane and leaves it idle. + const soloFails = new Set() + const solo = await stoppedButHeld({ releaseFails: soloFails }) + const soloItem = row(solo.svc.state(), solo.flow.id) + assert.equal(solo.svc.saveGraph({ workflowId: solo.flow.id, nodes: soloItem.nodes, edges: soloItem.edges }).ok, true) + soloFails.clear() + await solo.timers.tick(STEP_DEADLINE_MS) + const idle = row(solo.svc.state(), solo.flow.id) + assert.equal(idle.activeCount, 0, 'nothing is left holding the lane') + assert.equal(idle.history[0].failure?.code, 'paused-expired') + assert.equal(solo.timers.pending(), 0, 'and no timer is left behind') + + // An owner's own Pause still draws no deadline, and saving the graph does not put one back. + const byHand = harness() + const handFlow = make(byHand.svc) + await byHand.svc.assign({ workflowId: handFlow.id, taskId: '#2606', title: 'Paused by hand' }) + const handRun = row(byHand.svc.state(), handFlow.id).run + await byHand.svc.pause({ workflowId: handFlow.id, runId: handRun.id }) + const handItem = row(byHand.svc.state(), handFlow.id) + assert.equal(byHand.svc.saveGraph({ workflowId: handFlow.id, nodes: handItem.nodes, edges: handItem.edges }).ok, true) + assert.equal(byHand.timers.pending(), 0, 'a save puts no deadline back on the run the owner paused') + await byHand.timers.tick(STEP_DEADLINE_MS) + const still = row(byHand.svc.state(), handFlow.id) + assert.equal(still.run?.paused, true, 'the owner Pause is still not timed out') + assert.equal(still.activeCount, 1) + assert.equal(still.history.length, 0) + assert.deepEqual(byHand.released, [], 'and nothing is released behind the owner') +}) + test('a run nobody stopped still fails at its deadline and releases its Bots', async () => { const { svc, released, timers } = harness() const flow = make(svc)