diff --git a/.loopdeck/decisions.md b/.loopdeck/decisions.md index 8d4414c..58cbf79 100644 --- a/.loopdeck/decisions.md +++ b/.loopdeck/decisions.md @@ -297,3 +297,8 @@ _Older decisions archived to [decisions-archive.md](./decisions-archive.md)._ - **Status**: accepted - **Context**: Unattended Codex turns used read-only sandboxing with on-request approval, so Cargo could wait for a host popup that the worker UI could not show. - **Consequences**: Autonomous turns use `workspaceWrite` with `approvalPolicy: on-request`; interactive turns retain read-only approval cards, while all requests remain subject to LoopDeck's destructive-command floor. + +## 2026-10-30 — Execute run-plan phases one turn each with a ready-set scheduler +- **Status**: accepted +- **Context**: The run queue merged every queued phase into one LLM turn, so roles could not be attributed, budgeted per phase, or run concurrently. `prd-parallel-scheduling` makes the phase the unit of execution. +- **Consequences**: `scheduler.rs` validates the dependency graph (cycles rejected at plan creation and run start) and selects a ready set. Each phase runs in its own worktree and branch cut from the default branch, with its own provider session, token budget, and watchdog. `RunPlan.max_concurrency` defaults to four and is configurable in the plan-tonight picker. Dependents enter `Waiting` and are released only when every upstream is Completed or Delivered; a non-green upstream fails its dependents while independent in-flight phases continue. Restart reconciliation auto-resumes a phase only when its worktree and saved session both survive, and parks the rest with a reason. Dependents receive upstream output only as seeded handoff artifacts, never as unmerged branch code. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-budget-split.md b/.loopdeck/handoff-artifacts/parallel-scheduling-budget-split.md new file mode 100644 index 0000000..5d9639d --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-budget-split.md @@ -0,0 +1,32 @@ +--- +artifact: parallel-scheduling-budget-split +author_role: planner +phase: parallel-scheduling/budget-split +type: report +created: 2026-10-30 +summary: Token and wall-clock budgets apply per in-flight phase, so a budget kill ends only that phase and independent phases keep running. +cites: + - parallel-scheduling-one-turn-per-phase +--- + +## Summary +Each `PhaseTask` gets its own `TokenBudget` sized from `RunBudgets.per_phase_token_cap`, with no multiplication by batch size. Its watchdog is `min(phase wall-clock cap, remaining run wall-clock)`. `resolve_phase_token_cap` and `resolve_phase_timeout` keep their per-phase meaning. Token or wall-clock exhaustion marks only that phase Killed, fails its dependents, and lets siblings continue. + +## Requirements +- R1: Every phase turn owns a separate `TokenBudget` built from the per-phase cap. +- R2: The per-phase wall-clock watchdog is the smaller of the phase cap and the run's remaining cap. +- R3: Exceeding either cap marks only that phase Killed with the budget message. +- R4: A budget kill fails the phase's dependents and does not stop independent in-flight phases. +- R5: The run-level wall-clock cap still stops every unfinished phase when it expires. +- R6: Per-phase token usage and wall-clock seconds are recorded on the phase for the morning report. + +## Constraints +- C1: Budget enforcement reuses the existing `TokenBudget` and watchdog kill mechanics; no new timeout machinery. +- C2: The run-level cap remains a hard ceiling across all concurrent phases. + +## Non-Goals +- Per-phase cap overrides distinct from the plan-level value. +- Global or cross-project token budgets. + +## Open Questions +- Whether a phase should receive a partial refill when a sibling frees capacity is deferred. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-gated-start.md b/.loopdeck/handoff-artifacts/parallel-scheduling-gated-start.md new file mode 100644 index 0000000..99fc942 --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-gated-start.md @@ -0,0 +1,32 @@ +--- +artifact: parallel-scheduling-gated-start +author_role: planner +phase: parallel-scheduling/gated-start +type: report +created: 2026-10-30 +summary: Dependent phases hold in Waiting until every upstream phase lands green; a non-green upstream fails its Waiting dependents while independent in-flight phases continue. +cites: + - parallel-scheduling-parallel-batches +--- + +## Summary +`RunPhaseStatus::Waiting` is a real gated state. At run start `scheduler::mark_waiting` holds every queued phase with an unsatisfied dependency. `reconcile_gates` releases Waiting phases to Queued when all dependencies are green, and fails Waiting or Queued phases whose upstream ended Parked, Failed, Interrupted, or Killed. `fail_dependents` then cascades that failure to the rest of the chain. Running siblings are never touched. + +## Requirements +- R1: A phase with an unsatisfied `depends_on` set is `Waiting`, not `Queued`, and is never selected for a turn. +- R2: When every dependency is Completed or Delivered, a Waiting phase becomes Queued and eligible immediately. +- R3: A non-green upstream fails its transitively dependent Waiting and Queued phases, and the payload names the upstream and its reason. +- R4: Independent in-flight phases continue after another phase parks, fails, or hits its budget. +- R5: `StallPolicy::Halt` additionally parks every remaining not-yet-started phase. +- R6: `transition_phase` accepts Queued→Waiting, Waiting→Queued, and Waiting→Running. + +## Constraints +- C1: Gate reconciliation is pure and runs to a fixed point, so a failure can cascade within one pass. +- C2: A scheduler stall with no runnable phase fails the remaining phases instead of spinning. + +## Non-Goals +- Releasing a dependent on a WARN or BLOCK verdict; only Completed or Delivered count as green. +- Interactive approval of a gated start. + +## Open Questions +- Whether a UI control should let a user force-release a Waiting phase is deferred. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-one-turn-per-phase.md b/.loopdeck/handoff-artifacts/parallel-scheduling-one-turn-per-phase.md new file mode 100644 index 0000000..c2f37fb --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-one-turn-per-phase.md @@ -0,0 +1,31 @@ +--- +artifact: parallel-scheduling-one-turn-per-phase +author_role: planner +phase: parallel-scheduling/one-turn-per-phase +type: report +created: 2026-10-30 +summary: The run executor replaces the combined queued-batch turn with one independent turn per run-plan phase, each in its own worktree and branch. +cites: [] +--- + +## Summary +`execute_run` no longer calls `next_queued_batch` or `build_combined_phase_prompt` with many loops. It prepares each ready phase as its own `PhaseTask`, builds a single-loop prompt for it, spawns one provider turn, and records that phase's own verdict, tokens, and wall clock. The plan is mutated only by the serialized scheduler loop. + +## Requirements +- R1: Each run-plan phase executes in exactly one executor turn; no turn covers two phases. +- R2: Each phase gets its own prompt built from one loop location, its pinned interview answers, and its handoff topic. +- R3: Each phase records its own token usage and wall-clock seconds; nothing is divided across a batch. +- R4: The phase prompt names its own branch and forbids merging another phase's branch. +- R5: Only the scheduler loop writes `run-plan.yaml`; phase turns run in Tokio tasks and touch no plan state. +- R6: A missing `execution.yaml` focus slot is skipped, not treated as an error; `run-plan.yaml` stays authoritative per phase. + +## Constraints +- C1: `AppState.claude_sessions` is keyed by worktree path, so one worktree per phase keeps concurrent turns isolated. +- C2: Pending question, permission, plan, and interrupt slots use the phase worktree as the control key. + +## Non-Goals +- Concurrent fan-out and the Waiting gate, which are `parallel-scheduling/parallel-batches` and `gated-start`. +- Changes to the one-run-per-project admission lock. + +## Open Questions +- Whether a phase should inherit an interactive `execution.yaml` current pointer when it is not the first started is left to the UI-focus design. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-parallel-batches.md b/.loopdeck/handoff-artifacts/parallel-scheduling-parallel-batches.md new file mode 100644 index 0000000..461dee9 --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-parallel-batches.md @@ -0,0 +1,32 @@ +--- +artifact: parallel-scheduling-parallel-batches +author_role: planner +phase: parallel-scheduling/parallel-batches +type: report +created: 2026-10-30 +summary: Phases with satisfied, disjoint depends_on run concurrently in separate worktrees under RunPlan.max_concurrency, default four and configurable in the picker. +cites: + - parallel-scheduling-one-turn-per-phase +--- + +## Summary +A ready-set scheduler replaces topological batches. `scheduler::select_ready` returns queued phases whose dependencies are all green, capped by spare concurrency. `execute_run` prepares each one in its own worktree cut from the default branch, spawns a Tokio task, and joins completions with `JoinSet`. `RunPlan.max_concurrency` defaults to four and the plan-tonight wizard exposes it. + +## Requirements +- R1: `RunPlan.max_concurrency` is persisted and defaults to four; values outside 1..=16 are rejected at plan creation. +- R2: `scheduler::select_ready` returns only queued phases whose every `depends_on` phase is Completed or Delivered. +- R3: The executor starts ready phases until `max_concurrency` turns are in flight; a completion immediately frees a slot. +- R4: Each phase worktree is cut from the repository default branch; no phase stacks on a sibling's branch. +- R5: Each phase opens its own draft PR from its own branch. +- R6: The wizard passes the concurrency value through `createRunPlan` and shows it in the review summary. + +## Constraints +- C1: Worktree creation stays in the serialized scheduler loop, so concurrent `git worktree add` calls cannot race. +- C2: One worktree per phase keeps provider sessions keyed by path and prevents cross-phase session sharing. + +## Non-Goals +- Auto-merging concurrent branches; arbitration remains `prd-arbitration`. +- Cross-project scheduling and global budgets. + +## Open Questions +- Whether a provider-level rate limit should further cap effective concurrency is deferred. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-restart-reconcile.md b/.loopdeck/handoff-artifacts/parallel-scheduling-restart-reconcile.md new file mode 100644 index 0000000..8b56bae --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-restart-reconcile.md @@ -0,0 +1,32 @@ +--- +artifact: parallel-scheduling-restart-reconcile +author_role: planner +phase: parallel-scheduling/restart-reconcile +type: report +created: 2026-10-30 +summary: Restart reconciliation handles multiple in-flight phases, auto-resuming each phase whose worktree and saved provider session survive and parking the rest with a reason. +cites: + - parallel-scheduling-one-turn-per-phase +--- + +## Summary +`reconcile_running_phases` now classifies every `Running` phase instead of downgrading all of them to Interrupted. `scheduler::classify_interrupted_phase` requires both the phase worktree and a saved session id. A survivor returns to `Queued` with `resume_session_id` set; the executor spawns with `--resume` in the same worktree. Anything less parks with a reason that names what was lost. + +## Requirements +- R1: Every `Running` phase on disk is reconciled on the next queue attempt or status read. +- R2: A phase whose worktree exists and whose session id is saved returns to Queued with that session pinned for resume. +- R3: The resumed turn spawns in the recorded worktree with the saved provider session. +- R4: A phase missing either survivor becomes Parked, and the payload names the missing piece. +- R5: Legacy plans fall back to the plan-level worktree and the active transcript's last session id. +- R6: `transition_phase` accepts Running→Queued for this auto-resume edge. + +## Constraints +- C1: The decision is pure (`classify_interrupted_phase`) and unit-tested independently of the filesystem. +- C2: A stale `execution.yaml` current pointer left on an auto-resumed phase is abandoned so queueing can proceed. + +## Non-Goals +- Resuming a provider process across an app restart; a new process resumes the saved session id. +- Automatic retry of a parked restarted phase. + +## Open Questions +- Whether a resumed phase should replay its last partial turn or start a fresh instruction is still provider-defined. diff --git a/.loopdeck/handoff-artifacts/parallel-scheduling-tests.md b/.loopdeck/handoff-artifacts/parallel-scheduling-tests.md new file mode 100644 index 0000000..89c45cc --- /dev/null +++ b/.loopdeck/handoff-artifacts/parallel-scheduling-tests.md @@ -0,0 +1,34 @@ +--- +artifact: parallel-scheduling-tests +author_role: planner +phase: parallel-scheduling/tests +type: report +created: 2026-10-30 +summary: Scheduler property tests cover diamond graphs, cycle rejection, restart recovery, gates, and worktree cleanup isolation; the full suite passes. +cites: + - parallel-scheduling-parallel-batches + - parallel-scheduling-gated-start + - parallel-scheduling-restart-reconcile +--- + +## Summary +`src-tauri/src/scheduler.rs` ships fourteen unit tests: diamond validation and ordering, gate release only after both parents, transitive dependent failure, independent in-flight survival, Halt parking, cycle and dangling-edge rejection, concurrency caps, watchdog selection, restart decisions, branch uniqueness, and stall detection. `commands/run_queue.rs` adds worktree tests: disjoint phase paths, isolated cleanup, resume, recreation from a surviving branch, and default-branch basing. + +## Requirements +- R1: A diamond graph validates, orders roots first, and releases its leaf only when both parents are green. +- R2: A cycle, a self-dependency, and a dangling edge are rejected before any worktree exists. +- R3: A non-green upstream fails transitively dependent phases while an independent in-flight phase keeps Running. +- R4: Restart reconciliation auto-resumes only when worktree and session both survive; otherwise it parks with a reason. +- R5: Two phase worktrees are disjoint, and removing one never touches its sibling. +- R6: The full Rust library suite and the frontend test suite pass. + +## Constraints +- C1: Scheduler tests are pure and need no provider process or Tauri state. +- C2: Worktree tests use scratch git repositories under the system temp directory. + +## Non-Goals +- Live provider turns in the automated suite; the role demo remains an explicitly ignored test. +- Load or soak testing above four concurrent phases. + +## Open Questions +- Whether the ignored role demo should become a scheduled integration test is deferred. diff --git a/.loopdeck/loops.md b/.loopdeck/loops.md index 3cdc55c..dd7a2fb 100644 --- a/.loopdeck/loops.md +++ b/.loopdeck/loops.md @@ -4,10 +4,14 @@ _Older loops archived to [loops-archive.md](./loops-archive.md)._ ## Current -- **Status**: in_progress — `provider-contract` Phases 2–3 are implemented in worktree `run/provider-contract-normalize-claude-plus3-1aa529a93261`: both adapters dual-emit normalized `RunEvent`s, ts-rs bindings are CI-drift-checked, the run-queue consumer migrated, fixture/frontend contract tests pass, and the verify gate is PASS. Draft PR #110 is open. -- **Next**: review and merge draft PR #110; then start the next queued `execution-foundations` PRD. +- **Status**: in_progress — all six `prd-parallel-scheduling` loops are implemented in worktree `run/parallel-scheduling-one-turn-per-phase-plus5-d13a619fe69b`: per-phase turns, dependency-aware fan-out with per-phase worktrees, the `Waiting` gate, multi-phase restart reconciliation, per-phase budgets, and scheduler property/worktree tests. Verify gate is PASS; draft PR #113 is open. +- **Next**: review and merge the parallel-scheduling draft PR; then run a real overnight plan with two disjoint phases to confirm concurrent draft PRs on the user's machine. ## Next Steps +- [x] Resolve PR #113 against agent-handoff support on main; preserve bounded injection, citation attribution, and waiting-phase restart recovery. +- [ ] Split `commands/run_queue.rs` and `run_executor.rs` (both exceed 1,500 lines). +- [ ] Review & merge: https://github.com/suprie/loopdeck/pull/113 +- [x] `role-based-orchestration` / `prd-parallel-scheduling` — all six loops (one-turn-per-phase, parallel-batches, gated-start, restart-reconcile, budget-split, tests). - [x] Fix the Claude control_response schema and commit it. - [x] Create a fresh run worktree and plan from `feat/provider-event-fixtures`. - [x] `execution-foundations` / `prd-provider-contract` Phase 2 — normalize Claude and Codex provider output into the versioned `RunEvent` envelope. @@ -28,6 +32,11 @@ _Older loops archived to [loops-archive.md](./loops-archive.md)._ ## History +### 2026-10-30 — role-based-orchestration / prd-parallel-scheduling (all six loops) +- **Status**: completed +- **Completed**: 2026-10-30 +- **Summary**: The run executor now fires one turn per run-plan phase instead of one combined queued-batch turn. A new `scheduler.rs` validates the dependency graph (cycles, self-edges, and dangling edges are rejected at plan creation and again at run start), holds dependents in a persisted `Waiting` state, and releases them only when every upstream is Completed or Delivered. Ready phases fan out concurrently under `RunPlan.max_concurrency` (default 4, configurable in the plan-tonight picker), each in its own worktree and branch cut from the default branch, each with its own provider session, token budget, and wall-clock watchdog. A non-green upstream fails its Waiting dependents while independent in-flight phases continue. Restart reconciliation now classifies every `Running` phase, auto-resuming only when worktree and saved session both survive and parking the rest with a reason. Scheduler property tests cover generated DAGs, cycle rejection, and restart decisions; worktree tests prove disjoint paths and isolated cleanup. `cargo test` (747 passed), `cargo clippy --all-targets` (0 errors), `npx tsc --noEmit`, `npm run build`, and `npm run test:frontend` (34 passed) are green. + ### 2026-10-28 — execution-foundations / prd-provider-contract Phases 2–3 - **Status**: completed - **Completed**: 2026-10-28 @@ -99,6 +108,11 @@ _Older loops archived to [loops-archive.md](./loops-archive.md)._ ## History +### 2026-10-30 — role-based-orchestration / prd-parallel-scheduling (all six loops) +- **Status**: completed +- **Completed**: 2026-10-30 +- **Summary**: The run executor now fires one turn per run-plan phase instead of one combined queued-batch turn. A new `scheduler.rs` validates the dependency graph (cycles, self-edges, and dangling edges are rejected at plan creation and again at run start), holds dependents in a persisted `Waiting` state, and releases them only when every upstream is Completed or Delivered. Ready phases fan out concurrently under `RunPlan.max_concurrency` (default 4, configurable in the plan-tonight picker), each in its own worktree and branch cut from the default branch, each with its own provider session, token budget, and wall-clock watchdog. A non-green upstream fails its Waiting dependents while independent in-flight phases continue. Restart reconciliation now classifies every `Running` phase, auto-resuming only when worktree and saved session both survive and parking the rest with a reason. Scheduler property tests cover generated DAGs, cycle rejection, and restart decisions; worktree tests prove disjoint paths and isolated cleanup. `cargo test` (747 passed), `cargo clippy --all-targets` (0 errors), `npx tsc --noEmit`, `npm run build`, and `npm run test:frontend` (34 passed) are green. + ### 2026-08-30 — Memory hygiene: 3,000-token budget + compaction - **Status**: completed - **Completed**: 2026-08-30 diff --git a/src-tauri/src/commands/agent.rs b/src-tauri/src/commands/agent.rs index 921626f..9b37232 100644 --- a/src-tauri/src/commands/agent.rs +++ b/src-tauri/src/commands/agent.rs @@ -1700,6 +1700,29 @@ pub(crate) fn spawn_fresh_with_config( policy_root: &Path, agent_config: &crate::config::AgentConfig, force_autonomous: bool, +) -> Result>, AppError> { + spawn_fresh_with_config_resume( + state, + path, + policy_root, + agent_config, + force_autonomous, + None, + ) +} + +/// Resume-aware counterpart of [`spawn_fresh_with_config`]. Restart +/// reconciliation (`parallel-scheduling/restart-reconcile`) passes the saved +/// provider session id so an auto-resumed phase keeps its model context in a +/// surviving worktree. `None` preserves the fresh-session behavior every +/// other caller relies on. +pub(crate) fn spawn_fresh_with_config_resume( + state: &AppState, + path: &Path, + policy_root: &Path, + agent_config: &crate::config::AgentConfig, + force_autonomous: bool, + resume_session_id: Option<&str>, ) -> Result>, AppError> { // The caller has already performed the busy check / archive steps when it // is the normal `spawn_fresh` path. Multi-agent worktrees are unique, so @@ -1710,7 +1733,7 @@ pub(crate) fn spawn_fresh_with_config( resolve_permission_policy(state, policy_root) }; - let session = HarnessSession::spawn(path, agent_config, None, policy)?; + let session = HarnessSession::spawn(path, agent_config, resume_session_id, policy)?; let arc = Arc::new(tokio::sync::Mutex::new(session)); state .claude_sessions @@ -1896,11 +1919,55 @@ pub(crate) async fn start_fresh_and_record_streaming_in_root_with_config( control_key: Option<&Path>, force_autonomous: bool, plan_mode: bool, +) -> Result { + start_fresh_and_record_streaming_in_root_with_config_resume( + state, + path, + policy_root, + prompt, + title, + channel, + token_budget, + run_events, + agent_config, + control_key, + force_autonomous, + plan_mode, + None, + ) + .await +} + +/// Resume-aware counterpart of +/// [`start_fresh_and_record_streaming_in_root_with_config`]. The run-queue +/// executor passes a phase's saved session id only when restart +/// reconciliation classified it auto-resumable; every other caller goes +/// through the wrapper above with `None`. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn start_fresh_and_record_streaming_in_root_with_config_resume( + state: &AppState, + path: &Path, + policy_root: &Path, + prompt: &str, + title: Option, + channel: &Channel, + token_budget: Option<&TokenBudget>, + run_events: Option, + agent_config: Option<&crate::config::AgentConfig>, + control_key: Option<&Path>, + force_autonomous: bool, + plan_mode: bool, + resume_session_id: Option<&str>, ) -> Result { let session_arc = match agent_config { - Some(config) => { - spawn_fresh_with_config(state, path, policy_root, config, force_autonomous)? - } + Some(config) => spawn_fresh_with_config_resume( + state, + path, + policy_root, + config, + force_autonomous, + resume_session_id, + )?, None => spawn_fresh(state, path, policy_root, force_autonomous).await?, }; let mut session = session_arc.lock().await; diff --git a/src-tauri/src/commands/run_queue.rs b/src-tauri/src/commands/run_queue.rs index 820e6cf..dd188db 100644 --- a/src-tauri/src/commands/run_queue.rs +++ b/src-tauri/src/commands/run_queue.rs @@ -9,11 +9,14 @@ //! `commands::agent` already owns that orchestration for the same reason //! (see `run_executor.rs`'s module docs). -use super::agent::{mark_turn_terminal, start_fresh_and_record_streaming_in_root_with_config}; +use super::agent::{ + mark_turn_terminal, start_fresh_and_record_streaming_in_root_with_config, + start_fresh_and_record_streaming_in_root_with_config_resume, +}; use super::state::{ fire_interrupt, resolve_agent_config, resolve_agent_config_by_id, resolve_root, AppState, }; -use crate::agents::{AskUserQuestionSpec, ClaudeEvent, TokenBudget}; +use crate::agents::{AgentResponse, AskUserQuestionSpec, ClaudeEvent, TokenBudget}; use crate::config::AgentHarness; use crate::delivery; use crate::delivery_retry; @@ -32,6 +35,7 @@ use crate::run_executor::{ extract_verdict, ResolvedBudgets, RunHandle, RunVerdict, }; use crate::runplan::{self, InterviewStatus, RunBudgets, RunPhaseStatus, RunPlan, StallPolicy}; +use crate::scheduler; use chrono::Utc; use serde::Serialize; use std::collections::{HashMap, HashSet}; @@ -178,92 +182,58 @@ where } } -fn run_worktree_path(root: &Path, branch: &str) -> PathBuf { - root.join(".loopdeck").join("runs").join(branch) -} - -/// Branch name for a run's worktree: derived from the first queued phase's -/// execution ID (already `/`, e.g. -/// `wake-up/notify-after-completed`) so `git branch`/GitHub show what the run -/// is actually doing, not just an opaque `run-`. A `-plusN` suffix -/// notes when the run covers more than one phase. The plan id's trailing -/// UUID segment is kept as a short suffix so two runs touching the same -/// phase (e.g. a retry) never collide on branch name. -fn run_branch_name(plan: &RunPlan) -> String { - let slug = plan - .phases - .first() - .map(|p| p.execution_id.as_str()) - .unwrap_or(plan.id.as_str()); - let extra = plan.phases.len().saturating_sub(1); - let suffix = plan.id.rsplit('-').next().unwrap_or(plan.id.as_str()); - let name = if extra > 0 { - format!("{slug}-plus{extra}-{suffix}") - } else { - format!("{slug}-{suffix}") - }; - format!( - "run/{}", - name.replace(|c: char| !c.is_ascii_alphanumeric() && c != '-', "-") - ) -} - -/// Create the one isolated worktree used by every phase in a run. The path and -/// branch are persisted before a turn begins, so a failed/terminated run stays -/// inspectable and a restart does not create a second branch. +/// Ensure a phase's own isolated worktree exists (`parallel-scheduling`). +/// +/// Every phase gets one worktree and one branch, cut from the repository's +/// default branch — no stacking. Dependents receive upstream output only via +/// handoff artifacts, never via an upstream branch's unmerged code (the +/// approved branch-scoping answer). One worktree per in-flight phase also +/// keeps the session cache honest: `AppState.claude_sessions` is keyed by +/// worktree path, so concurrent turns can never share a provider process. /// /// Resume behavior (`prd-verified-delivery-reconciliation` Phase 2): when the /// persisted worktree is gone from Git's inventory but its **branch still -/// exists** (user deleted the tree, or `git worktree prune` ran), the -/// worktree is recreated at the same path from the surviving branch instead -/// of hard-stopping — no second branch, no lost work. Only a missing branch -/// (nothing left to resume from) preserves the run for inspection. -fn ensure_worktree(root: &Path, plan: &mut RunPlan) -> Result { - if let Some(path) = &plan.environment.worktree_path { - if git::worktree_list(root) - .map_err(AppError::RunPlan)? - .iter() - .any(|known| known == path) - { - return Ok(path.clone()); - } - let branch = plan.environment.worktree_branch.as_deref().ok_or_else(|| { - AppError::RunPlan(format!( - "run worktree {} is missing and no branch was recorded; \ - preserving the run for inspection", - path.display() - )) - })?; - if git::branch_exists(root, branch) { - let worktree = - git::worktree_add_existing(root, path, branch).map_err(AppError::RunPlan)?; - info!( - "recreated run worktree {} from surviving branch {branch}", - worktree.path.display() - ); - return Ok(worktree.path); - } - return Err(AppError::RunPlan(format!( - "run worktree {} and branch {branch} are both missing; \ - preserving the run for inspection", - path.display() - ))); +/// exists**, the worktree is recreated at the same path from the surviving +/// branch instead of hard-stopping. Only a missing branch preserves the run +/// for inspection. +/// +/// Returns `(path, branch, created)`; the caller persists both fields on the +/// phase before spawning the turn. +fn ensure_phase_worktree( + root: &Path, + plan: &RunPlan, + index: usize, +) -> Result<(PathBuf, String, bool), AppError> { + let phase = &plan.phases[index]; + let branch = phase + .worktree_branch + .clone() + .unwrap_or_else(|| scheduler::phase_branch_name(&plan.id, &phase.execution_id, index)); + let path = phase + .worktree_path + .clone() + .unwrap_or_else(|| scheduler::phase_worktree_path(root, &branch)); + + let known = git::worktree_list(root).map_err(AppError::RunPlan)?; + if known.iter().any(|candidate| candidate == &path) { + return Ok((path, branch, false)); + } + if git::branch_exists(root, &branch) { + let worktree = + git::worktree_add_existing(root, &path, &branch).map_err(AppError::RunPlan)?; + info!( + "recreated phase worktree {} from surviving branch {branch}", + worktree.path.display() + ); + return Ok((worktree.path, worktree.branch, true)); } - let branch = run_branch_name(plan); - let path = run_worktree_path(root, &branch); + ignore_managed_runs_dir(root); std::fs::create_dir_all(path.parent().expect("worktree path has a parent"))?; - // Clean handoff (`prd-verified-delivery-reconciliation` Phase 4): every - // new run branch is cut from the repo's default branch — fresh, never - // stacked on a delivered/stray checkout the main worktree happens to be - // on. A fix landing on a *different* branch must not silently ride into - // the next run's base. let base = git::default_branch(root).unwrap_or_else(|| "HEAD".to_string()); let worktree = git::worktree_add_from(root, &path, &branch, &base).map_err(AppError::RunPlan)?; - plan.environment.worktree_path = Some(worktree.path.clone()); - plan.environment.worktree_branch = Some(worktree.branch); - Ok(worktree.path) + Ok((worktree.path, worktree.branch, true)) } /// Persist the clean-handoff record at delivery (Phase 4, `clean-handoff`). @@ -271,9 +241,9 @@ fn ensure_worktree(root: &Path, plan: &mut RunPlan) -> Result /// starts from* — the `.loopdeck/runs//` worktree is only created /// when the next loop actually starts. Best-effort: a failed write parks a /// note on the last phase, it never fails the delivery itself. -fn record_handoff(root: &Path, plan: &RunPlan, worktree: &Path, pr_url: &str) { +fn record_handoff(root: &Path, branch: &str, worktree: &Path, pr_url: &str) { let record = handoff::HandoffRecord { - delivered_branch: plan.environment.worktree_branch.clone().unwrap_or_default(), + delivered_branch: branch.to_string(), pr_url: pr_url.to_string(), worktree: worktree.to_path_buf(), next_base: git::default_branch(root).unwrap_or_else(|| "main".to_string()), @@ -284,37 +254,23 @@ fn record_handoff(root: &Path, plan: &RunPlan, worktree: &Path, pr_url: &str) { } } -/// Promote the agent-authored phase artifacts from the isolated worktree into -/// the project's persistent handoff store. Validation and duplicate detection -/// live in `handoff`; this helper only applies the batch boundary and returns -/// the first error so the executor can park the phase without losing the -/// original artifact. -fn emit_phase_artifacts( +/// Promote one phase's agent-authored artifact from its worktree into the +/// project's persistent handoff store. Validation and duplicate detection +/// live in `handoff`; a missing or mismatched artifact errors so the executor +/// can park the phase without losing the original. +fn emit_phase_artifact( root: &Path, worktree: &Path, - plan: &RunPlan, - locs: &[(String, epic::LoopLocation, Vec)], + execution_id: &str, + topic: Option<&str>, ) -> Result<(), AppError> { - for (execution_id, _, _) in locs { - let phase = plan - .phases - .iter() - .find(|phase| phase.execution_id == *execution_id) - .ok_or_else(|| { - AppError::RunPlan(format!( - "phase \"{execution_id}\" is missing from its run plan" - )) - })?; - // Handoffs are opt-in per phase (`prd-agent-handoff` Phase 1): a phase - // without a configured topic has no artifact contract, so legacy plans - // keep executing unchanged (ADR-3). Once a topic is configured the - // slug is authoritative — the executor never derives one from the - // execution id, and a missing/mismatched artifact parks the phase. - let Some(topic) = phase.handoff_topic.as_deref() else { - continue; - }; - handoff::copy_agent_artifact(root, worktree, execution_id, topic)?; - } + // Handoffs are opt-in per phase (`prd-agent-handoff` Phase 1): a phase + // without a configured topic has no artifact contract, so legacy plans + // keep executing unchanged (ADR-3). + let Some(topic) = topic else { + return Ok(()); + }; + handoff::copy_agent_artifact(root, worktree, execution_id, topic)?; Ok(()) } @@ -636,16 +592,32 @@ pub async fn create_run_plan( // and sparse — phases without an entry stay on the default agent. assignments: Option>, handoff_topics: Option>, + // Maximum phases that may execute at once (`prd-parallel-scheduling` + // Phase 2). `None` keeps the four-phase default; the picker may set any + // value in 1..=MAX_RUN_MAX_CONCURRENCY. + max_concurrency: Option, state: State<'_, AppState>, ) -> Result { let root = resolve_root(&state, &path)?; validate_budgets(&budgets)?; + let max_concurrency = max_concurrency.unwrap_or(limits::DEFAULT_RUN_MAX_CONCURRENCY); + if max_concurrency == 0 || max_concurrency > limits::MAX_RUN_MAX_CONCURRENCY { + return Err(AppError::RunPlan(format!( + "max concurrency must be between 1 and {} phases", + limits::MAX_RUN_MAX_CONCURRENCY + ))); + } if execution_ids.is_empty() { return Err(AppError::RunPlan( "select at least one phase to queue".into(), )); } + if execution_ids.iter().collect::>().len() != execution_ids.len() { + return Err(AppError::RunPlan( + "a phase can only appear once in a run plan".into(), + )); + } { let handles = state.run_handles.lock().map_err(|_| AppError::LockError)?; @@ -732,12 +704,17 @@ pub async fn create_run_plan( handoff_topics.unwrap_or_default(), )?; plan.budgets = budgets; + plan.max_concurrency = max_concurrency; + // Reject a cycle before it is persisted: a plan with one would hold every + // member Waiting forever when the run starts. + scheduler::validate_dependency_graph(&plan.phases)?; runplan::save(&root, &plan)?; info!( - "run plan queued for {} with {} phase(s)", + "run plan queued for {} with {} phase(s), max concurrency {}", root.display(), - plan.phases.len() + plan.phases.len(), + plan.max_concurrency ); Ok(plan) } @@ -778,7 +755,7 @@ pub async fn queue_run( if !plan .phases .iter() - .any(|p| p.status == RunPhaseStatus::Queued) + .any(|p| matches!(p.status, RunPhaseStatus::Queued | RunPhaseStatus::Waiting)) { return Err(AppError::RunPlan( "the run plan has no queued phases".into(), @@ -786,7 +763,8 @@ pub async fn queue_run( } if let Some(phase) = plan.phases.iter().find(|p| { - p.status == RunPhaseStatus::Queued && p.interview_status == InterviewStatus::Pending + matches!(p.status, RunPhaseStatus::Queued | RunPhaseStatus::Waiting) + && p.interview_status == InterviewStatus::Pending }) { return Err(AppError::RunPlan(format!( "phase \"{}\" has not been interviewed — answer or skip its pre-flight interview before queuing the run", @@ -865,12 +843,22 @@ pub async fn cancel_run(path: String, state: State<'_, AppState>) -> Result<(), )); } - // Phase sessions run in the isolated worktree, while legacy/manual - // sessions are keyed by the registered root. Signal both safely so Stop - // remains immediate during the migration to worktree-backed runs. + // Phase sessions run in their own isolated worktrees, while legacy/manual + // sessions are keyed by the registered root. Signal every active phase + // worktree and the legacy plan-level worktree so Stop reaches all + // concurrent turns immediately. let _ = fire_interrupt(&state, &root)?; - if let Some(worktree) = runplan::load(&root)?.and_then(|plan| plan.environment.worktree_path) { - let _ = fire_interrupt(&state, &worktree)?; + if let Some(plan) = runplan::load(&root)? { + if let Some(worktree) = plan.environment.worktree_path.as_ref() { + let _ = fire_interrupt(&state, worktree)?; + } + for phase in plan.phases.iter().filter(|phase| { + scheduler::is_active(phase.status) || phase.status == RunPhaseStatus::Running + }) { + if let Some(worktree) = phase.worktree_path.as_ref() { + let _ = fire_interrupt(&state, worktree)?; + } + } } info!("cancel_run fired for {}", root.display()); Ok(()) @@ -1104,7 +1092,20 @@ fn reconcile_inactive_execution_state( now, false, )?, - RunPhaseStatus::Queued | RunPhaseStatus::Running | RunPhaseStatus::Waiting => { + // A `Queued` phase that restart reconciliation auto-resumed is + // restart residue: release the stale focus pointer so the next queue + // attempt is not blocked by a loop nothing is driving. A plain queued + // phase that merely matches a manually promoted current loop keeps + // it — the user is still working that loop. + RunPhaseStatus::Queued if phase.resume_session_id.is_some() => state.abandon_current( + format!( + "reconciled auto-resumed run phase \"{}\" left queued after restart", + phase.execution_id + ), + now, + false, + )?, + RunPhaseStatus::Queued | RunPhaseStatus::Waiting | RunPhaseStatus::Running => { return Ok(None) } }; @@ -1341,12 +1342,16 @@ pub(crate) fn requeue_terminal_phase( loop { let mut changed = false; for phase in &mut plan.phases { - let blocked_by_requeued_phase = phase.status == RunPhaseStatus::Parked - && phase - .park_payload - .as_deref() - .is_some_and(|payload| payload.starts_with("blocked: depends on parked phase")) - && phase.depends_on.iter().any(|id| requeued.contains(id)); + // Dependents fail (not park) under the parallel scheduler; accept + // both markers so a retry of the upstream clears the cascade. + let blocked_by_requeued_phase = + matches!( + phase.status, + RunPhaseStatus::Parked | RunPhaseStatus::Failed + ) && phase.park_payload.as_deref().is_some_and(|payload| { + payload.starts_with("blocked: depends on parked phase") + || payload.starts_with("failed: upstream phase") + }) && phase.depends_on.iter().any(|id| requeued.contains(id)); if blocked_by_requeued_phase { phase.status = RunPhaseStatus::Queued; phase.park_payload = None; @@ -1382,106 +1387,6 @@ fn requeue_all_terminal_phases(plan: &mut RunPlan) -> usize { } requeued } -/// Indices of every phase currently `Queued`, in plan order — the unit of -/// work `execute_run` hands to one combined LLM turn. `None` once nothing is -/// left to run. -/// -/// Per-phase staffing (`prd-role-foundations` Phase 4): one combined turn can -/// only run under one agent, so the batch is the maximal *prefix* of queued -/// phases sharing the same `assigned_agent_id`. A staffing change splits the -/// turn — dev builds, then QA verifies, as separate turns. All-unassigned -/// plans (the pre-assignment shape) still batch into one turn exactly as -/// before. -/// -/// Handoff boundary (`prd-agent-handoff` Phase 2): a phase with a direct -/// `depends_on` ancestor that publishes a handoff topic starts a new turn. -/// Its ancestor's artifact was emitted by an earlier turn, so this turn can -/// inline it. Phases with no publishing ancestor batch exactly as before. -pub(crate) fn next_queued_batch(plan: &RunPlan) -> Option> { - let indices: Vec = plan - .phases - .iter() - .enumerate() - .filter(|(_, phase)| phase.status == RunPhaseStatus::Queued) - .map(|(i, _)| i) - .collect(); - if indices.is_empty() { - return None; - } - let staffing = plan.phases[indices[0]].assigned_agent_id.clone(); - let mut batch = vec![indices[0]]; - for &idx in &indices[1..] { - if plan.phases[idx].assigned_agent_id != staffing { - break; - } - if run_executor::consumes_handoff(plan, &plan.phases[idx]) { - break; - } - batch.push(idx); - } - Some(batch) -} - -/// Split a combined turn's total token usage evenly across every phase in -/// the batch that produced it — there is no per-loop usage signal to divide -/// on more precisely. -fn set_batch_token_usage(plan: &mut RunPlan, batch: &[usize], total: u64, batch_len: u64) { - let per_phase = total / batch_len.max(1); - for &idx in batch { - plan.phases[idx].token_usage = per_phase; - } -} - -fn kill_batch(plan: &mut RunPlan, batch: &[usize], reason: &str) { - for &idx in batch { - plan.phases[idx].status = RunPhaseStatus::Killed; - plan.phases[idx].park_payload = Some(reason.to_string()); - } -} - -fn park_batch(plan: &mut RunPlan, batch: &[usize], reason: &str) { - for &idx in batch { - plan.phases[idx].status = RunPhaseStatus::Parked; - plan.phases[idx].park_payload = Some(reason.to_string()); - } -} - -fn park_blocked_dependents(plan: &mut RunPlan, execution_id: &str) { - for blocked_id in run_executor::phases_blocked_by_park(plan, execution_id, plan.stall_policy) { - if let Some(phase) = plan - .phases - .iter_mut() - .find(|phase| phase.execution_id == blocked_id) - { - phase.status = RunPhaseStatus::Parked; - phase.park_payload = Some(format!( - "blocked: depends on parked phase \"{execution_id}\"" - )); - } - } -} - -/// Park a batch whose assigned agent no longer resolves in the roster -/// (`prd-role-foundations` Phase 4): the payload names the missing agent for -/// the morning report, and the plan's `stall_policy` decides — via the same -/// dependent-parking every other park site uses — whether the rest of the -/// plan stays eligible. Split out of `execute_run` so the policy behavior is -/// testable without a live turn. -fn park_missing_agent(plan: &mut RunPlan, batch: &[usize], agent_id: &str) { - park_batch( - plan, - batch, - &format!("assigned agent \"{agent_id}\" no longer exists in the agent roster"), - ); - let batch_ids: Vec = batch - .iter() - .map(|&idx| plan.phases[idx].execution_id.clone()) - .collect(); - for execution_id in &batch_ids { - park_blocked_dependents(plan, execution_id); - } -} - /// Run one queued phase's pre-flight interview turn (Phase 3) — a bounded /// session driven through the *streaming* pipeline /// (`start_fresh_and_record_streaming`, no-op sink `Channel`, same trick @@ -1823,697 +1728,770 @@ pub async fn skip_phase_interview( /// /// A non-green verify verdict or any other turn-level error is unchanged /// from Phase 2: the phase is `Failed` and the run stops. -async fn execute_run( - app: &AppHandle, +/// One phase's turn, prepared by the scheduler and moved into its own +/// Tokio task. Everything is owned so the future is `'static`; the plan +/// itself is never moved into a task — only the main scheduler loop mutates +/// and persists it, which is what keeps concurrent phase bookkeeping safe. +struct PhaseTask { + index: usize, + execution_id: String, + worktree: PathBuf, + branch: String, + loc: epic::LoopLocation, + prompt: String, + agent_config: crate::config::AgentConfig, + title: String, + channel: Channel, + run_events: Option, + token_cap: u64, + phase_timeout: Duration, + run_timeout: Duration, + resume_session_id: Option, + injections: HashMap>, +} + +struct PhaseTurnOutcome { + index: usize, + execution_id: String, + worktree: PathBuf, + branch: String, + loc: epic::LoopLocation, + elapsed: Duration, + tokens: u64, + exceeded: bool, + exceeded_message: String, + result: Result, + injections: HashMap>, +} + +/// Run one phase turn in its own task: bootstrap the phase worktree, spawn +/// the provider turn under this phase's own token and wall-clock budget, and +/// return everything the scheduler needs to record the outcome. Never +/// touches `RunPlan`. +async fn run_phase_turn( state: &AppState, root: &Path, - cancel: &Arc, -) -> Result<(), AppError> { - let Some(mut plan) = runplan::load(root)? else { - return Ok(()); - }; - let _keep_awake = KeepAwake::acquire(); - let run_started = Instant::now(); - let worktree = ensure_worktree(root, &mut plan)?; - runplan::save(root, &plan)?; - let bootstrap_timeout = resolved_run_timeout(&plan).saturating_sub(run_started.elapsed()); - match bootstrap_worktree(&worktree, bootstrap_timeout, cancel).await { - Ok(BootstrapStop::Completed) => {} - Ok(BootstrapStop::Cancelled) | Ok(BootstrapStop::TimedOut) => { - let reason = if cancel.load(Ordering::SeqCst) { - "run cancelled during worktree bootstrap" - } else { - "total run wall-clock budget exceeded during worktree bootstrap" + cancel: &AtomicBool, + task: PhaseTask, +) -> PhaseTurnOutcome { + let started = Instant::now(); + let run_remaining = task.run_timeout; + let (bootstrap_timeout, bootstrap_reason) = + scheduler::phase_watchdog(task.phase_timeout, run_remaining); + let bootstrapped = bootstrap_worktree(&task.worktree, bootstrap_timeout, cancel).await; + let outcome = match bootstrapped { + Ok(BootstrapStop::Completed) => { + let token_budget = TokenBudget::new(task.token_cap); + let run_events = task.run_events.clone(); + let turn = start_fresh_and_record_streaming_in_root_with_config_resume( + state, + &task.worktree, + root, + &task.prompt, + Some(task.title.clone()), + &task.channel, + Some(&token_budget), + run_events, + Some(&task.agent_config), + // Per-phase control key: sibling phases must never collide on + // pending-question, permission, plan, or interrupt slots. + Some(&task.worktree), + true, + false, + task.resume_session_id.as_deref(), + ); + let (watchdog_timeout, watchdog_reason) = scheduler::phase_watchdog( + task.phase_timeout, + run_remaining.saturating_sub(started.elapsed()), + ); + info!( + target: "loopdeck::run_queue", + execution_id = %task.execution_id, + worktree = %task.worktree.display(), + watchdog_secs = watchdog_timeout.as_secs(), + watchdog_reason, + "run queue starting phase turn" + ); + let result = match race_with_watchdog(watchdog_timeout, turn).await { + WatchdogOutcome::Completed(result) => result, + WatchdogOutcome::TimedOut => { + let _ = terminate_session(state, &task.worktree); + let error = AppError::RunPlan(format!( + "{watchdog_reason} after {} seconds", + watchdog_timeout.as_secs() + )); + mark_turn_terminal(root, &error); + Err(error) + } }; - if let Some(phase) = plan - .phases - .iter_mut() - .find(|phase| phase.status == RunPhaseStatus::Queued) - { - phase.status = RunPhaseStatus::Killed; - phase.park_payload = Some(reason.into()); - } - plan.wall_clock_secs = run_started.elapsed().as_secs(); - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - return Ok(()); - } - Err(error) => { - if let Some(phase) = plan - .phases - .iter_mut() - .find(|phase| phase.status == RunPhaseStatus::Queued) - { - phase.status = RunPhaseStatus::Failed; - phase.park_payload = Some(error.to_string()); - } - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - return Err(error); + (result, token_budget) } + Ok(BootstrapStop::Cancelled) => ( + Err(AppError::RunPlan( + "run cancelled during worktree bootstrap".into(), + )), + TokenBudget::new(task.token_cap), + ), + Ok(BootstrapStop::TimedOut) => ( + Err(AppError::RunPlan(format!( + "{bootstrap_reason} during worktree bootstrap after {} seconds", + bootstrap_timeout.as_secs() + ))), + TokenBudget::new(task.token_cap), + ), + Err(error) => (Err(error), TokenBudget::new(task.token_cap)), + }; + let (result, token_budget) = outcome; + let mut tokens = token_budget.used(); + let mut exceeded = false; + if let Ok(response) = &result { + let observed = response + .usage + .as_ref() + .map(|usage| usage.input_tokens.saturating_add(usage.output_tokens)) + .unwrap_or_default(); + tokens = tokens.max(observed); + exceeded = token_budget.observe_total(observed); + } + PhaseTurnOutcome { + index: task.index, + execution_id: task.execution_id, + worktree: task.worktree, + branch: task.branch, + loc: task.loc, + elapsed: started.elapsed(), + tokens, + exceeded, + exceeded_message: token_budget.exceeded_message(), + injections: task.injections, + result, } +} - loop { - // Handoff consumers whose upstream artifact is not ready wait - // (`prd-agent-handoff` Phase 2). Re-checked after every turn: a - // satisfied dependency re-queues the phase; a terminal-bad upstream - // parks it. - if run_executor::recheck_waiting_phases(&mut plan)? { - // A waiting phase parked because its upstream failed still honors - // the plan's stall policy: its own dependents park transitively - // (or the whole queue under `Halt`) instead of running without it. - let parked_consumers: Vec = plan - .phases +/// Build the single-phase prompt: the existing loop prompt for exactly one +/// loop, plus this phase's handoff contract and the upstream artifacts the +/// dependency gate guarantees exist. +fn build_phase_prompt( + plan: &RunPlan, + index: usize, + loc: &epic::LoopLocation, + branch: &str, +) -> String { + let phase = &plan.phases[index]; + let phase_timeout = resolved_phase_timeout(plan); + let run_timeout = resolved_run_timeout(plan); + let mut prompt = build_combined_phase_prompt( + &[( + phase.execution_id.clone(), + loc.clone(), + phase.interview.clone(), + )], + plan.consent.draft_pr_authorized, + ResolvedBudgets { + phase_token_cap: resolved_phase_token_cap(plan), + phase_wall_clock_secs: phase_timeout.as_secs(), + run_wall_clock_secs: run_timeout.as_secs(), + }, + ); + prompt.push_str(&format!( + "\n\nThis phase runs in its own worktree on branch `{branch}`, cut from the repository's default branch. Do not merge or rebase onto another phase's branch." + )); + if let Some(topic) = phase.handoff_topic.as_deref() { + // Only phases opted into a handoff carry an artifact contract; the + // topic is fixed by the plan so a downstream consumer's reference + // never changes. + prompt.push_str(&format!( + "\n\nThis phase must write its required contract artifact to `.loopdeck/handoff-artifacts/{topic}.md`. If the general instructions above name a different artifact topic, this configured topic wins. Use YAML frontmatter with `artifact: {topic}`, `author_role` set to the assigned role name or `planner`, `phase: {}`, `type: report`, `created` as today's date, and a one-line `summary`; then include `## Summary` and concise sections matching `docs/epics/role-based-orchestration/handoff-artifact-contract.md`. Do not choose or derive a different topic.", + phase.execution_id + )); + } + let upstream: Vec<(&str, &str)> = phase + .depends_on + .iter() + .filter_map(|dependency| { + plan.phases .iter() - .filter(|phase| { - phase.status == RunPhaseStatus::Parked - && phase.park_payload.as_deref().is_some_and(|payload| { - payload.starts_with("blocked: upstream handoff phase") - }) - }) - .map(|phase| phase.execution_id.clone()) - .collect(); - for execution_id in parked_consumers { - park_blocked_dependents(&mut plan, &execution_id); - } - runplan::save(root, &plan)?; + .find(|candidate| &candidate.execution_id == dependency) + .and_then(|upstream| upstream.handoff_topic.as_deref()) + .map(|topic| (dependency.as_str(), topic)) + }) + .collect(); + if !upstream.is_empty() { + prompt.push_str( + "\n\nUpstream handoff artifacts (already published by completed phases; read them from this worktree):\n", + ); + for (dependency, topic) in upstream { + prompt.push_str(&format!( + "- phase `{dependency}` → `.loopdeck/handoffs/{topic}.md`\n" + )); } - let Some(batch) = next_queued_batch(&plan) else { - break; - }; - plan.wall_clock_secs = run_started.elapsed().as_secs(); - if run_started.elapsed() >= resolved_run_timeout(&plan) { - kill_batch(&mut plan, &batch, "total run wall-clock budget exceeded"); - run_executor::park_waiting_at_run_end( - &mut plan, - "run ended (wall-clock budget) with an unmet handoff dependency", - )?; - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - notify_run_terminal( - app, - root, - "Selasar run killed", - "Total run wall-clock budget exceeded", - ); + prompt.push_str( + "Depend only on these artifacts — never on another phase's branch code. In your own artifact, cite every section and item you used in a `## Handoff citations` block per the contract.", + ); + } + prompt +} + +/// Promote a phase into `execution.yaml`'s `current` pointer, but only when +/// no sibling owns it. `run-plan.yaml` is the authoritative per-phase record; +/// `current` is a UI-focus pointer and must never make two parallel turns +/// fight over one slot. +fn begin_phase_execution( + root: &Path, + execution_id: &str, + loc: &epic::LoopLocation, +) -> Result<(), AppError> { + let loaded = execution::load(root)?; + if let Some(current) = loaded.state.current.as_ref() { + if current.id != execution_id { return Ok(()); } - if cancel.load(Ordering::SeqCst) { - kill_batch(&mut plan, &batch, "run cancelled"); - run_executor::park_waiting_at_run_end( - &mut plan, - "run cancelled with an unmet handoff dependency", - )?; - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - notify_run_terminal(app, root, "Selasar run killed", "Run was cancelled"); - return Ok(()); + // Resume of the phase that already owns the pointer: nothing to do. + return Ok(()); + } + let promoted = loaded.state.promote_loop_into_current( + execution_id, + &loc.title, + LoopOrigin { + epic: loc.epic.clone(), + prd: loc.prd.clone(), + phase: loc.phase.clone(), + }, + Utc::now(), + )?; + execution::save(root, &promoted, loaded.state.revision)?; + Ok(()) +} + +/// Terminal bookkeeping for a non-green phase. Only touches `current` when +/// this phase owns it; a parallel sibling's focus pointer is left alone. +fn end_phase_execution(root: &Path, execution_id: &str, reason: String) -> Result<(), AppError> { + let loaded = execution::load(root)?; + let owns_current = loaded + .state + .current + .as_ref() + .map(|current| current.id == execution_id) + .unwrap_or(false); + if !owns_current { + return Ok(()); + } + let abandoned = loaded.state.abandon_current(reason, Utc::now(), false)?; + execution::save(root, &abandoned, loaded.state.revision)?; + Ok(()) +} + +/// Green-phase bookkeeping: relink or complete the current loop and attach +/// the delivery evidence, but only when this phase owns `current`. +#[allow(clippy::too_many_arguments)] +fn complete_phase_execution( + root: &Path, + execution_id: &str, + final_id: &str, + replacement: Option<&epic::LoopLocation>, + branch: &str, + worktree: &Path, + url: &str, + rubric: Option, +) -> Result<(), AppError> { + let loaded = execution::load(root)?; + let owns_current = loaded + .state + .current + .as_ref() + .map(|current| current.id == execution_id) + .unwrap_or(false); + if !owns_current { + return Ok(()); + } + let revision = loaded.state.revision; + let mut state = loaded.state; + if let Some(current) = state.current.as_mut() { + current.id = final_id.to_string(); + if let Some(location) = replacement { + current.title = location.title.clone(); + current.origin = LoopOrigin { + epic: location.epic.clone(), + prd: location.prd.clone(), + phase: location.phase.clone(), + }; } + current.delivery = Some(delivery::DeliveryLinks { + branch: Some(branch.to_string()), + commit: git::head_commit(worktree), + pr_url: Some(url.to_string()), + pr_provider: url.contains("github.com").then(|| "github".to_string()), + rubric, + }); + } + let completed = state.complete_current(Utc::now(), None, false)?; + execution::save(root, &completed, revision)?; + Ok(()) +} + +/// Park one phase with a reason, fail its not-yet-started dependents, and +/// persist. Independent in-flight phases keep running (approved gated-start +/// answer). +fn park_phase( + root: &Path, + plan: &mut RunPlan, + index: usize, + reason: String, +) -> Result<(), AppError> { + let execution_id = plan.phases[index].execution_id.clone(); + plan.phases[index].status = RunPhaseStatus::Parked; + plan.phases[index].park_payload = Some(reason.clone()); + plan.environment.worktree_kept = true; + scheduler::fail_dependents(plan, &execution_id, &reason); + runplan::save(root, plan)?; + end_phase_execution(root, &execution_id, reason)?; + Ok(()) +} + +/// Apply one finished phase turn to the plan. Runs **only** in the scheduler +/// loop, which serializes every durable write; phase tasks themselves never +/// touch the plan. +fn apply_phase_outcome( + app: &AppHandle, + state: &AppState, + root: &Path, + plan: &mut RunPlan, + run_started: Instant, + outcome: PhaseTurnOutcome, +) -> Result<(), AppError> { + let index = outcome.index; + let execution_id = outcome.execution_id.clone(); + plan.phases[index].wall_clock_secs = outcome.elapsed.as_secs(); + plan.phases[index].token_usage = outcome.tokens; + plan.wall_clock_secs = run_started.elapsed().as_secs(); + + if outcome.exceeded { + // Per-phase budget kill: end only this phase, never the run. + let reason = outcome.exceeded_message.clone(); + plan.phases[index].status = RunPhaseStatus::Killed; + plan.phases[index].park_payload = Some(reason.clone()); + plan.environment.worktree_kept = true; + scheduler::fail_dependents(plan, &execution_id, &reason); + runplan::save(root, plan)?; + let _ = terminate_session(state, &outcome.worktree); + end_phase_execution(root, &execution_id, reason)?; + notify_run_terminal( + app, + root, + "Selasar phase killed", + "A phase exceeded its token budget — independent phases continue", + ); + return Ok(()); + } - let mut locs: Vec<(String, epic::LoopLocation, Vec)> = - Vec::with_capacity(batch.len()); - for &idx in &batch { - let execution_id = plan.phases[idx].execution_id.clone(); - match epic::find_loop_by_id(root, &execution_id) { - Some(loc) => locs.push((execution_id, loc, plan.phases[idx].interview.clone())), - None => { - plan.phases[idx].status = RunPhaseStatus::Failed; - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - return Err(AppError::RunPlan(format!( - "queued phase \"{execution_id}\" no longer exists in docs/epics/" - ))); + match outcome.result { + Ok(response) => { + if !response.session_id.is_empty() { + plan.phases[index].session_id = Some(response.session_id.clone()); + } + let verdict = extract_verdict(&response.result); + if verdict == Some(RunVerdict::Pass) { + let topic = plan.phases[index].handoff_topic.clone(); + if let Err(error) = + emit_phase_artifact(root, &outcome.worktree, &execution_id, topic.as_deref()) + { + return park_phase( + root, + plan, + index, + format!("handoff artifact emission failed: {error}"), + ); } + record_handoff_citations( + root, + plan, + &[( + execution_id.clone(), + outcome.loc.clone(), + plan.phases[index].interview.clone(), + )], + &outcome.injections, + &response.result, + ); } - } - - // Per-phase staffing (`prd-role-foundations` Phase 4): resolve the - // batch's shared roster entry — `next_queued_batch` guarantees every - // phase in the batch carries the same `assigned_agent_id`. A missing - // roster entry is a config error the user can repair, so the batch - // parks (recoverable via requeue) rather than failing the run; - // dependents park per the stall policy, independent phases continue. - let assigned_agent_id = plan.phases[batch[0]].assigned_agent_id.clone(); - let agent_config = match assigned_agent_id.as_deref() { - Some(agent_id) => match resolve_agent_config_by_id(state, agent_id) { - Ok(config) => config, - Err(error) => { + if verdict == Some(RunVerdict::Pass) && plan.consent.draft_pr_authorized { + // Retained before any delivery decision so a recoverable + // record can carry the rubric evidence (retry-recovery). + let rubric = delivery::extract_rubric_result(&response.result, chrono::Utc::now()); + // The PR link: the turn's own report first; then — the + // crash-retry path — an already-open PR for this phase branch. + let url = extract_draft_pr_url(&response.result) + .or_else(|| git::open_pr_for_branch(&outcome.worktree, &outcome.branch)); + let Some(url) = url else { let reason = - format!("assigned agent \"{agent_id}\" is not on the roster: {error}"); - park_missing_agent(&mut plan, &batch, agent_id); - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - notify_run_terminal( - app, + "green verification passed, but no unattended draft PR URL was recorded"; + // Recoverable delivery record (Phase 4, `retry-recovery`): + // how far delivery got in Git and why it stopped, so one + // idempotent retry command can resume from that stage. No + // automatic retry. + if let Err(error) = delivery_retry::save( root, - "Selasar run parked", - &format!("Phase parked: {reason}"), + &delivery_retry::DeliveryRetryRecord { + execution_ids: vec![execution_id.clone()], + branch: outcome.branch.clone(), + worktree: outcome.worktree.clone(), + stage: delivery_retry::detect_stage(&outcome.worktree), + reason: reason.to_string(), + rubric: rubric.clone(), + pr_title: outcome.loc.title.clone(), + recorded_at: chrono::Utc::now(), + }, + ) { + warn!("could not persist the recoverable delivery record: {error}"); + } + return park_phase(root, plan, index, reason.to_string()); + }; + + // Delivery gates, evaluated fresh from this turn's own + // verifier report — never a stale persisted verdict. + // Idempotent finish is exempt: when the checklist item is + // already checked and an open PR claims the branch, this is a + // retry completing bookkeeping, not a new delivery. + let already_delivered = epic::loop_checked(root, &execution_id) == Some(true); + if !already_delivered { + let branch_matches = git::current_branch(&outcome.worktree) + .map(|live| live == outcome.branch) + .unwrap_or(false); + let checked = epic::loop_checked(root, &execution_id); + let blocks = delivery::evaluate_delivery_gates( + checked.map(|checked| !checked), + branch_matches, + checked.is_some(), + rubric.as_ref(), ); - continue; + if !blocks.is_empty() { + let reason = format!( + "delivery gate blocked: {}", + blocks + .iter() + .map(std::string::ToString::to_string) + .collect::>() + .join("; ") + ); + return park_phase(root, plan, index, reason); + } } - }, - None => resolve_agent_config(state)?, - }; - // Mark running in both the plan and execution.yaml before spawning - // the turn, so a crash mid-turn leaves truthful on-disk state - // (reconciled to Interrupted/back-to-current by the next read) rather - // than a phase that silently never started. - for &idx in &batch { - plan.phases[idx].status = RunPhaseStatus::Running; - } - runplan::save(root, &plan)?; - - // `execution.yaml` only tracks a single `current` loop — the first - // phase in the batch stands in as that UI-focus pointer while the - // combined turn covers all of them; `run-plan.yaml`'s per-phase - // `status` (below) remains the accurate per-loop record. - let (lead_execution_id, lead_loc, _) = &locs[0]; - let loaded = execution::load(root)?; - let promoted = loaded.state.promote_loop_into_current( - lead_execution_id, - &lead_loc.title, - LoopOrigin { - epic: lead_loc.epic.clone(), - prd: lead_loc.prd.clone(), - phase: lead_loc.phase.clone(), - }, - chrono::Utc::now(), - )?; - execution::save(root, &promoted, loaded.state.revision)?; - - let batch_len = batch.len() as u64; - let phase_token_cap = resolved_phase_token_cap(&plan).saturating_mul(batch_len); - let phase_timeout = resolved_phase_timeout(&plan) * batch.len() as u32; - let run_timeout = resolved_run_timeout(&plan); - let mut prompt = build_combined_phase_prompt( - &locs, - plan.consent.draft_pr_authorized, - ResolvedBudgets { - phase_token_cap, - phase_wall_clock_secs: phase_timeout.as_secs(), - run_wall_clock_secs: run_timeout.as_secs(), - }, - ); - for (execution_id, _, _) in &locs { - let topic = plan - .phases - .iter() - .find(|phase| phase.execution_id == *execution_id) - .and_then(|phase| phase.handoff_topic.as_deref()); - // Only phases opted into a handoff carry an artifact contract; the - // topic is fixed by the plan so a downstream consumer's reference - // never changes. - if let Some(topic) = topic { - prompt.push_str(&format!( - "\n\nFor phase `{execution_id}`, write its required contract artifact to `.loopdeck/handoff-artifacts/{topic}.md`. Use YAML frontmatter with `artifact: {topic}`, `author_role` set to the assigned role name or `planner`, `phase: {execution_id}`, `type: report`, `created` as today's date, and a one-line `summary`; then include `## Summary` and concise sections matching `docs/epics/role-based-orchestration/handoff-artifact-contract.md`. Do not choose or derive a different topic." - )); - } - } - // Downstream injection (`prd-agent-handoff` Phase 2): a consumer batch - // started a new turn precisely so its ancestor's artifact already - // exists in the store. Inline every such artifact, bounded to 8 KiB, - // and name its store path for citation. A missing/oversized artifact - // parks the batch rather than improvising a replacement. - let injections = match handoff_injections(root, &plan, &locs) { - Ok(injections) => injections, - Err(error) => { - let reason = format!("handoff artifact injection failed: {error}"); - park_batch(&mut plan, &batch, &reason); - plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); + let replacement_id = if epic::find_loop_by_id(root, &execution_id).is_some() { + None + } else { + epic::find_replacement_loop_id(root, &outcome.loc) + }; + let final_id = replacement_id + .clone() + .unwrap_or_else(|| execution_id.clone()); + let replacement_loc = replacement_id + .as_deref() + .and_then(|id| epic::find_loop_by_id(root, id)); + let final_status = if epic::find_loop_by_id(root, &final_id).is_some() { + RunPhaseStatus::Completed + } else { + RunPhaseStatus::Delivered + }; + plan.phases[index].execution_id = final_id.clone(); + plan.phases[index].status = final_status; + plan.phases[index].park_payload = Some( + if final_status == RunPhaseStatus::Completed { + format!("draft PR: {url}") + } else { + format!( + "draft PR: {url}\ndelivered — PRD link needs repair before this can appear in progress" + ) + }, + ); + if replacement_id.is_some() { + // Keep downstream edges pointing at the surviving id. + for phase in &mut plan.phases { + for dependency in &mut phase.depends_on { + if dependency == &execution_id { + *dependency = final_id.clone(); + } + } + } } - runplan::save(root, &plan)?; - let loaded = execution::load(root)?; - let abandoned = loaded - .state - .abandon_current(reason, chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - continue; - } - }; - for (execution_id, _, _) in &locs { - if let Some(artifacts) = injections.get(execution_id) { - prompt.push_str(&handoff::render_injection(execution_id, artifacts)); + // The PR URL is the delivery boundary: only now may the + // checklist declare the loop done. Idempotent. + epic::complete_prd_loop( + root, + &outcome.loc.epic, + &outcome.loc.prd, + &outcome.loc.title, + )?; + runplan::save(root, plan)?; + complete_phase_execution( + root, + &execution_id, + &final_id, + replacement_loc.as_ref(), + &outcome.branch, + &outcome.worktree, + &url, + rubric, + )?; + // Clean handoff: this branch, PR, and worktree stay for + // review; the next phase/run starts from the default branch. + record_handoff(root, &outcome.branch, &outcome.worktree, &url); + return Ok(()); } + + let reason = match verdict { + Some(RunVerdict::Pass) => "green verification passed, but queue-time consent for an unattended draft PR is required".to_string(), + Some(RunVerdict::Warn) => "verify verdict: WARN".to_string(), + Some(RunVerdict::Block) => "verify verdict: BLOCK".to_string(), + _ => "no verify verdict found in the turn's final response".to_string(), + }; + park_phase(root, plan, index, reason) } - let event_app = app.clone(); - let event_project_path = root.to_string_lossy().into_owned(); - let event_execution_id = locs - .iter() - .map(|(id, _, _)| id.as_str()) - .collect::>() - .join(","); - let channel_execution_id = event_execution_id.clone(); - let run_project_path = event_project_path.clone(); - let run_execution_id = channel_execution_id.clone(); - let channel: Channel = Channel::new(move |event| { - let event = match event.deserialize::() { - Ok(event) => event, - Err(error) => { - warn!("failed to decode unattended run activity: {error}"); - return Ok(()); - } + Err(AppError::TurnParked { + detail, + parked_questions_json, + }) => { + // Store the structured question payload when available, so the + // morning report can reconstruct AskUserQuestionCard components. + let payload = match parked_questions_json { + Some(ref json) => format!("__QUESTIONS__{json}__END__ {detail}"), + None => detail, }; - if let Err(error) = event_app.emit( - RUN_QUEUE_EVENT, - RunQueueEvent { - project_path: event_project_path.clone(), - execution_id: channel_execution_id.clone(), - event, - }, - ) { - warn!("failed to emit unattended run activity: {error}"); + park_phase(root, plan, index, payload) + } + Err(error) => { + let message = error.to_string(); + let killed_by_budget = message.contains("wall-clock budget exceeded") + || message.contains("token budget exceeded"); + let status = if killed_by_budget { + RunPhaseStatus::Killed + } else { + RunPhaseStatus::Failed + }; + plan.phases[index].status = status; + plan.phases[index].park_payload = Some(message.clone()); + plan.environment.worktree_kept = true; + // A non-green phase fails its dependents; independent in-flight + // phases continue (approved gated-start answer). + scheduler::fail_dependents(plan, &execution_id, &message); + runplan::save(root, plan)?; + if message.contains("token budget exceeded") { + let _ = terminate_session(state, &outcome.worktree); } - Ok(()) - }); - let run_events = Some(RunEventChannel::new( - plan.id.clone(), - normalized_run_channel(app.clone(), run_project_path, run_execution_id), - )); - let token_budget = TokenBudget::new(phase_token_cap); - let phase_started = Instant::now(); - let run_remaining = run_timeout.saturating_sub(run_started.elapsed()); - let (watchdog_timeout, watchdog_reason) = if run_remaining <= phase_timeout { - (run_remaining, "total run wall-clock budget exceeded") - } else { - (phase_timeout, "phase wall-clock budget exceeded") - }; - let turn_title = if locs.len() == 1 { - lead_loc.title.clone() - } else { - format!("{} queued loops", locs.len()) - }; - // Profile-pinned turn (`prd-role-foundations` Phase 4): when the - // batch has an assigned agent, its resolved config — charter included - // — replaces the default-config spawn, so each phase group runs under - // its own role. `None` falls back to the default-agent resolution - // inside the wrapper, unchanged from Phase 2. - let turn = start_fresh_and_record_streaming_in_root_with_config( - state, - &worktree, - root, - &prompt, - Some(turn_title), - &channel, - Some(&token_budget), - run_events, - // The batch's staffing resolved above — the assigned roster - // entry's connection settings *and* charter, or the default - // agent config when the phase is unassigned. - Some(&agent_config), - None, - true, - false, - ); - info!( - target: "loopdeck::run_queue", - run_id = %plan.id, - execution_ids = %event_execution_id, - worktree = %worktree.display(), - watchdog_secs = watchdog_timeout.as_secs(), - watchdog_reason, - "run queue starting Codex turn" - ); - let outcome = match race_with_watchdog(watchdog_timeout, turn).await { - WatchdogOutcome::Completed(result) => result, - WatchdogOutcome::TimedOut => { - // `timeout` (inside `race_with_watchdog`) drops the turn future - // first, releasing its session lock. Removing the last cached - // Arc then invokes the harness Drop path: graceful EOF, bounded - // wait, and SIGKILL fallback. - warn!( - target: "loopdeck::run_queue", - run_id = %plan.id, - execution_ids = %event_execution_id, - elapsed_secs = phase_started.elapsed().as_secs(), - watchdog_secs = watchdog_timeout.as_secs(), - watchdog_reason, - "run queue watchdog timed out Codex turn" + end_phase_execution(root, &execution_id, message.clone())?; + if killed_by_budget { + notify_run_terminal( + app, + root, + "Selasar phase killed", + "A phase hit its budget — independent phases continue", ); - terminate_session(state, &worktree)?; - let error = AppError::RunPlan(format!( - "{watchdog_reason} after {} seconds", - watchdog_timeout.as_secs() - )); - mark_turn_terminal(root, &error); - Err(error) } - }; - let per_phase_secs = phase_started.elapsed().as_secs() / batch_len.max(1); - for &idx in &batch { - plan.phases[idx].wall_clock_secs = per_phase_secs; + Ok(()) + } + } +} + +/// Mark every unfinished phase terminal with one reason. Used by the run-level +/// cancel and total-wall-clock paths, which stop the whole run by design. +fn stop_unfinished_phases(plan: &mut RunPlan, reason: &str) { + for phase in &mut plan.phases { + if scheduler::is_active(phase.status) { + phase.status = RunPhaseStatus::Killed; + phase.park_payload = Some(reason.to_string()); } + } +} + +/// The run executor. One turn per run-plan phase, with phases whose +/// `depends_on` edges are satisfied running concurrently in separate +/// worktrees up to `RunPlan::max_concurrency`. +async fn execute_run( + app: &AppHandle, + state: &AppState, + root: &Path, + cancel: &Arc, +) -> Result<(), AppError> { + let Some(mut plan) = runplan::load(root)? else { + return Ok(()); + }; + let _keep_awake = KeepAwake::acquire(); + let run_started = Instant::now(); + ignore_managed_runs_dir(root); + // A cycle would leave every member Waiting forever; reject before any + // worktree or provider process exists. + scheduler::validate_dependency_graph(&plan.phases)?; + // Legacy plans start all-Queued; compute the initial gate now. Then + // release any phase whose upstream already completed in a prior attempt. + scheduler::mark_waiting(&mut plan); + scheduler::reconcile_gates(&mut plan); + runplan::save(root, &plan)?; + + let mut tasks: tokio::task::JoinSet = tokio::task::JoinSet::new(); + let mut running: HashMap = HashMap::new(); + + loop { plan.wall_clock_secs = run_started.elapsed().as_secs(); - set_batch_token_usage(&mut plan, &batch, token_budget.used(), batch_len); - match outcome { - Ok(response) => { - let tokens = response - .usage - .as_ref() - .map(|usage| usage.input_tokens.saturating_add(usage.output_tokens)) - .unwrap_or_default(); - let exceeded = token_budget.observe_total(tokens); - set_batch_token_usage(&mut plan, &batch, token_budget.used(), batch_len); - if exceeded { - kill_batch(&mut plan, &batch, &token_budget.exceeded_message()); - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - let loaded = execution::load(root)?; - let abandoned = loaded.state.abandon_current( - "phase token budget exceeded", - chrono::Utc::now(), - false, - )?; - execution::save(root, &abandoned, loaded.state.revision)?; - notify_run_terminal( - app, - root, - "Selasar run killed", - "Phase token budget exceeded", - ); - return Ok(()); - } - let verdict = extract_verdict(&response.result); - if verdict == Some(RunVerdict::Pass) { - if let Err(error) = emit_phase_artifacts(root, &worktree, &plan, &locs) { - let reason = format!("handoff artifact emission failed: {error}"); - park_batch(&mut plan, &batch, &reason); - plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); - } + if cancel.load(Ordering::SeqCst) { + for worktree in running.values() { + let _ = terminate_session(state, worktree); + let _ = fire_interrupt(state, worktree); + } + stop_unfinished_phases(&mut plan, "run cancelled"); + plan.environment.worktree_kept = true; + runplan::save(root, &plan)?; + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + // Aborting a task after it inserted its session leaves the child + // in the cache map; evict again so no orphan process survives. + for worktree in running.values() { + let _ = terminate_session(state, worktree); + } + notify_run_terminal(app, root, "Selasar run killed", "Run was cancelled"); + return Ok(()); + } + + if run_started.elapsed() >= resolved_run_timeout(&plan) { + for worktree in running.values() { + let _ = terminate_session(state, worktree); + } + stop_unfinished_phases(&mut plan, "total run wall-clock budget exceeded"); + plan.environment.worktree_kept = true; + runplan::save(root, &plan)?; + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + for worktree in running.values() { + let _ = terminate_session(state, worktree); + } + notify_run_terminal( + app, + root, + "Selasar run killed", + "Total run wall-clock budget exceeded", + ); + return Ok(()); + } + + // Open the gate for any phase whose upstreams all landed green, and + // fail dependents of upstreams that did not. + let (released, failed) = scheduler::reconcile_gates(&mut plan); + if released + failed > 0 { + runplan::save(root, &plan)?; + } + + // Fill spare concurrency slots with ready phases. + let max = plan.max_concurrency.max(1) as usize; + let slots = max.saturating_sub(running.len()); + if slots > 0 { + let in_flight: HashSet = running + .keys() + .map(|&index| plan.phases[index].execution_id.clone()) + .collect(); + for index in scheduler::select_ready(&plan, &in_flight, slots) { + match prepare_phase(app, state, root, &mut plan, index, run_started) { + Ok(task) => { + let task_index = task.index; + let worktree = task.worktree.clone(); + let app_handle = app.clone(); + let run_root = root.to_path_buf(); + let cancel_flag = Arc::clone(cancel); + plan.phases[index].status = RunPhaseStatus::Running; + plan.phases[index].park_payload = None; runplan::save(root, &plan)?; - let loaded = execution::load(root)?; - let abandoned = - loaded - .state - .abandon_current(reason, chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - continue; + running.insert(task_index, worktree); + tasks.spawn(async move { + let state = app_handle.state::(); + run_phase_turn(&state, &run_root, &cancel_flag, task).await + }); } - // Report attribution (`prd-agent-handoff` Phase 3): - // record each consumed upstream artifact with its - // from-role, to-role, path, and coverage-only citation - // result. Best-effort: a ledger write never fails delivery. - record_handoff_citations(root, &plan, &locs, &injections, &response.result); - } - if verdict == Some(RunVerdict::Pass) && plan.consent.draft_pr_authorized { - // Retained before any delivery decision so a recoverable - // record can carry the rubric evidence (retry-recovery). - let rubric = - delivery::extract_rubric_result(&response.result, chrono::Utc::now()); - // The PR link: the turn's own report first; then — the - // crash-retry path — an already-open PR for this run's - // branch (PR creation succeeded on a previous attempt - // before the bookkeeping ran). - let url = extract_draft_pr_url(&response.result).or_else(|| { - plan.environment - .worktree_branch - .as_deref() - .and_then(|branch| git::open_pr_for_branch(&worktree, branch)) - }); - let Some(url) = url else { - let reason = "green verification passed, but no unattended draft PR URL was recorded"; - // Recoverable delivery record (Phase 4, - // `retry-recovery`): how far the delivery got in Git - // and why it stopped, so one idempotent retry command - // can resume from that stage. No automatic retry. - if let Err(error) = delivery_retry::save( - root, - &delivery_retry::DeliveryRetryRecord { - execution_ids: locs - .iter() - .map(|(execution_id, _, _)| execution_id.clone()) - .collect(), - branch: plan - .environment - .worktree_branch - .clone() - .unwrap_or_default(), - worktree: worktree.clone(), - stage: delivery_retry::detect_stage(&worktree), - reason: reason.to_string(), - rubric: rubric.clone(), - pr_title: locs - .first() - .map(|(_, location, _)| location.title.clone()) - .unwrap_or_default(), - recorded_at: chrono::Utc::now(), - }, - ) { - warn!("could not persist the recoverable delivery record: {error}"); - } - park_batch(&mut plan, &batch, reason); + Err(error) => { + // Preparation failure is local to this phase. A missing + // roster entry parks (recoverable via requeue); any + // other preparation error fails the phase. Either way + // its dependents fail and independent phases continue. + let (status, reason) = match error { + PhasePrepareError::Parked(reason) => (RunPhaseStatus::Parked, reason), + PhasePrepareError::Failed(reason) => ( + RunPhaseStatus::Failed, + format!("phase preparation failed: {reason}"), + ), + }; + let execution_id = plan.phases[index].execution_id.clone(); + plan.phases[index].status = status; + plan.phases[index].park_payload = Some(reason.clone()); plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); - } + scheduler::fail_dependents(&mut plan, &execution_id, &reason); runplan::save(root, &plan)?; - let loaded = execution::load(root)?; - let abandoned = - loaded - .state - .abandon_current(reason, chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - continue; - }; - - // Delivery gates (Phase 3), evaluated fresh from this - // turn's own verifier report — never a stale persisted - // verdict. Idempotent finish is exempt: when every - // checklist item is already checked and an open PR claims - // the branch, this is a retry completing bookkeeping a - // previous attempt already delivered, not a new delivery. - let already_delivered = locs.iter().all(|(execution_id, _, _)| { - epic::loop_checked(root, execution_id) == Some(true) - }); - if !already_delivered { - let recorded_branch = - plan.environment.worktree_branch.clone().unwrap_or_default(); - let branch_matches = git::current_branch(&worktree) - .map(|live| live == recorded_branch) - .unwrap_or(false); - let lead_checked = epic::loop_checked(root, &locs[0].0); - let blocks = delivery::evaluate_delivery_gates( - lead_checked.map(|checked| !checked), - branch_matches, - lead_checked.is_some(), - rubric.as_ref(), - ); - if !blocks.is_empty() { - let reason = format!( - "delivery gate blocked: {}", - blocks - .iter() - .map(std::string::ToString::to_string) - .collect::>() - .join("; ") - ); - park_batch(&mut plan, &batch, &reason); - plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); - } - runplan::save(root, &plan)?; - let loaded = execution::load(root)?; - let abandoned = - loaded - .state - .abandon_current(reason, chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - continue; - } - } - let mut automatic_relinks = Vec::new(); - for (&idx, (execution_id, original, _)) in batch.iter().zip(&locs) { - let replacement = if epic::find_loop_by_id(root, execution_id).is_some() { - None - } else { - epic::find_replacement_loop_id(root, original) - }; - if let Some(replacement_id) = replacement { - automatic_relinks.push(( - execution_id.clone(), - replacement_id.clone(), - original.clone(), - )); - plan.phases[idx].execution_id = replacement_id; - } - plan.phases[idx].status = if epic::find_loop_by_id( - root, - &plan.phases[idx].execution_id, - ) - .is_some() - { - RunPhaseStatus::Completed - } else { - RunPhaseStatus::Delivered - }; - plan.phases[idx].park_payload = Some( - if plan.phases[idx].status == RunPhaseStatus::Completed { - format!("draft PR: {url}") - } else { - format!("draft PR: {url}\ndelivered — PRD link needs repair before this can appear in progress") - }, - ); - } - // The PR URL is the delivery boundary: only now may the - // checklist declare the loop done. Unlike the manual - // toggle, this operation is idempotent and can never - // reopen an already-checked item on a retry. - for (_, location, _) in &locs { - epic::complete_prd_loop( - root, - &location.epic, - &location.prd, - &location.title, - )?; - } - for (old_id, new_id, _) in &automatic_relinks { - for phase in &mut plan.phases { - for dependency in &mut phase.depends_on { - if dependency == old_id { - *dependency = new_id.clone(); - } - } - } } - runplan::save(root, &plan)?; - - let loaded = execution::load(root)?; - let revision = loaded.state.revision; - let mut execution_state = loaded.state; - if let Some(current) = execution_state.current.as_mut() { - if let Some((_, replacement_id, location)) = automatic_relinks - .iter() - .find(|(old_id, _, _)| *old_id == current.id) - { - current.id = replacement_id.clone(); - current.title = location.title.clone(); - current.origin = LoopOrigin { - epic: location.epic.clone(), - prd: location.prd.clone(), - phase: location.phase.clone(), - }; - } - // The terminal delivery record (Phase 3, loop - // `complete-after-pr`): branch, commit, PR, and the - // rubric result that gated it — persisted only now - // that PR creation has actually succeeded. - // `complete_current` carries it into the history - // entry; the checklist (already updated above) - // remains the only completion authority. - current.delivery = Some(delivery::DeliveryLinks { - branch: plan.environment.worktree_branch.clone(), - commit: git::head_commit(&worktree), - pr_url: Some(url.clone()), - pr_provider: url.contains("github.com").then(|| "github".to_string()), - rubric: rubric.clone(), - }); - } - let completed = - execution_state.complete_current(chrono::Utc::now(), None, false)?; - execution::save(root, &completed, revision)?; - // Clean handoff (Phase 4): the record that the next run - // starts fresh from the default branch while this branch, - // PR, and worktree stay retained for review. - record_handoff(root, &plan, &worktree, &url); - } else { - let reason = match verdict { - Some(RunVerdict::Pass) => "green verification passed, but queue-time consent for an unattended draft PR is required".to_string(), - Some(RunVerdict::Warn) => "verify verdict: WARN".to_string(), - Some(RunVerdict::Block) => "verify verdict: BLOCK".to_string(), - _ => "no verify verdict found in the turn's final response".to_string(), - }; - park_batch(&mut plan, &batch, &reason); - plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); - } - runplan::save(root, &plan)?; - - let loaded = execution::load(root)?; - let abandoned = - loaded - .state - .abandon_current(reason, chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - continue; } } - Err(AppError::TurnParked { - detail, - parked_questions_json, - }) => { - // Store the structured question payload when available, so the - // morning report can reconstruct AskUserQuestionCard components. - let payload = match parked_questions_json { - Some(ref json) => format!("__QUESTIONS__{json}__END__ {detail}"), - None => detail.clone(), - }; - park_batch(&mut plan, &batch, &payload); - plan.environment.worktree_kept = true; - for (execution_id, _, _) in &locs { - park_blocked_dependents(&mut plan, execution_id); - } - runplan::save(root, &plan)?; + } - let loaded = execution::load(root)?; - let abandoned = loaded.state.abandon_current( - format!("phase parked: {detail}"), - chrono::Utc::now(), - false, - )?; - execution::save(root, &abandoned, loaded.state.revision)?; - // No early return: under `ContinueIndependent` the loop - // naturally advances to try the next still-`Queued` phase; - // under `Halt`, `phases_blocked_by_park` already parked every - // remaining phase, so the top-of-iteration status check skips - // them and the run ends here regardless. + if running.is_empty() { + if !scheduler::has_active_phases(&plan) { + break; } - Err(e) => { - let killed_by_budget = e.to_string().contains("wall-clock budget exceeded") - || e.to_string().contains("token budget exceeded"); - let status = if killed_by_budget { - RunPhaseStatus::Killed - } else { - RunPhaseStatus::Failed - }; - for &idx in &batch { - plan.phases[idx].status = status; - plan.phases[idx].park_payload = Some(e.to_string()); - } - plan.environment.worktree_kept = true; - runplan::save(root, &plan)?; - - if e.to_string().contains("token budget exceeded") { - terminate_session(state, &worktree)?; + // Active phases but nothing runnable and nothing in flight: the + // gate can never open (for example every upstream was parked + // before the gate ran). Fail them instead of spinning. + let stalled: Vec = plan + .phases + .iter() + .filter(|phase| scheduler::is_active(phase.status)) + .map(|phase| phase.execution_id.clone()) + .collect(); + for phase in &mut plan.phases { + if scheduler::is_active(phase.status) { + phase.status = RunPhaseStatus::Failed; + phase.park_payload = Some(format!( + "scheduler stalled: no runnable phase (dependency gate blocked: {})", + stalled.join(", ") + )); } + } + plan.environment.worktree_kept = true; + runplan::save(root, &plan)?; + break; + } - let loaded = execution::load(root)?; - let abandoned = - loaded - .state - .abandon_current(e.to_string(), chrono::Utc::now(), false)?; - execution::save(root, &abandoned, loaded.state.revision)?; - if killed_by_budget { - notify_run_terminal( - app, - root, - "Selasar run killed", - "Phase wall-clock budget exceeded", - ); + // Wait for one phase to finish, but wake periodically so cancel and + // the total-run cap stay responsive while long turns run. + tokio::select! { + joined = tasks.join_next() => { + match joined { + Some(Ok(outcome)) => { + running.remove(&outcome.index); + if let Err(error) = + apply_phase_outcome(app, state, root, &mut plan, run_started, outcome) + { + // Bookkeeping failure is fatal for the run, but + // sibling turns must not be left orphaned. + for worktree in running.values() { + let _ = terminate_session(state, worktree); + } + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + return Err(error); + } + } + Some(Err(join_error)) => { + // A panicked turn task is unrecoverable for the phases + // it owned; their statuses cannot be trusted. + let reason = format!("phase task failed unexpectedly: {join_error}"); + for (index, _) in running.drain() { + plan.phases[index].status = RunPhaseStatus::Failed; + plan.phases[index].park_payload = Some(reason.clone()); + let execution_id = plan.phases[index].execution_id.clone(); + scheduler::fail_dependents(&mut plan, &execution_id, &reason); + } + plan.environment.worktree_kept = true; + runplan::save(root, &plan)?; + } + None => {} } - run_executor::park_waiting_at_run_end( - &mut plan, - "run ended with an unmet handoff dependency", - )?; - runplan::save(root, &plan)?; - return if killed_by_budget { Ok(()) } else { Err(e) }; } + _ = tokio::time::sleep(Duration::from_millis(250)) => {} } } @@ -2534,7 +2512,7 @@ async fn execute_run( let any_parked = plan .phases .iter() - .any(|p| p.status == RunPhaseStatus::Parked); + .any(|p| matches!(p.status, RunPhaseStatus::Parked | RunPhaseStatus::Waiting)); if all_completed { notify_run_terminal( @@ -2555,15 +2533,126 @@ async fn execute_run( Ok(()) } -/// Cleanup policy for the run's isolated worktree, applied once every phase -/// has been attempted. `prd-unattended-ship.md` said "prune on success"; -/// `prd-verified-delivery-reconciliation` Phase 4 (`clean-handoff`) -/// supersedes that for delivered runs: a fully-delivered worktree is -/// **retained** under `.loopdeck/runs/` with its branch + unmerged PR for -/// review, and the *next* run gets a fresh worktree cut from the default -/// branch instead. Non-delivered endings (parked/failed/killed) keep + flag, -/// exactly as before. Split out of `execute_run` so the decision is testable -/// against a real worktree without needing a live `claude_session` turn. +/// Why a phase could not be prepared. A missing roster entry is recoverable +/// (park, user fixes the roster and retries); every other preparation error +/// is a hard failure for that phase only. +enum PhasePrepareError { + Failed(String), + Parked(String), +} + +/// Prepare one ready phase: create or recover its worktree, resolve staffing, +/// seed upstream artifacts, build the single-phase prompt, and promote it in +/// `execution.yaml` when the focus pointer is free. Returns the task to spawn. +/// Persists the phase's worktree identity before the turn starts, so a crash +/// leaves truthful on-disk state. +fn prepare_phase( + app: &AppHandle, + state: &AppState, + root: &Path, + plan: &mut RunPlan, + index: usize, + run_started: Instant, +) -> Result { + let execution_id = plan.phases[index].execution_id.clone(); + let Some(loc) = epic::find_loop_by_id(root, &execution_id) else { + return Err(PhasePrepareError::Failed(format!( + "queued phase \"{execution_id}\" no longer exists in docs/epics/" + ))); + }; + + let agent_config = match plan.phases[index].assigned_agent_id.as_deref() { + Some(agent_id) => resolve_agent_config_by_id(state, agent_id).map_err(|error| { + PhasePrepareError::Parked(format!( + "assigned agent \"{agent_id}\" is not on the roster: {error}" + )) + })?, + None => resolve_agent_config(state) + .map_err(|error| PhasePrepareError::Failed(error.to_string()))?, + }; + + let (worktree, branch, _created) = ensure_phase_worktree(root, plan, index) + .map_err(|error| PhasePrepareError::Failed(error.to_string()))?; + plan.phases[index].worktree_path = Some(worktree.clone()); + plan.phases[index].worktree_branch = Some(branch.clone()); + if let Err(error) = handoff::seed_worktree_store(root, &worktree) { + warn!( + "could not seed upstream handoff artifacts into {}: {error}", + worktree.display() + ); + } + let injections = handoff_injections( + root, + plan, + &[( + execution_id.clone(), + loc.clone(), + plan.phases[index].interview.clone(), + )], + ) + .map_err(|error| { + PhasePrepareError::Parked(format!("handoff artifact injection failed: {error}")) + })?; + let mut prompt = build_phase_prompt(plan, index, &loc, &branch); + if let Some(artifacts) = injections.get(&execution_id) { + prompt.push_str(&handoff::render_injection(&execution_id, artifacts)); + } + let resume_session_id = plan.phases[index].resume_session_id.clone(); + plan.phases[index].resume_session_id = None; + begin_phase_execution(root, &execution_id, &loc) + .map_err(|error| PhasePrepareError::Failed(error.to_string()))?; + + let event_app = app.clone(); + let event_project_path = root.to_string_lossy().into_owned(); + let channel_execution_id = execution_id.clone(); + let channel: Channel = Channel::new(move |event| { + let event = match event.deserialize::() { + Ok(event) => event, + Err(error) => { + warn!("failed to decode unattended run activity: {error}"); + return Ok(()); + } + }; + if let Err(error) = event_app.emit( + RUN_QUEUE_EVENT, + RunQueueEvent { + project_path: event_project_path.clone(), + execution_id: channel_execution_id.clone(), + event, + }, + ) { + warn!("failed to emit unattended run activity: {error}"); + } + Ok(()) + }); + let run_events = Some(RunEventChannel::new( + plan.id.clone(), + normalized_run_channel( + app.clone(), + root.to_string_lossy().into_owned(), + execution_id.clone(), + ), + )); + + Ok(PhaseTask { + index, + execution_id, + worktree, + branch, + title: loc.title.clone(), + loc, + prompt, + agent_config, + channel, + run_events, + token_cap: resolved_phase_token_cap(plan), + phase_timeout: resolved_phase_timeout(plan), + run_timeout: resolved_run_timeout(plan).saturating_sub(run_started.elapsed()), + resume_session_id, + injections, + }) +} + fn finalize_worktree(plan: &mut RunPlan) { plan.environment.worktree_kept = true; } @@ -2813,144 +2902,7 @@ mod unattended_tests { } #[test] - fn next_queued_batch_collects_every_queued_index_in_order() { - let mut plan = parked_chain(); - plan.phases[1].status = RunPhaseStatus::Completed; - - let batch = next_queued_batch(&plan).expect("two phases are still queued"); - - assert_eq!(batch, vec![0, 2]); - } - - #[test] - fn next_queued_batch_is_none_once_nothing_is_queued() { - let mut plan = parked_chain(); - for phase in &mut plan.phases { - phase.status = RunPhaseStatus::Completed; - } - - assert_eq!(next_queued_batch(&plan), None); - } - - /// Per-phase staffing (`prd-role-foundations` Phase 4): a staffing change - /// splits the combined turn — dev-builds / QA-verifies queue as two - /// batches — while consecutive same-staffing (or all-unassigned) phases - /// keep sharing one turn. - #[test] - fn next_queued_batch_splits_on_staffing_change() { - let mut plan = parked_chain(); - plan.phases[0].assigned_agent_id = Some("dev-uuid".into()); - plan.phases[1].assigned_agent_id = Some("dev-uuid".into()); - plan.phases[2].assigned_agent_id = Some("qa-uuid".into()); - - assert_eq!(next_queued_batch(&plan), Some(vec![0, 1])); - plan.phases[0].status = RunPhaseStatus::Completed; - plan.phases[1].status = RunPhaseStatus::Completed; - assert_eq!(next_queued_batch(&plan), Some(vec![2])); - plan.phases[2].status = RunPhaseStatus::Completed; - assert_eq!(next_queued_batch(&plan), None); - - // Unassigned phases share the default agent's group and still batch - // together — the pre-assignment behaviour. - let plan = parked_chain(); - assert_eq!(next_queued_batch(&plan), Some(vec![0, 1, 2])); - } - - // ── Per-phase agent assignment (`prd-role-foundations` Phase 4) ────── - - #[test] - fn next_queued_batch_takes_the_whole_queue_when_no_agent_is_assigned() { - let plan = parked_chain(); - - let batch = next_queued_batch(&plan).expect("all phases queued"); - - // No assignments — every phase shares `None`, so the combined turn - // still covers the whole queue (pre-Phase-4 behavior preserved). - assert_eq!(batch, vec![0, 1, 2]); - } - - #[test] - fn next_queued_batch_takes_only_consecutive_same_agent_phases() { - let mut plan = parked_chain(); - // dev, dev, qa → first batch is the two dev phases; qa stays queued. - plan.phases[0].assigned_agent_id = Some("dev-id".into()); - plan.phases[1].assigned_agent_id = Some("dev-id".into()); - plan.phases[2].assigned_agent_id = Some("qa-id".into()); - - let batch = next_queued_batch(&plan).expect("phases are queued"); - - assert_eq!(batch, vec![0, 1]); - - plan.phases[0].status = RunPhaseStatus::Completed; - plan.phases[1].status = RunPhaseStatus::Completed; - let next = next_queued_batch(&plan).expect("qa phase still queued"); - assert_eq!(next, vec![2]); - } - - #[test] - fn next_queued_batch_splits_when_assignment_differs_from_default() { - let mut plan = parked_chain(); - // default, dev, default → three groups of one. - plan.phases[1].assigned_agent_id = Some("dev-id".into()); - - let first = next_queued_batch(&plan).expect("phases are queued"); - assert_eq!(first, vec![0]); - - plan.phases[0].status = RunPhaseStatus::Completed; - let second = next_queued_batch(&plan).expect("dev phase queued"); - assert_eq!(second, vec![1]); - - plan.phases[1].status = RunPhaseStatus::Completed; - let third = next_queued_batch(&plan).expect("default phase queued"); - assert_eq!(third, vec![2]); - } - - #[test] - fn next_queued_batch_regroups_across_a_non_queued_phase() { - let mut plan = parked_chain(); - // Queued set is phases 0 and 2 (1 completed), both dev → one batch, - // because grouping is over the eligible list, not raw plan indices. - plan.phases[0].assigned_agent_id = Some("dev-id".into()); - plan.phases[1].status = RunPhaseStatus::Completed; - plan.phases[1].assigned_agent_id = None; - plan.phases[2].assigned_agent_id = Some("dev-id".into()); - - let batch = next_queued_batch(&plan).expect("two phases queued"); - - assert_eq!(batch, vec![0, 2]); - } - - // ── Handoff turn boundary (`prd-agent-handoff` Phase 2) ───────────── - - #[test] - fn next_queued_batch_starts_a_new_turn_before_a_handoff_consumer() { - let mut plan = parked_chain(); - plan.phases[0].handoff_topic = Some("phase-a-plan".into()); - - // phase-b depends on the publisher, so it must run in its own turn - // after phase-a's artifact exists. - assert_eq!(next_queued_batch(&plan), Some(vec![0])); - - plan.phases[0].status = RunPhaseStatus::Completed; - assert_eq!(next_queued_batch(&plan), Some(vec![1, 2])); - } - - #[test] - fn next_queued_batch_keeps_non_consumers_together_before_the_boundary() { - let mut plan = parked_chain(); - // phase-b publishes; phase-c consumes it. phase-a and phase-b still - // share the first turn, then the consumer starts a new one. - plan.phases[1].handoff_topic = Some("phase-b-plan".into()); - - assert_eq!(next_queued_batch(&plan), Some(vec![0, 1])); - - plan.phases[0].status = RunPhaseStatus::Completed; - plan.phases[1].status = RunPhaseStatus::Completed; - assert_eq!(next_queued_batch(&plan), Some(vec![2])); - } - - #[test] - fn handoff_injections_load_the_publishers_artifact_for_the_consumer() { + fn handoff_injections_preserve_the_publishers_snapshot_for_attribution() { let dir = std::env::temp_dir().join(format!("loopdeck-inject-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(dir.join(".loopdeck/handoffs")).unwrap(); let mut plan = parked_chain(); @@ -2976,53 +2928,36 @@ mod unattended_tests { phase: "ph".into(), title: "t".into(), }; - let injections = handoff_injections(&dir, &plan, &[("phase-b".into(), loc, vec![])]) + let locs = vec![("phase-b".into(), loc, vec![])]; + let injections = handoff_injections(&dir, &plan, &locs) .expect("consumer receives its publisher artifact"); let artifacts = &injections["phase-b"]; assert_eq!(artifacts.len(), 1); assert_eq!(artifacts[0].topic, "phase-a-plan"); assert!(artifacts[0].artifact.body.contains("R1 do the thing")); + let rendered = handoff::render_injection("phase-b", artifacts); + assert!(rendered.contains("R1 do the thing")); + assert!(rendered.contains("Handoff citations")); + + // The shared store may change while a consumer turn is running. + // Attribution must use the exact snapshot injected into that turn. + std::fs::remove_file(dir.join(handoff::store_relative_path("phase-a-plan"))).unwrap(); + plan.phases[1].assigned_agent_name = Some("reviewer".into()); + record_handoff_citations(&dir, &plan, &locs, &injections, "No citations."); + let entries = handoff::handoffs_for_run(&dir, &plan.id).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].from_role, "planner"); + assert_eq!(entries[0].to_role, "reviewer"); + assert_eq!(entries[0].citation, handoff::CitationResult::Missing); + assert!(handoff::handoffs_for_run(&dir, "another-run") + .unwrap() + .is_empty()); + assert!(handoff_injections(&dir, &plan, &locs).is_err()); std::fs::remove_dir_all(dir).ok(); } - #[test] - fn park_missing_agent_names_the_agent_and_parks_only_policy_dependents() { - let mut plan = parked_chain(); - // phase-b and phase-c depend on phase-a (authored chain); park its - // batch for a missing agent under ContinueIndependent. - park_missing_agent(&mut plan, &[0], "gone-agent-id"); - - assert_eq!(plan.phases[0].status, RunPhaseStatus::Parked); - assert!( - plan.phases[0] - .park_payload - .as_deref() - .unwrap() - .contains("gone-agent-id"), - "payload must name the missing agent" - ); - // Dependents park as blocked; that's the authored chain here. - assert_eq!(plan.phases[1].status, RunPhaseStatus::Parked); - assert_eq!(plan.phases[2].status, RunPhaseStatus::Parked); - } - - #[test] - fn park_missing_agent_leaves_independent_phases_queued_under_continue() { - let mut plan = parked_chain(); - // Break the chain: phase-c no longer depends on anything, so only the - // batch itself parks and the independent phase stays eligible. - plan.phases[2].depends_on.clear(); - - park_missing_agent(&mut plan, &[0], "gone-agent-id"); - - assert_eq!(plan.phases[0].status, RunPhaseStatus::Parked); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Parked); - assert_eq!(plan.phases[2].status, RunPhaseStatus::Queued); - assert!(plan.phases[2].park_payload.is_none()); - } - // ── Handoff topics (`prd-agent-handoff` Phase 1) ───────────────────── fn topic(execution_id: &str, topic: &str) -> runplan::PhaseHandoffTopic { @@ -3081,39 +3016,6 @@ mod unattended_tests { .all(|phase| phase.handoff_topic.is_none())); } - #[test] - fn kill_batch_and_park_batch_apply_to_every_index_in_the_batch() { - let mut plan = parked_chain(); - - kill_batch(&mut plan, &[0, 2], "total run wall-clock budget exceeded"); - assert_eq!(plan.phases[0].status, RunPhaseStatus::Killed); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Queued); - assert_eq!(plan.phases[2].status, RunPhaseStatus::Killed); - assert_eq!( - plan.phases[0].park_payload.as_deref(), - Some("total run wall-clock budget exceeded") - ); - - park_batch(&mut plan, &[0, 1, 2], "verify verdict: WARN"); - assert!(plan - .phases - .iter() - .all(|phase| phase.status == RunPhaseStatus::Parked)); - assert!(plan - .phases - .iter() - .all(|phase| phase.park_payload.as_deref() == Some("verify verdict: WARN"))); - } - - #[test] - fn set_batch_token_usage_splits_the_total_evenly_across_the_batch() { - let mut plan = parked_chain(); - - set_batch_token_usage(&mut plan, &[0, 1, 2], 900, 3); - - assert!(plan.phases.iter().all(|phase| phase.token_usage == 300)); - } - #[test] fn budget_defaults_are_hard_and_named() { let plan = plan(); @@ -3144,51 +3046,6 @@ mod unattended_tests { assert_eq!(resolved_run_timeout(&plan), Duration::from_secs(67)); } - fn plan_with_id_and_phases(id: &str, execution_ids: &[String]) -> RunPlan { - run_executor::build_run_plan( - id.into(), - PathBuf::from("/repo"), - Utc::now(), - execution_ids, - StallPolicy::ContinueIndependent, - false, - &[], - ) - } - - #[test] - fn run_branch_name_uses_first_phase_slug() { - let plan = plan_with_id_and_phases( - "run-a3a5f38c-9369-464e-acb4-922a0bfc6c7e", - &["wake-up/notify-after-completed".into()], - ); - assert_eq!( - run_branch_name(&plan), - "run/wake-up-notify-after-completed-922a0bfc6c7e" - ); - } - - #[test] - fn run_branch_name_notes_multi_phase_runs() { - let plan = plan_with_id_and_phases( - "run-a3a5f38c-9369-464e-acb4-922a0bfc6c7e", - &[ - "wake-up/notify-after-completed".into(), - "wake-up/report-per-phase-verdict".into(), - ], - ); - assert_eq!( - run_branch_name(&plan), - "run/wake-up-notify-after-completed-plus1-922a0bfc6c7e" - ); - } - - #[test] - fn run_branch_name_sanitizes_unsafe_characters() { - let plan = plan_with_id_and_phases("run a/b", &[]); - assert_eq!(run_branch_name(&plan), "run/run-a-b-run-a-b"); - } - #[tokio::test] async fn stuck_bootstrap_child_is_killed_by_timeout() { let mut child = tokio::process::Command::new("/bin/sh") @@ -3335,20 +3192,19 @@ mod unattended_tests { } } - /// Create: `ensure_worktree` on a plan with no recorded environment - /// creates a real linked worktree and persists its path/branch. + /// Create: `ensure_phase_worktree` on a fresh plan creates a real linked + /// worktree, cuts it from the default branch, and returns the identity the + /// caller persists on the phase. #[test] - fn ensure_worktree_creates_and_persists_environment() { + fn ensure_phase_worktree_creates_a_phase_scoped_worktree() { let dir = temp_git_repo("create"); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); + let plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); + let (worktree, branch, created) = + ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); - assert_eq!( - plan.environment.worktree_path.as_deref(), - Some(worktree.as_path()) - ); - assert!(plan.environment.worktree_branch.is_some()); + assert!(created); + assert!(branch.starts_with("run/phase-1-p1-"), "{branch}"); assert!(git::worktree_list(&dir).unwrap().contains(&worktree)); git::worktree_remove(&dir, &worktree).ok(); @@ -3356,11 +3212,11 @@ mod unattended_tests { } /// Clean handoff (`prd-verified-delivery-reconciliation` Phase 4): a new - /// run branch is cut from the default branch, not from whatever the main + /// phase branch is cut from the default branch, not from whatever the main /// worktree has checked out. A fix committed to a stray branch must not - /// ride into the next run's base. + /// ride into the next phase's base. #[test] - fn ensure_worktree_bases_new_branch_on_default_branch_not_head() { + fn ensure_phase_worktree_bases_new_branch_on_default_branch_not_head() { let dir = temp_git_repo("base"); let run_git = |args: &[&str]| { std::process::Command::new("git") @@ -3379,8 +3235,8 @@ mod unattended_tests { run_git(&["commit", "-m", "stray fix"]); let stray_tip = String::from_utf8(run_git(&["rev-parse", "HEAD"]).stdout).unwrap(); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); + let plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); + let (worktree, _, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); let tip = String::from_utf8( std::process::Command::new("git") @@ -3393,7 +3249,11 @@ mod unattended_tests { .stdout, ) .unwrap(); - assert_eq!(tip.trim(), main_tip.trim(), "run branch must start at main"); + assert_eq!( + tip.trim(), + main_tip.trim(), + "phase branch must start at main" + ); assert_ne!(tip.trim(), stray_tip.trim()); assert!(!worktree.join("stray.txt").exists()); @@ -3406,11 +3266,10 @@ mod unattended_tests { #[test] fn record_handoff_persists_a_loadable_handoff_record() { let dir = temp_git_repo("handoff"); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); - let branch = plan.environment.worktree_branch.clone().unwrap(); + let plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); + let (worktree, branch, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); - record_handoff(&dir, &plan, &worktree, "https://github.com/o/r/pull/42"); + record_handoff(&dir, &branch, &worktree, "https://github.com/o/r/pull/42"); let record = handoff::load(&dir) .unwrap() @@ -3430,17 +3289,24 @@ mod unattended_tests { std::fs::remove_dir_all(&dir).ok(); } - /// Resume: calling `ensure_worktree` again on a plan whose recorded - /// worktree still exists is idempotent — no second worktree/branch. + /// Resume: calling `ensure_phase_worktree` again for a phase whose + /// recorded worktree still exists is idempotent — no second worktree or + /// branch. #[test] - fn ensure_worktree_resumes_an_existing_worktree_without_duplicating_it() { + fn ensure_phase_worktree_resumes_without_duplicating() { let dir = temp_git_repo("resume"); let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let first = ensure_worktree(&dir, &mut plan).expect("create worktree"); + let (first, first_branch, _) = + ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); + plan.phases[0].worktree_path = Some(first.clone()); + plan.phases[0].worktree_branch = Some(first_branch.clone()); - let second = ensure_worktree(&dir, &mut plan).expect("resume worktree"); + let (second, second_branch, created) = + ensure_phase_worktree(&dir, &plan, 0).expect("resume worktree"); assert_eq!(first, second); + assert_eq!(first_branch, second_branch); + assert!(!created); assert_eq!( git::worktree_list(&dir) .unwrap() @@ -3455,69 +3321,48 @@ mod unattended_tests { std::fs::remove_dir_all(&dir).ok(); } - /// A restart whose recorded worktree was deleted out-of-band (e.g. by - /// hand, or a prior crash mid-remove) fails loudly rather than silently - /// creating a second branch for the same run. - #[test] - fn ensure_worktree_errors_when_recorded_worktree_is_missing() { - let dir = temp_git_repo("missing"); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - plan.environment.worktree_path = Some(dir.join("nonexistent-linked-worktree")); - - let err = ensure_worktree(&dir, &mut plan).unwrap_err(); - assert!(err.to_string().contains("missing")); - - std::fs::remove_dir_all(&dir).ok(); - } - /// Resume-from-branch (`prd-verified-delivery-reconciliation` Phase 2): /// a recorded worktree removed from Git's inventory while its branch /// survived is recreated at the same path from that branch — no second /// branch, no hard stop. #[test] - fn ensure_worktree_recreates_from_surviving_branch() { + fn ensure_phase_worktree_recreates_from_surviving_branch() { let dir = temp_git_repo("recreate"); let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let first = ensure_worktree(&dir, &mut plan).expect("create worktree"); - let branch = plan.environment.worktree_branch.clone().unwrap(); + let (first, branch, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); + plan.phases[0].worktree_path = Some(first.clone()); + plan.phases[0].worktree_branch = Some(branch.clone()); git::worktree_remove(&dir, &first).expect("remove worktree, keep branch"); assert!(!first.exists()); assert!(git::branch_exists(&dir, &branch)); - let second = ensure_worktree(&dir, &mut plan).expect("recreate from branch"); + let (second, second_branch, created) = + ensure_phase_worktree(&dir, &plan, 0).expect("recreate from branch"); assert_eq!(first, second, "recreated at the same recorded path"); + assert_eq!(branch, second_branch); + assert!(created); assert!(second.exists()); - assert_eq!( - git::worktree_list(&dir) - .unwrap() - .iter() - .filter(|p| **p == second) - .count(), - 1 - ); git::worktree_remove(&dir, &second).ok(); std::fs::remove_dir_all(&dir).ok(); } - /// New managed worktrees land under `.loopdeck/runs/` and the ignore rule - /// keeping them out of the main worktree's untracked view is appended - /// exactly once. + /// Every phase worktree lives under `.loopdeck/runs/`, and the ignore + /// rule keeping managed worktrees out of the main checkout's untracked + /// view is appended exactly once. #[test] - fn ensure_worktree_ignores_managed_runs_dir_once() { + fn ensure_phase_worktree_ignores_managed_runs_dir_once() { let dir = temp_git_repo("ignore"); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); + let plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); - let second = ensure_worktree(&dir, &mut plan).expect("resume"); + let (worktree, _, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); // Git canonicalizes the worktree path (`/var` → `/private/var` on // macOS); compare against the canonical repo dir. let canonical_dir = std::fs::canonicalize(&dir).unwrap(); assert!(worktree.starts_with(canonical_dir.join(".loopdeck").join("runs"))); - assert_eq!(worktree, second); let gitignore = std::fs::read_to_string(dir.join(".gitignore")).unwrap(); assert_eq!( gitignore @@ -3532,51 +3377,75 @@ mod unattended_tests { std::fs::remove_dir_all(&dir).ok(); } + /// Parallel scheduling: two phases get two disjoint worktrees on two + /// branches cut from the same default tip. This is the cleanup-race + /// guard: removing one phase's worktree must never touch its sibling. + #[test] + fn phase_worktrees_are_disjoint_and_cleanup_is_isolated() { + let dir = temp_git_repo("parallel"); + let plan = plan_with_phases( + &dir, + vec![completed_phase("phase-a"), completed_phase("phase-b")], + ); + + let (first, first_branch, _) = ensure_phase_worktree(&dir, &plan, 0).expect("a"); + let (second, second_branch, _) = ensure_phase_worktree(&dir, &plan, 1).expect("b"); + + assert_ne!(first, second); + assert_ne!(first_branch, second_branch); + // Git canonicalizes the worktree path (`/var` → `/private/var` on + // macOS), so compare against the canonical repo root. + let canonical_dir = std::fs::canonicalize(&dir).unwrap(); + assert!(first.starts_with(canonical_dir.join(".loopdeck"))); + assert!(second.starts_with(canonical_dir.join(".loopdeck"))); + + git::worktree_remove(&dir, &first).expect("remove a"); + assert!(!first.exists()); + assert!(second.exists(), "sibling worktree must survive cleanup"); + + git::worktree_remove(&dir, &second).ok(); + std::fs::remove_dir_all(&dir).ok(); + } + /// Retain-on-delivery (`clean-handoff`, superseding the old prune-on- - /// success): every phase delivered → the worktree stays on disk under - /// `.loopdeck/runs/`, flagged kept, with its recorded path/branch intact - /// so the branch + PR stay reviewable. + /// success): every phase delivered → each phase worktree stays on disk + /// under `.loopdeck/runs/`, flagged kept, with its recorded path/branch + /// intact so the branch + PR stay reviewable. #[test] - fn finalize_worktree_retains_a_delivered_worktree_for_review() { + fn finalize_worktree_retains_every_delivered_phase_worktree() { let dir = temp_git_repo("retain"); - let mut plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); + let plan = plan_with_phases(&dir, vec![completed_phase("phase-1")]); + let (worktree, branch, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); + let mut plan = plan; finalize_worktree(&mut plan); assert!(plan.environment.worktree_kept); - assert_eq!( - plan.environment.worktree_path.as_deref(), - Some(worktree.as_path()) - ); - assert!(plan.environment.worktree_branch.is_some()); assert!( worktree.exists(), "the delivered worktree must remain on disk" ); + assert!(git::branch_exists(&dir, &branch)); git::worktree_remove(&dir, &worktree).ok(); std::fs::remove_dir_all(&dir).ok(); } - /// Keep-on-failure: any phase not delivered → same retention, so a failed - /// run stays inspectable. + /// Keep-on-failure: any phase not delivered → the same retention, so a + /// failed run stays inspectable. #[test] fn finalize_worktree_keeps_on_any_non_completed_phase() { let dir = temp_git_repo("keep-failed"); - let mut plan = plan_with_phases( + let plan = plan_with_phases( &dir, vec![completed_phase("phase-1"), failed_phase("phase-2")], ); - let worktree = ensure_worktree(&dir, &mut plan).expect("create worktree"); + let (worktree, _, _) = ensure_phase_worktree(&dir, &plan, 0).expect("create worktree"); + let mut plan = plan; finalize_worktree(&mut plan); assert!(plan.environment.worktree_kept); - assert_eq!( - plan.environment.worktree_path.as_deref(), - Some(worktree.as_path()) - ); assert!(worktree.exists(), "a kept worktree must remain on disk"); git::worktree_remove(&dir, &worktree).ok(); diff --git a/src-tauri/src/handoff.rs b/src-tauri/src/handoff.rs index 05e4d9b..8e4e283 100644 --- a/src-tauri/src/handoff.rs +++ b/src-tauri/src/handoff.rs @@ -528,6 +528,44 @@ pub fn copy_agent_artifact( Ok(artifact) } +/// Copy the persistent artifact store into a phase worktree's +/// `.loopdeck/handoffs/` before that phase's turn starts. +/// +/// Every phase worktree is cut from the default branch, so a dependent phase +/// cannot otherwise see an upstream artifact the executor copied into the +/// main checkout minutes earlier. Seeding makes the dependency explicit: +/// downstream sessions read only committed handoff contracts, never an +/// upstream branch's unmerged code (the approved branch-scoping answer). +/// +/// Existing files in the worktree are never overwritten. Returns how many +/// artifacts were copied; a missing store is a no-op. +pub fn seed_worktree_store(repo_path: &Path, worktree: &Path) -> Result { + let store = store_dir(repo_path); + if !store.is_dir() { + return Ok(0); + } + let destination = worktree.join(".loopdeck").join(HANDOFFS_DIR); + let mut copied = 0; + for entry in std::fs::read_dir(&store)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("md") { + continue; + } + let Some(name) = path.file_name() else { + continue; + }; + std::fs::create_dir_all(&destination)?; + let target = destination.join(name); + if target.exists() { + continue; + } + std::fs::copy(&path, &target)?; + copied += 1; + } + Ok(copied) +} + fn parse_artifact(contents: &str, topic: &str, path: PathBuf) -> Result { let Some(rest) = contents.strip_prefix("---\n") else { return Err(AppError::RunPlan(format!( diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 1a8544b..b09e167 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -31,6 +31,7 @@ pub mod retry; mod run_executor; mod runplan; mod scanner; +mod scheduler; mod secret_scan; mod secrets; mod skills; diff --git a/src-tauri/src/limits.rs b/src-tauri/src/limits.rs index 4ddfdd9..6df1958 100644 --- a/src-tauri/src/limits.rs +++ b/src-tauri/src/limits.rs @@ -107,6 +107,17 @@ pub const DEFAULT_RUN_PHASE_WALL_CLOCK_SECS: u64 = 90 * 60; /// Default maximum duration of a whole overnight run. pub const DEFAULT_RUN_TOTAL_WALL_CLOCK_SECS: u64 = 8 * 60 * 60; +/// Default number of run-plan phases that may execute at the same time. +/// +/// Four keeps the load and cost predictable: at most four worktrees, four +/// provider sessions, and four token streams per run. +pub const DEFAULT_RUN_MAX_CONCURRENCY: u32 = 4; + +/// Upper bound for a plan's `max_concurrency`. A plan that asks for more is +/// rejected at creation rather than silently clamped, so the picker cannot +/// promise a concurrency the executor will not honor. +pub const MAX_RUN_MAX_CONCURRENCY: u32 = 16; + /// Max bytes of a spec / PRD / epic Markdown file read wholesale. /// /// These live under `docs/epics/` (human-authored) and `.loopdeck/` (agent- diff --git a/src-tauri/src/role_assignment_tests.rs b/src-tauri/src/role_assignment_tests.rs index 220bfb4..b4d24cb 100644 --- a/src-tauri/src/role_assignment_tests.rs +++ b/src-tauri/src/role_assignment_tests.rs @@ -113,10 +113,13 @@ async fn two_phase_role_demo_attributes_each_phase_to_its_role() { Some(roster.qa_id.as_str()) ); - // 2. The staffing change splits the combined turn: one batch per role. + // 2. One turn per phase (`prd-parallel-scheduling`): the authored chain + // makes only the dev phase ready first. Staffing no longer splits a + // combined turn because there is no combined turn — each phase gets + // its own. assert_eq!( - crate::commands::run_queue::next_queued_batch(&plan), - Some(vec![0]) + crate::scheduler::select_ready(&plan, &Default::default(), 4), + vec![0] ); // 3. Execute each staffing batch through the executor's exact seam: @@ -191,10 +194,10 @@ fn unassigned_phase_stays_on_the_default_agent() { ); assert!(plan.phases[0].assigned_agent_id.is_none()); assert!(plan.phases[0].assigned_agent_name.is_none()); - // All-unassigned plans still batch into one combined turn. + // An unassigned phase is still schedulable: it is the one ready phase. assert_eq!( - crate::commands::run_queue::next_queued_batch(&plan), - Some(vec![0]) + crate::scheduler::select_ready(&plan, &Default::default(), 4), + vec![0] ); } diff --git a/src-tauri/src/role_demo_tests.rs b/src-tauri/src/role_demo_tests.rs index 41ab52a..434e35a 100644 --- a/src-tauri/src/role_demo_tests.rs +++ b/src-tauri/src/role_demo_tests.rs @@ -27,7 +27,6 @@ use crate::agents::{ClaudeEvent, TokenBudget}; use crate::commands::agent::start_fresh_and_record_streaming_in_root_with_config; -use crate::commands::run_queue::next_queued_batch; use crate::commands::state::resolve_agent_config_by_id; use crate::commands::state::AppState; use crate::config::{AgentConfig, GlobalConfig, NamedAgentConfig, RoleCharter}; @@ -193,9 +192,16 @@ async fn role_demo_two_phase_plan_dev_builds_qa_verifies() { .expect("plan present"); let batch_count_before = plan.phases.len(); - // ── Execute: the executor's own batching + resolution + spawn path. ── - while let Some(batch) = next_queued_batch(&plan) { - let assigned = plan.phases[batch[0]] + // ── Execute: one scheduler-selected phase turn at a time. ── + let mut executed = 0usize; + loop { + crate::scheduler::mark_waiting(&mut plan); + crate::scheduler::reconcile_gates(&mut plan); + let in_flight = HashSet::new(); + let Some(&index) = crate::scheduler::select_ready(&plan, &in_flight, 1).first() else { + break; + }; + let assigned = plan.phases[index] .assigned_agent .clone() .expect("demo phases are all assigned"); @@ -206,25 +212,23 @@ async fn role_demo_two_phase_plan_dev_builds_qa_verifies() { "charter must ride the config" ); - let locs: Vec<(String, epic::LoopLocation)> = batch - .iter() - .map(|&idx| { - let execution_id = plan.phases[idx].execution_id.clone(); - let loc = - epic::find_loop_by_id(&repo, &execution_id).expect("fixture loop resolves"); - (execution_id, loc) - }) - .collect(); - // One agent per batch — the Phase 4 split guarantee under test. - assert_eq!(batch.len(), 1, "each demo phase has its own agent"); + let execution_id = plan.phases[index].execution_id.clone(); + let loc: epic::LoopLocation = + epic::find_loop_by_id(&repo, &execution_id).expect("fixture loop resolves"); + // One turn per phase — the scheduler never combines phases. + assert_eq!( + plan.phases + .iter() + .filter(|phase| phase.status == RunPhaseStatus::Running) + .count(), + 0, + "each demo phase runs in its own turn" + ); - for &idx in &batch { - plan.phases[idx].status = RunPhaseStatus::Running; - } + plan.phases[index].status = RunPhaseStatus::Running; runplan::save(&repo, &plan).expect("persist running state"); - let (_, loc) = &locs[0]; - let prompt = demo_prompt(&locs[0].0, &loc.title, &loc.phase); + let prompt = demo_prompt(&execution_id, &loc.title, &loc.phase); let channel: Channel = Channel::new(|_| Ok(())); let token_budget = TokenBudget::new(300_000); let started = Instant::now(); @@ -250,14 +254,13 @@ async fn role_demo_two_phase_plan_dev_builds_qa_verifies() { .as_ref() .map(|usage| usage.input_tokens.saturating_add(usage.output_tokens)) .unwrap_or_default(); - for &idx in &batch { - plan.phases[idx].status = RunPhaseStatus::Completed; - plan.phases[idx].token_usage = tokens; - plan.phases[idx].wall_clock_secs = started.elapsed().as_secs(); - } + plan.phases[index].status = RunPhaseStatus::Completed; + plan.phases[index].token_usage = tokens; + plan.phases[index].wall_clock_secs = started.elapsed().as_secs(); + executed += 1; runplan::save(&repo, &plan).expect("persist completed phase"); } - assert_eq!(batch_count_before, 2); + assert_eq!(executed, batch_count_before, "both demo phases ran"); // ── Assert the work really happened: dev built, QA verified. ── let math = std::fs::read_to_string(repo.join("math.js")).expect("dev turn wrote math.js"); diff --git a/src-tauri/src/role_foundations_tests.rs b/src-tauri/src/role_foundations_tests.rs index fc51e21..bf4af8d 100644 --- a/src-tauri/src/role_foundations_tests.rs +++ b/src-tauri/src/role_foundations_tests.rs @@ -137,10 +137,12 @@ fn legacy_run_plan_without_assignment_fields_keeps_the_default_agent() { assert!(phase.assigned_agent_name.is_none()); assert!(phase.handoff_topic.is_none()); } + // Legacy plans have no authored edges, so both phases are ready at once; + // the executor still gives each phase its own turn. assert_eq!( - crate::commands::run_queue::next_queued_batch(&plan), - Some(vec![0, 1]), - "all-unassigned phases still share one combined turn" + crate::scheduler::select_ready(&plan, &Default::default(), 4), + vec![0, 1], + "legacy plans remain schedulable under per-phase turns" ); std::fs::remove_dir_all(dir).ok(); @@ -183,10 +185,11 @@ fn assignment_and_handoff_topic_round_trip_through_the_plan_file() { loaded.phases[1].handoff_topic.as_deref(), Some("demo-qa-verify") ); - // dev builds, QA verifies: the staffing change splits the combined turn. + // dev builds, QA verifies: the authored chain makes only the dev phase + // ready first. Each phase then runs in its own turn. assert_eq!( - crate::commands::run_queue::next_queued_batch(&loaded), - Some(vec![0]) + crate::scheduler::select_ready(&loaded, &Default::default(), 4), + vec![0] ); std::fs::remove_dir_all(dir).ok(); diff --git a/src-tauri/src/run_executor.rs b/src-tauri/src/run_executor.rs index fa61f19..057c590 100644 --- a/src-tauri/src/run_executor.rs +++ b/src-tauri/src/run_executor.rs @@ -142,12 +142,10 @@ fn phase_goal_block(execution_id: &str, loc: &LoopLocation, interview: &[PinnedA block } -/// Build the prompt for one orchestrated turn covering *every* phase in -/// `phases` at once — the run queue always merges its currently-queued -/// phases into a single LLM call rather than firing one turn per phase (an -/// overnight run of N loops is one combined session, not N sequential ones). -/// A single-element slice produces the same shape of prompt as running just -/// that one phase always did. +/// Build the prompt for an orchestrated turn covering the phases in +/// `phases`. Production (`prd-parallel-scheduling`) always passes a +/// single-element slice: one phase, one turn. The multi-element shape is +/// retained for the legacy combined-turn contract and its tests. pub(crate) fn build_combined_phase_prompt( phases: &[(String, LoopLocation, Vec)], draft_pr_authorized: bool, @@ -448,6 +446,12 @@ pub(crate) fn extract_batch_interview_answers(text: &str) -> HashMap(plan: &'a RunPlan, phase: &RunPhase) -> V .collect() } -/// Whether this phase must consume an upstream handoff artifact. -pub(crate) fn consumes_handoff(plan: &RunPlan, phase: &RunPhase) -> bool { - !publishing_ancestors(plan, phase).is_empty() -} - -fn is_terminal_good(status: RunPhaseStatus) -> bool { - matches!( - status, - RunPhaseStatus::Completed | RunPhaseStatus::Delivered - ) -} - -fn is_terminal_bad(status: RunPhaseStatus) -> bool { - matches!( - status, - RunPhaseStatus::Parked - | RunPhaseStatus::Failed - | RunPhaseStatus::Interrupted - | RunPhaseStatus::Killed - ) -} - -/// Re-evaluate every handoff consumer's eligibility after a turn -/// (`prd-agent-handoff` Phase 2). A `Queued` consumer whose publishing -/// ancestor is not yet `Completed`/`Delivered` moves to `Waiting`; a -/// `Waiting` consumer returns to `Queued` once its ancestors are good, and -/// parks when any ancestor is terminal-bad. Runs to a fixed point so a chain -/// settles in one call. Returns `true` when anything changed. -pub(crate) fn recheck_waiting_phases(plan: &mut RunPlan) -> Result { - let mut changed = false; - loop { - let mut pass_changed = false; - for index in 0..plan.phases.len() { - let status = plan.phases[index].status; - if !matches!(status, RunPhaseStatus::Queued | RunPhaseStatus::Waiting) { - continue; - } - let ancestors: Vec<(String, RunPhaseStatus)> = plan.phases[index] - .depends_on - .iter() - .filter_map(|id| { - plan.phases - .iter() - .find(|candidate| &candidate.execution_id == id) - }) - .filter(|ancestor| ancestor.handoff_topic.is_some()) - .map(|ancestor| (ancestor.execution_id.clone(), ancestor.status)) - .collect(); - if ancestors.is_empty() { - continue; - } - - match status { - RunPhaseStatus::Queued => { - if ancestors - .iter() - .any(|(_, ancestor_status)| !is_terminal_good(*ancestor_status)) - { - plan.phases[index].status = - crate::transition::transition_phase(status, RunPhaseStatus::Waiting)?; - pass_changed = true; - } - } - RunPhaseStatus::Waiting => { - if let Some((upstream, _)) = ancestors - .iter() - .find(|(_, ancestor_status)| is_terminal_bad(*ancestor_status)) - { - plan.phases[index].status = - crate::transition::transition_phase(status, RunPhaseStatus::Parked)?; - plan.phases[index].park_payload = Some(format!( - "blocked: upstream handoff phase \"{upstream}\" did not complete" - )); - pass_changed = true; - } else if ancestors - .iter() - .all(|(_, ancestor_status)| is_terminal_good(*ancestor_status)) - { - plan.phases[index].status = - crate::transition::transition_phase(status, RunPhaseStatus::Queued)?; - pass_changed = true; - } - } - _ => unreachable!("filtered to Queued and Waiting above"), - } - } - if !pass_changed { - return Ok(changed); - } - changed = true; - } -} - /// Park every still-`Waiting` phase when the run can make no further /// progress, so an unattended run never leaves one waiting forever. Returns /// `true` when anything changed. @@ -659,6 +570,10 @@ pub(crate) fn build_run_plan( assigned_agent_id: assignment.map(|a| a.agent_id.clone()), assigned_agent_name: assignment.and_then(|a| a.agent_name.clone()), handoff_topic: None, + worktree_path: None, + worktree_branch: None, + session_id: None, + resume_session_id: None, } }) .collect(); @@ -674,32 +589,45 @@ pub(crate) fn build_run_plan( environment: RunEnvironment::default(), wall_clock_secs: 0, stall_policy, + // `create_run_plan` overwrites this with the picker's value; the + // builder itself stays deterministic for tests and callers that do + // not expose the knob. + max_concurrency: crate::limits::DEFAULT_RUN_MAX_CONCURRENCY, phases, } } -/// Startup reconciliation, mirroring `conversation::reconcile_interrupted`: -/// a `Running` phase left on disk from a killed/crashed process is not -/// actually running on a fresh process — nothing has resumed it yet — so -/// downgrade it to `Interrupted` (P1: "resumable across app restarts"). A -/// `Waiting` phase is re-queued for the same reason: no turn is live to -/// re-check its handoff dependency after a restart. -/// `queue_run` also calls this before deciding whether a run is already -/// active, so a stale `Running` phase from a prior crash can't block a new -/// queue attempt forever. +/// Startup reconciliation for **multiple** in-flight phases +/// (`parallel-scheduling/restart-reconcile`, superseding the single-phase +/// `Interrupted` downgrade). +/// +/// A `Running` phase left on disk from a killed/crashed process is not +/// actually running on a fresh process. Per the approved restart answer, the +/// next queue attempt **auto-resumes** each phase whose worktree and saved +/// provider session both survive — re-queuing it so the executor starts a +/// fresh turn in the surviving tree. Anything less parks the phase with a +/// reason for the morning report instead of guessing. +/// +/// `queue_run` and `get_run_status` call this before deciding whether a run is +/// active, so a stale `Running` phase can't block a new attempt forever. /// /// Returns `true` if the plan was rewritten. A missing plan is a no-op. pub fn reconcile_running_phases(repo_path: &Path) -> Result { let Some(mut plan) = runplan::load(repo_path)? else { return Ok(false); }; + if !plan.phases.iter().any(|phase| { + matches!( + phase.status, + RunPhaseStatus::Running | RunPhaseStatus::Waiting + ) + }) { + return Ok(false); + } + + let worktrees = crate::git::worktree_list(repo_path).unwrap_or_default(); let mut changed = false; for phase in &mut plan.phases { - if phase.status == RunPhaseStatus::Running { - phase.status = - crate::transition::transition_phase(phase.status, RunPhaseStatus::Interrupted)?; - changed = true; - } // A `Waiting` phase from a crashed run has no live turn to re-check // it. Re-queue it so the next executor pass re-derives the hold and // parks it if its upstream is terminal-bad, rather than waiting @@ -709,6 +637,43 @@ pub fn reconcile_running_phases(repo_path: &Path) -> Result { crate::transition::transition_phase(phase.status, RunPhaseStatus::Queued)?; changed = true; } + if phase.status != RunPhaseStatus::Running { + continue; + } + let worktree = phase + .worktree_path + .clone() + .or_else(|| plan.environment.worktree_path.clone()); + let worktree_survives = worktree + .as_ref() + .is_some_and(|path| path.is_dir() || worktrees.iter().any(|known| known == path)); + let session_survives = + phase.session_id.is_some() || crate::conversation::last_session_id(repo_path).is_some(); + match crate::scheduler::classify_interrupted_phase(worktree_survives, session_survives) { + crate::scheduler::ResumeDecision::AutoResume => { + phase.status = + crate::transition::transition_phase(phase.status, RunPhaseStatus::Queued)?; + phase.resume_session_id = phase.session_id.clone(); + phase.park_payload = Some( + "resumed after app restart: worktree and provider session both survived".into(), + ); + changed = true; + } + crate::scheduler::ResumeDecision::Park => { + let missing = match (worktree_survives, session_survives) { + (false, false) => "worktree and provider session", + (false, true) => "worktree", + (true, false) => "saved provider session", + (true, true) => unreachable!(), + }; + phase.status = + crate::transition::transition_phase(phase.status, RunPhaseStatus::Parked)?; + phase.park_payload = Some(format!( + "interrupted by app restart: {missing} did not survive; requeue to retry" + )); + changed = true; + } + } } if changed { runplan::save(repo_path, &plan)?; @@ -1081,13 +1046,15 @@ mod tests { } #[test] - fn reconcile_running_phases_downgrades_running_to_interrupted() { + fn reconcile_running_phases_auto_resumes_when_worktree_and_session_survive() { use crate::runplan::{RunPhase, RunPlan}; use chrono::{TimeZone, Utc}; use std::path::PathBuf; let dir = std::env::temp_dir().join(format!("loopdeck-runexec-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); + let survived_worktree = dir.join("phase-one-worktree"); + std::fs::create_dir_all(&survived_worktree).unwrap(); let plan = RunPlan { id: "run-1".to_string(), @@ -1098,10 +1065,13 @@ mod tests { environment: Default::default(), wall_clock_secs: 0, stall_policy: Default::default(), + max_concurrency: 4, phases: vec![ RunPhase { execution_id: "p/1".into(), status: RunPhaseStatus::Running, + worktree_path: Some(survived_worktree), + session_id: Some("sess-survived".into()), interview: vec![], interview_status: Default::default(), depends_on: vec![], @@ -1131,7 +1101,18 @@ mod tests { assert!(changed); let reloaded = runplan::load(&dir).unwrap().unwrap(); - assert_eq!(reloaded.phases[0].status, RunPhaseStatus::Interrupted); + // Per the approved restart answer, both survivors mean auto-resume: + // the phase returns to Queued with its saved session pinned. + assert_eq!(reloaded.phases[0].status, RunPhaseStatus::Queued); + assert_eq!( + reloaded.phases[0].resume_session_id.as_deref(), + Some("sess-survived") + ); + assert!(reloaded.phases[0] + .park_payload + .as_deref() + .unwrap() + .contains("resumed after app restart")); assert_eq!(reloaded.phases[1].status, RunPhaseStatus::Queued); // Idempotent: a second pass is a no-op. @@ -1197,6 +1178,7 @@ mod tests { environment: Default::default(), wall_clock_secs: 0, stall_policy: StallPolicy::ContinueIndependent, + max_concurrency: 4, phases, } } @@ -1375,6 +1357,7 @@ mod tests { environment: RunEnvironment::default(), wall_clock_secs: 0, stall_policy: StallPolicy::ContinueIndependent, + max_concurrency: 4, phases, } } @@ -1559,34 +1542,80 @@ mod tests { #[test] fn state_machine_resume_after_restart() { - // Test the reconcile_running_phases behavior: a Running phase left - // on disk from a crash should be downgraded to Interrupted when the - // app restarts. + // A Running phase left on disk from a crash is auto-resumed when its + // worktree and saved provider session both survive; the dependent + // stays Queued and the scheduler gates it at run start. let dir = std::env::temp_dir().join(format!("loopdeck-sm-resume-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); + let survived_worktree = dir.join("phase-one-worktree"); + std::fs::create_dir_all(&survived_worktree).unwrap(); - let plan = make_plan_with_statuses(vec![ + let mut plan = make_plan_with_statuses(vec![ ("phase1", RunPhaseStatus::Running, vec![]), // crashed mid-run ("phase2", RunPhaseStatus::Queued, vec!["phase1"]), ]); + plan.phases[0].worktree_path = Some(survived_worktree); + plan.phases[0].session_id = Some("sess-1".into()); runplan::save(&dir, &plan).unwrap(); - // Simulate app restart: reconcile should downgrade Running → Interrupted let changed = reconcile_running_phases(&dir).unwrap(); assert!(changed); let reconciled = runplan::load(&dir).unwrap().unwrap(); - assert_eq!(reconciled.phases[0].status, RunPhaseStatus::Interrupted); + assert_eq!(reconciled.phases[0].status, RunPhaseStatus::Queued); + assert_eq!( + reconciled.phases[0].resume_session_id.as_deref(), + Some("sess-1") + ); assert_eq!(reconciled.phases[1].status, RunPhaseStatus::Queued); - // Second reconcile is a no-op + // Second reconcile is a no-op (nothing is Running any more). let changed_again = reconcile_running_phases(&dir).unwrap(); assert!(!changed_again); std::fs::remove_dir_all(&dir).ok(); } + /// A crashed phase whose worktree did not survive parks with a reason + /// instead of being retried blindly. + #[test] + fn reconcile_parks_when_the_worktree_did_not_survive() { + let dir = std::env::temp_dir().join(format!("loopdeck-sm-park-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + + let mut plan = make_plan_with_statuses(vec![("phase1", RunPhaseStatus::Running, vec![])]); + plan.phases[0].worktree_path = Some(dir.join("gone-worktree")); + plan.phases[0].session_id = Some("sess-1".into()); + runplan::save(&dir, &plan).unwrap(); + + assert!(reconcile_running_phases(&dir).unwrap()); + let reconciled = runplan::load(&dir).unwrap().unwrap(); + assert_eq!(reconciled.phases[0].status, RunPhaseStatus::Parked); + let reason = reconciled.phases[0].park_payload.as_deref().unwrap(); + assert!(reason.contains("worktree"), "{reason}"); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn reconcile_requeues_waiting_phases_without_a_running_sibling() { + let dir = + std::env::temp_dir().join(format!("loopdeck-wait-restart-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let plan = make_plan_with_statuses(vec![ + ("upstream", RunPhaseStatus::Completed, vec![]), + ("consumer", RunPhaseStatus::Waiting, vec!["upstream"]), + ]); + runplan::save(&dir, &plan).unwrap(); + assert!(reconcile_running_phases(&dir).unwrap()); + let reconciled = runplan::load(&dir).unwrap().unwrap(); + assert_eq!(reconciled.phases[0].status, RunPhaseStatus::Completed); + assert_eq!(reconciled.phases[1].status, RunPhaseStatus::Queued); + assert!(!reconcile_running_phases(&dir).unwrap()); + std::fs::remove_dir_all(&dir).ok(); + } + // ── Waiting on handoff dependencies (`prd-agent-handoff` Phase 2) ─── fn publishing(id: &str, topic: &str) -> runplan::RunPhase { @@ -1603,57 +1632,6 @@ mod tests { ]) } - #[test] - fn recheck_marks_a_consumer_waiting_until_its_upstream_completes() { - let mut plan = handoff_chain(); - - assert!(recheck_waiting_phases(&mut plan).unwrap()); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Waiting); - assert_eq!(plan.phases[0].status, RunPhaseStatus::Queued); - assert_eq!(plan.phases[2].status, RunPhaseStatus::Queued); - // Idempotent once settled. - assert!(!recheck_waiting_phases(&mut plan).unwrap()); - - plan.phases[0].status = RunPhaseStatus::Completed; - assert!(recheck_waiting_phases(&mut plan).unwrap()); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Queued); - } - - #[test] - fn recheck_parks_a_waiting_consumer_when_upstream_is_terminal_bad() { - let mut plan = handoff_chain(); - plan.phases[0].status = RunPhaseStatus::Parked; - plan.phases[0].park_payload = Some("needs a human".into()); - - assert!(recheck_waiting_phases(&mut plan).unwrap()); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Parked); - assert!(plan.phases[1] - .park_payload - .as_deref() - .unwrap() - .contains("upstream handoff phase \"upstream\"")); - } - - #[test] - fn recheck_ignores_dependencies_without_a_handoff_topic() { - let mut plan = plan_with(vec![queued("a", &[]), queued("b", &["a"])]); - assert!(!recheck_waiting_phases(&mut plan).unwrap()); - assert_eq!(plan.phases[1].status, RunPhaseStatus::Queued); - } - - #[test] - fn recheck_settles_a_chained_handoff_in_one_call() { - let mut b = publishing("b", "b-plan"); - b.depends_on = vec!["a".into()]; - let mut plan = plan_with(vec![publishing("a", "a-plan"), b, queued("c", &["b"])]); - plan.phases[0].status = RunPhaseStatus::Completed; - - recheck_waiting_phases(&mut plan).unwrap(); - // b's publisher is done, so b is eligible; c waits on b. - assert_eq!(plan.phases[1].status, RunPhaseStatus::Queued); - assert_eq!(plan.phases[2].status, RunPhaseStatus::Waiting); - } - #[test] fn park_waiting_at_run_end_parks_only_waiting_phases() { let mut plan = handoff_chain(); diff --git a/src-tauri/src/runplan.rs b/src-tauri/src/runplan.rs index 7890694..cc195bb 100644 --- a/src-tauri/src/runplan.rs +++ b/src-tauri/src/runplan.rs @@ -46,12 +46,14 @@ pub enum StallPolicy { pub enum RunPhaseStatus { #[default] Queued, - Running, - /// Blocked on an upstream handoff artifact, not on a human - /// (`prd-agent-handoff` Phase 2). Transient: the executor re-checks it - /// after every turn and parks it only when upstream is terminal-bad or - /// the run ends. + /// Held by the dependency gate: this phase is eligible to run only after + /// every phase in its `depends_on` completes green. The scheduler releases + /// it to `Queued` when the gate opens, or marks it `Failed` when an + /// upstream phase ends non-green. Never set at queue time — the executor + /// computes the initial Waiting set when the run starts, so pre-existing + /// plans still start from all-`Queued`. Waiting, + Running, Parked, Delivered, Completed, @@ -177,6 +179,27 @@ pub struct RunPhase { /// from the execution id. #[serde(default, skip_serializing_if = "Option::is_none")] pub handoff_topic: Option, + /// Isolated worktree this phase executes in. One worktree per in-flight + /// phase keeps the existing session cache keying honest: sessions are + /// keyed by worktree path, so concurrent phases can never share a + /// provider process. `None` until the phase first starts (legacy plans + /// used the single plan-level worktree). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worktree_path: Option, + /// Branch cut for this phase's worktree. Every phase branch is cut from + /// the repository's default branch — no stacking — and opens its own + /// draft PR. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worktree_branch: Option, + /// Provider session id of this phase's last turn. Retained so restart + /// reconciliation can decide whether a saved session still exists before + /// auto-resuming an in-flight phase. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Session id the next turn for this phase resumes. Set only when restart + /// reconciliation auto-resumes a phase; cleared once the turn starts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resume_session_id: Option, } /// Queue-time per-phase staffing choice (`prd-role-foundations` Phase 4): @@ -220,10 +243,20 @@ pub struct RunPlan { pub wall_clock_secs: u64, #[serde(default)] pub stall_policy: StallPolicy, + /// Maximum number of phases the executor may run at the same time. + /// `parallel-scheduling/parallel-batches` adds this knob: the default is + /// four, and the picker may lower or raise it per plan. Absent in older + /// plan files, so the serde default keeps them loadable. + #[serde(default = "default_max_concurrency")] + pub max_concurrency: u32, #[serde(default)] pub phases: Vec, } +fn default_max_concurrency() -> u32 { + limits::DEFAULT_RUN_MAX_CONCURRENCY +} + /// Path to a project's run plan. pub fn run_plan_path(repo_path: &Path) -> PathBuf { repo_path.join(".loopdeck").join("run-plan.yaml") @@ -275,7 +308,7 @@ pub enum PhaseVerdict { Failed, Parked, Running, - /// Blocked on an unmet upstream handoff dependency, not on a human. + /// Still held behind an unsatisfied dependency gate when the run ended. Waiting, } @@ -421,6 +454,7 @@ mod tests { environment: RunEnvironment::default(), wall_clock_secs: 0, stall_policy: StallPolicy::Halt, + max_concurrency: 4, phases: vec![RunPhase { execution_id: "prd-run-queue/phase-1".to_string(), status: RunPhaseStatus::Queued, @@ -460,6 +494,11 @@ phases: assert_eq!(plan.consent, RunConsent::default()); assert_eq!(plan.budgets, RunBudgets::default()); assert_eq!(plan.stall_policy, StallPolicy::ContinueIndependent); + assert_eq!( + plan.max_concurrency, + crate::limits::DEFAULT_RUN_MAX_CONCURRENCY, + "older plans without the field get the four-phase default" + ); assert_eq!(plan.phases.len(), 1); let phase = &plan.phases[0]; assert_eq!(phase.status, RunPhaseStatus::Queued); @@ -522,6 +561,7 @@ phases: environment: RunEnvironment::default(), wall_clock_secs: 3600, stall_policy: StallPolicy::ContinueIndependent, + max_concurrency: 4, phases: vec![ // Completed phase with draft PR RunPhase { @@ -665,6 +705,36 @@ phases: assert!(url.is_none()); } + #[test] + fn report_derives_waiting_verdict_for_a_gated_phase() { + let phase = RunPhase { + execution_id: "x".into(), + status: RunPhaseStatus::Waiting, + ..Default::default() + }; + let (verdict, url) = derive_verdict(&phase); + assert_eq!(verdict, PhaseVerdict::Waiting); + assert!(url.is_none()); + } + + #[test] + fn max_concurrency_round_trips_and_legacy_plans_default_to_four() { + let mut plan = sample_plan(); + plan.max_concurrency = 2; + let yaml = serde_yaml::to_string(&plan).unwrap(); + let back: RunPlan = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!(back.max_concurrency, 2); + + let legacy: RunPlan = serde_yaml::from_str( + "id: run-legacy\nproject: /repo\ncreated: 2026-07-28T09:00:00Z\nphases: []\n", + ) + .unwrap(); + assert_eq!( + legacy.max_concurrency, + crate::limits::DEFAULT_RUN_MAX_CONCURRENCY + ); + } + #[test] fn report_pass_without_pr_url_when_consent_missing() { // Completed with no draft PR in payload → should still be Pass but no URL diff --git a/src-tauri/src/scheduler.rs b/src-tauri/src/scheduler.rs new file mode 100644 index 0000000..b2e4e74 --- /dev/null +++ b/src-tauri/src/scheduler.rs @@ -0,0 +1,825 @@ +//! Dependency-aware phase scheduler — `prd-parallel-scheduling`. +//! +//! Pure scheduling policy for run-plan phases: graph validation (no cycles, +//! no dangling edges), the Waiting gate that holds dependents until upstream +//! phases land green, the ready-set the executor draws from, dependent +//! failure propagation, and the per-phase watchdog budget. No filesystem, +//! Tauri, or clock dependencies — the executor owns persistence and the +//! actual turns, so every rule here is unit-testable. +//! +//! Design (PRD Design stub): a **ready-set with a semaphore**, not +//! topological batches. The executor asks which phases are ready, starts up +//! to `RunPlan::max_concurrency` of them, and re-evaluates whenever one +//! finishes. That lets a fast phase's dependents start while a slow sibling +//! on another branch is still running. + +use crate::error::AppError; +use crate::runplan::{RunPhase, RunPhaseStatus, RunPlan, StallPolicy}; +use std::collections::{HashSet, VecDeque}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// A phase counts as green only when it actually delivered. +pub fn is_success(status: RunPhaseStatus) -> bool { + matches!( + status, + RunPhaseStatus::Completed | RunPhaseStatus::Delivered + ) +} + +/// A phase will never change status again on its own. +pub fn is_terminal(status: RunPhaseStatus) -> bool { + matches!( + status, + RunPhaseStatus::Completed + | RunPhaseStatus::Delivered + | RunPhaseStatus::Parked + | RunPhaseStatus::Failed + | RunPhaseStatus::Interrupted + | RunPhaseStatus::Killed + ) +} + +/// Whether a phase still needs the scheduler's attention. +pub fn is_active(status: RunPhaseStatus) -> bool { + matches!( + status, + RunPhaseStatus::Queued | RunPhaseStatus::Waiting | RunPhaseStatus::Running + ) +} + +fn status_label(status: RunPhaseStatus) -> &'static str { + match status { + RunPhaseStatus::Queued => "queued", + RunPhaseStatus::Waiting => "waiting", + RunPhaseStatus::Running => "running", + RunPhaseStatus::Parked => "parked", + RunPhaseStatus::Delivered => "delivered", + RunPhaseStatus::Completed => "completed", + RunPhaseStatus::Failed => "failed", + RunPhaseStatus::Interrupted => "interrupted", + RunPhaseStatus::Killed => "killed", + } +} + +/// Validate a plan's dependency graph before any turn starts. A hand-edited +/// plan (or a future edge editor) must never reach the executor with a +/// dangling, self-referential, or cyclic edge: a cycle would leave every +/// member Waiting forever, quietly wasting the overnight window. +pub fn validate_dependency_graph(phases: &[RunPhase]) -> Result<(), AppError> { + let mut seen = HashSet::new(); + for phase in phases { + if phase.execution_id.trim().is_empty() { + return Err(AppError::RunPlan( + "every run-plan phase needs a non-empty execution id".into(), + )); + } + if !seen.insert(phase.execution_id.as_str()) { + return Err(AppError::RunPlan(format!( + "phase \"{}\" appears more than once in the plan", + phase.execution_id + ))); + } + } + for phase in phases { + for dependency in &phase.depends_on { + if dependency == &phase.execution_id { + return Err(AppError::RunPlan(format!( + "phase \"{}\" depends on itself", + phase.execution_id + ))); + } + if !seen.contains(dependency.as_str()) { + return Err(AppError::RunPlan(format!( + "phase \"{}\" depends on \"{dependency}\", which is not part of this plan", + phase.execution_id + ))); + } + } + } + topological_order(phases).map(|_| ()) +} + +/// Deterministic topological order (authored order breaks ties). Errors with +/// the cycle members when the graph has a cycle — this is the single cycle +/// detector both plan creation and run start use. +pub fn topological_order(phases: &[RunPhase]) -> Result, AppError> { + let mut remaining: Vec = (0..phases.len()).collect(); + let mut resolved: HashSet<&str> = HashSet::new(); + let mut order = Vec::with_capacity(phases.len()); + while !remaining.is_empty() { + let before = remaining.len(); + let mut next = Vec::new(); + // Only nodes whose dependencies were already resolved *before* this + // pass may resolve now. Resolving mid-pass would let a cycle resolve + // itself whenever its back-edge happens to sit at a later index. + let mut newly = Vec::new(); + for &index in &remaining { + let phase = &phases[index]; + let ready = phase + .depends_on + .iter() + .all(|dependency| resolved.contains(dependency.as_str())); + if ready { + newly.push(index); + } else { + next.push(index); + } + } + for index in newly { + let phase = &phases[index]; + resolved.insert(phase.execution_id.as_str()); + order.push(phase.execution_id.clone()); + } + remaining = next; + if remaining.len() == before { + let members: Vec<&str> = remaining + .iter() + .map(|&index| phases[index].execution_id.as_str()) + .collect(); + return Err(AppError::RunPlan(format!( + "dependency cycle in the run plan among: {}", + members.join(", ") + ))); + } + } + Ok(order) +} + +/// Hold every `Queued` phase with an unsatisfied dependency in `Waiting`. +/// Called once when the executor starts, so legacy plans (all phases +/// `Queued`) keep their on-disk queue shape until the run actually begins. +/// Returns how many phases were moved to Waiting. +pub fn mark_waiting(plan: &mut RunPlan) -> usize { + let successful: HashSet = plan + .phases + .iter() + .filter(|phase| is_success(phase.status)) + .map(|phase| phase.execution_id.clone()) + .collect(); + let mut moved = 0; + for phase in &mut plan.phases { + if phase.status == RunPhaseStatus::Queued + && phase + .depends_on + .iter() + .any(|dependency| !successful.contains(dependency)) + { + phase.status = RunPhaseStatus::Waiting; + moved += 1; + } + } + moved +} + +/// Recompute the dependency gate after any phase reaches a terminal state: +/// +/// - `Waiting` phases whose dependencies are all green are released to +/// `Queued`. +/// - `Waiting`/`Queued` phases with an upstream that ended non-green become +/// `Failed`, naming that upstream in `park_payload` (the approved stall +/// answer: *fail dependents, keep going*). +/// +/// Runs to a fixed point so a failed phase can fail its own dependents in the +/// same pass. Never touches `Running` phases: an independent in-flight phase +/// always finishes on its own. Returns `(released, failed)` counts. +pub fn reconcile_gates(plan: &mut RunPlan) -> (usize, usize) { + let mut released = 0; + let mut failed = 0; + // At most one pass per phase; a wider bound cannot loop forever. + for _ in 0..=plan.phases.len() { + let snapshot: Vec<(String, RunPhaseStatus, Option)> = plan + .phases + .iter() + .map(|phase| { + ( + phase.execution_id.clone(), + phase.status, + phase.park_payload.clone(), + ) + }) + .collect(); + let mut changes: Vec<(usize, RunPhaseStatus, Option)> = Vec::new(); + for (index, phase) in plan.phases.iter().enumerate() { + if !matches!( + phase.status, + RunPhaseStatus::Queued | RunPhaseStatus::Waiting + ) { + continue; + } + let mut all_green = true; + let mut failure: Option<(String, String)> = None; + for dependency in &phase.depends_on { + let Some((upstream_id, status, payload)) = snapshot + .iter() + .find(|(candidate_id, _, _)| candidate_id == dependency) + else { + // Validation rejects dangling edges; treat a missing edge + // defensively as not-yet-green rather than running early. + all_green = false; + continue; + }; + if is_success(*status) { + continue; + } + all_green = false; + if is_terminal(*status) { + let label = status_label(*status).to_string(); + failure = Some(( + upstream_id.clone(), + match payload.as_deref() { + Some(reason) if !reason.is_empty() => format!("{label}: {reason}"), + _ => label, + }, + )); + } + } + if let Some((upstream_id, detail)) = failure { + changes.push(( + index, + RunPhaseStatus::Failed, + Some(format!( + "failed: upstream phase \"{upstream_id}\" did not complete green ({detail})" + )), + )); + } else if all_green && phase.status == RunPhaseStatus::Waiting { + changes.push((index, RunPhaseStatus::Queued, None)); + } + } + if changes.is_empty() { + break; + } + for (index, status, payload) in changes { + plan.phases[index].status = status; + if status == RunPhaseStatus::Failed { + failed += 1; + } else if status == RunPhaseStatus::Queued { + released += 1; + } + plan.phases[index].park_payload = payload; + } + } + (released, failed) +} + +/// Mark every not-yet-started dependent of `failed_id` as `Failed`, with the +/// upstream reason. `StallPolicy::Halt` additionally parks every remaining +/// `Queued`/`Waiting` phase that is not already failed, preserving strict +/// sequence. `Running` phases are never touched — the approved gated-start +/// answer keeps independent in-flight work going. +/// +/// Returns the number of phases whose status changed. +pub fn fail_dependents(plan: &mut RunPlan, failed_id: &str, reason: &str) -> usize { + let affected: HashSet = dependents_of(plan, failed_id).into_iter().collect(); + + let halt = plan.stall_policy == StallPolicy::Halt; + let mut changed = 0; + for phase in &mut plan.phases { + if !matches!( + phase.status, + RunPhaseStatus::Queued | RunPhaseStatus::Waiting + ) { + continue; + } + if affected.contains(&phase.execution_id) { + phase.status = RunPhaseStatus::Failed; + phase.park_payload = Some(format!( + "failed: upstream phase \"{failed_id}\" did not complete green ({reason})" + )); + changed += 1; + } else if halt { + phase.status = RunPhaseStatus::Parked; + phase.park_payload = Some(format!( + "halted: run stopped after phase \"{failed_id}\": {reason}" + )); + changed += 1; + } + } + changed +} + +/// The phases the executor may start right now, in authored order, capped at +/// `limit`. A phase is ready when it is `Queued`, not already in flight, and +/// every dependency is green. +pub fn select_ready(plan: &RunPlan, in_flight: &HashSet, limit: usize) -> Vec { + if limit == 0 { + return Vec::new(); + } + let successful: HashSet<&str> = plan + .phases + .iter() + .filter(|phase| is_success(phase.status)) + .map(|phase| phase.execution_id.as_str()) + .collect(); + plan.phases + .iter() + .enumerate() + .filter(|(_, phase)| phase.status == RunPhaseStatus::Queued) + .filter(|(_, phase)| !in_flight.contains(&phase.execution_id)) + .filter(|(_, phase)| { + phase + .depends_on + .iter() + .all(|dependency| successful.contains(dependency.as_str())) + }) + .map(|(index, _)| index) + .take(limit) + .collect() +} + +/// Per-phase watchdog: whichever cap expires first — this phase's own +/// wall-clock budget or the run's remaining wall-clock budget. Returns the +/// timeout and the reason to record when it fires. +pub fn phase_watchdog( + phase_timeout: Duration, + run_remaining: Duration, +) -> (Duration, &'static str) { + if run_remaining <= phase_timeout { + (run_remaining, "total run wall-clock budget exceeded") + } else { + (phase_timeout, "phase wall-clock budget exceeded") + } +} + +/// Restart-reconciliation decision for one phase left `Running` on disk. +/// Auto-resume requires both survivors: the worktree the phase ran in and the +/// saved provider session. Anything less parks the phase with a reason for +/// the morning report instead of guessing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResumeDecision { + AutoResume, + Park, +} + +pub fn classify_interrupted_phase( + worktree_survives: bool, + session_survives: bool, +) -> ResumeDecision { + if worktree_survives && session_survives { + ResumeDecision::AutoResume + } else { + ResumeDecision::Park + } +} + +/// Deterministic per-phase branch name. Every phase branch is cut from the +/// repository default branch — no stacking — and carries its authoured index +/// so two different execution ids that normalize to the same slug can never +/// collide. +pub fn phase_branch_name(plan_id: &str, execution_id: &str, index: usize) -> String { + let suffix = plan_id.rsplit('-').next().unwrap_or(plan_id); + let slug: String = execution_id + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character + } else { + '-' + } + }) + .collect::() + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + let slug = if slug.is_empty() { + "phase".to_string() + } else { + slug + }; + format!("run/{slug}-p{}-{suffix}", index + 1) +} + +/// Managed worktree path for a phase branch (inside `.loopdeck/runs/`). +pub fn phase_worktree_path(root: &Path, branch: &str) -> PathBuf { + root.join(".loopdeck").join("runs").join(branch) +} + +/// A plan is schedulable while any phase is still Queued, Waiting, or +/// Running. Used by the executor loop's termination check and by `queue_run`. +pub fn has_active_phases(plan: &RunPlan) -> bool { + plan.phases.iter().any(|phase| is_active(phase.status)) +} + +/// Breadth-first set of phases that transitively depend on `execution_id` +/// (excluding itself). Exposed for the morning report and tests. +pub fn dependents_of(plan: &RunPlan, execution_id: &str) -> Vec { + let mut affected: HashSet = HashSet::from([execution_id.to_string()]); + let mut queue = VecDeque::from([execution_id.to_string()]); + while let Some(current) = queue.pop_front() { + for phase in &plan.phases { + if phase + .depends_on + .iter() + .any(|dependency| dependency == ¤t) + && affected.insert(phase.execution_id.clone()) + { + queue.push_back(phase.execution_id.clone()); + } + } + } + affected.remove(execution_id); + let mut result: Vec = affected.into_iter().collect(); + result.sort(); + result +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runplan::{RunBudgets, RunConsent, RunPhase}; + use chrono::Utc; + + fn phase(id: &str, depends_on: &[&str]) -> RunPhase { + RunPhase { + execution_id: id.to_string(), + depends_on: depends_on.iter().map(|value| value.to_string()).collect(), + ..Default::default() + } + } + + fn plan(phases: Vec) -> RunPlan { + RunPlan { + id: "run-abc12345".into(), + project: PathBuf::from("/repo"), + created: Utc::now(), + consent: RunConsent::default(), + budgets: RunBudgets::default(), + environment: Default::default(), + wall_clock_secs: 0, + stall_policy: StallPolicy::ContinueIndependent, + max_concurrency: 4, + phases, + } + } + + fn diamond() -> RunPlan { + // a → b, a → c, (b, c) → d + plan(vec![ + phase("a", &[]), + phase("b", &["a"]), + phase("c", &["a"]), + phase("d", &["b", "c"]), + ]) + } + + fn set_status(plan: &mut RunPlan, id: &str, status: RunPhaseStatus) { + plan.phases + .iter_mut() + .find(|phase| phase.execution_id == id) + .expect("phase exists") + .status = status; + } + + #[test] + fn diamond_graph_validates_and_orders_roots_first() { + let mut plan = diamond(); + validate_dependency_graph(&plan.phases).expect("diamond is acyclic"); + let order = topological_order(&plan.phases).expect("order"); + assert_eq!(order, vec!["a", "b", "c", "d"]); + + // Only the root is ready before anything lands. + mark_waiting(&mut plan); + assert_eq!(plan.phases[0].status, RunPhaseStatus::Queued); + assert!(plan.phases[1..] + .iter() + .all(|phase| phase.status == RunPhaseStatus::Waiting)); + let ready = select_ready(&plan, &HashSet::new(), 4); + assert_eq!(ready, vec![0]); + } + + #[test] + fn diamond_releases_a_leaf_only_when_both_parents_are_green() { + let mut plan = diamond(); + mark_waiting(&mut plan); + set_status(&mut plan, "a", RunPhaseStatus::Completed); + + let (released, failed) = reconcile_gates(&mut plan); + assert_eq!((released, failed), (2, 0)); + assert_eq!(plan.phases[1].status, RunPhaseStatus::Queued); + assert_eq!(plan.phases[2].status, RunPhaseStatus::Queued); + assert_eq!(plan.phases[3].status, RunPhaseStatus::Waiting); + + set_status(&mut plan, "b", RunPhaseStatus::Completed); + let (released, _) = reconcile_gates(&mut plan); + assert_eq!(released, 0); + assert_eq!(plan.phases[3].status, RunPhaseStatus::Waiting); + + set_status(&mut plan, "c", RunPhaseStatus::Delivered); + let (released, _) = reconcile_gates(&mut plan); + assert_eq!(released, 1); + assert_eq!(plan.phases[3].status, RunPhaseStatus::Queued); + } + + #[test] + fn non_green_upstream_fails_every_waiting_dependent_transitively() { + let mut plan = diamond(); + mark_waiting(&mut plan); + set_status(&mut plan, "a", RunPhaseStatus::Completed); + reconcile_gates(&mut plan); + set_status(&mut plan, "b", RunPhaseStatus::Parked); + plan.phases[1].park_payload = Some("verify verdict: BLOCK".into()); + + let (_, failed) = reconcile_gates(&mut plan); + assert_eq!(failed, 1); + let d = &plan.phases[3]; + assert_eq!(d.status, RunPhaseStatus::Failed); + let reason = d.park_payload.as_deref().expect("reason"); + assert!(reason.contains("upstream phase \"b\""), "{reason}"); + assert!(reason.contains("verify verdict: BLOCK"), "{reason}"); + // The independent sibling c is untouched. + assert_eq!(plan.phases[2].status, RunPhaseStatus::Queued); + } + + #[test] + fn fail_dependents_keeps_independent_in_flight_phases_running() { + let mut plan = plan(vec![phase("a", &[]), phase("b", &["a"]), phase("c", &[])]); + set_status(&mut plan, "a", RunPhaseStatus::Killed); + set_status(&mut plan, "b", RunPhaseStatus::Waiting); + set_status(&mut plan, "c", RunPhaseStatus::Running); + + let changed = fail_dependents(&mut plan, "a", "phase token budget exceeded"); + assert_eq!(changed, 1); + assert_eq!(plan.phases[1].status, RunPhaseStatus::Failed); + assert!(plan.phases[1] + .park_payload + .as_deref() + .unwrap() + .contains("phase token budget exceeded")); + // A budget kill ends only that phase; the in-flight sibling keeps going. + assert_eq!(plan.phases[2].status, RunPhaseStatus::Running); + } + + #[test] + fn halt_policy_parks_every_remaining_phase() { + let mut plan = plan(vec![phase("a", &[]), phase("b", &["a"]), phase("c", &[])]); + plan.stall_policy = StallPolicy::Halt; + set_status(&mut plan, "a", RunPhaseStatus::Failed); + set_status(&mut plan, "b", RunPhaseStatus::Waiting); + set_status(&mut plan, "c", RunPhaseStatus::Queued); + + fail_dependents(&mut plan, "a", "turn error"); + assert_eq!(plan.phases[1].status, RunPhaseStatus::Failed); + assert_eq!(plan.phases[2].status, RunPhaseStatus::Parked); + assert!(plan.phases[2] + .park_payload + .as_deref() + .unwrap() + .starts_with("halted:")); + } + + #[test] + fn cycle_is_rejected_and_names_the_members() { + let cyclic = plan(vec![ + phase("a", &["c"]), + phase("b", &["a"]), + phase("c", &["b"]), + ]); + let error = validate_dependency_graph(&cyclic.phases).expect_err("cycle rejected"); + let message = error.to_string(); + assert!(message.contains("cycle"), "{message}"); + assert!(message.contains('a') && message.contains('b') && message.contains('c')); + } + + #[test] + fn self_dependency_and_dangling_edges_are_rejected() { + let self_dep = plan(vec![phase("a", &["a"])]); + assert!(validate_dependency_graph(&self_dep.phases) + .expect_err("self dependency") + .to_string() + .contains("depends on itself")); + + let dangling = plan(vec![phase("a", &["ghost"])]); + assert!(validate_dependency_graph(&dangling.phases) + .expect_err("dangling") + .to_string() + .contains("not part of this plan")); + } + + #[test] + fn select_ready_respects_the_concurrency_limit() { + let plan = plan(vec![phase("a", &[]), phase("b", &[]), phase("c", &[])]); + assert_eq!(select_ready(&plan, &HashSet::new(), 2), vec![0, 1]); + let in_flight = HashSet::from(["a".to_string()]); + assert_eq!(select_ready(&plan, &in_flight, 4), vec![1, 2]); + } + + #[test] + fn watchdog_picks_the_nearer_cap() { + let phase = Duration::from_secs(90); + let run = Duration::from_secs(30); + assert_eq!( + phase_watchdog(phase, run), + (run, "total run wall-clock budget exceeded") + ); + assert_eq!( + phase_watchdog(phase, Duration::from_secs(300)), + (phase, "phase wall-clock budget exceeded") + ); + } + + #[test] + fn restart_decision_needs_both_worktree_and_session() { + assert_eq!( + classify_interrupted_phase(true, true), + ResumeDecision::AutoResume + ); + assert_eq!( + classify_interrupted_phase(true, false), + ResumeDecision::Park + ); + assert_eq!( + classify_interrupted_phase(false, true), + ResumeDecision::Park + ); + } + + #[test] + fn phase_branches_are_unique_even_when_slugs_collide() { + let first = phase_branch_name("run-abc123", "prd-x/loop-a", 0); + let second = phase_branch_name("run-abc123", "prd-x-loop-a", 1); + assert_ne!(first, second); + assert!(first.starts_with("run/prd-x-loop-a-p1-")); + assert!(second.starts_with("run/prd-x-loop-a-p2-")); + } + + #[test] + fn dependents_of_walks_the_whole_chain() { + let plan = plan(vec![ + phase("a", &[]), + phase("b", &["a"]), + phase("c", &["b"]), + phase("d", &[]), + ]); + assert_eq!(dependents_of(&plan, "a"), vec!["b", "c"]); + assert!(dependents_of(&plan, "d").is_empty()); + } + + #[test] + fn mark_waiting_leaves_roots_and_completed_phases_alone() { + let mut plan = diamond(); + plan.phases[0].status = RunPhaseStatus::Queued; + plan.phases[2].status = RunPhaseStatus::Completed; + let moved = mark_waiting(&mut plan); + assert_eq!(moved, 2); // b and d; c is already completed + assert_eq!(plan.phases[0].status, RunPhaseStatus::Queued); + assert_eq!(plan.phases[2].status, RunPhaseStatus::Completed); + assert_eq!(plan.phases[1].status, RunPhaseStatus::Waiting); + assert_eq!(plan.phases[3].status, RunPhaseStatus::Waiting); + } + + // ── Property tests over generated graphs (`parallel-scheduling/tests`) ── + // + // A deterministic 64-bit LCG keeps the "random" graphs reproducible in CI + // without adding a property-test dependency. Each case is a real graph + // property, not a snapshot: every generated DAG must validate, and the + // ready-set simulation must eventually green every phase without ever + // starting one before its dependencies. + + struct Lcg(u64); + + impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + } + + /// Generate a DAG: phase `i` may depend only on phases with a lower + /// index, which makes a cycle impossible by construction. + fn random_dag(seed: u64, size: usize) -> RunPlan { + let mut rng = Lcg(seed); + let ids: Vec = (0..size).map(|index| format!("p{index}")).collect(); + let phases = ids + .iter() + .enumerate() + .map(|(index, id)| { + let mut deps = Vec::new(); + for candidate in 0..index { + if rng.below(4) == 0 { + deps.push(ids[candidate].clone()); + } + } + RunPhase { + execution_id: id.clone(), + depends_on: deps, + ..Default::default() + } + }) + .collect(); + plan(phases) + } + + #[test] + fn property_every_generated_dag_validates_and_runs_to_green() { + for seed in 0..64 { + let mut plan = random_dag(seed, 8); + validate_dependency_graph(&plan.phases) + .unwrap_or_else(|error| panic!("seed {seed}: {error}")); + mark_waiting(&mut plan); + let mut in_flight: HashSet = HashSet::new(); + let mut executed = 0; + // Simulate the executor with a concurrency of three. + for _ in 0..(plan.phases.len() * 4) { + let ready = select_ready(&plan, &in_flight, 3); + for start in ready { + let phase = &plan.phases[start]; + for dependency in &phase.depends_on { + let upstream = plan + .phases + .iter() + .find(|candidate| &candidate.execution_id == dependency) + .expect("dependency exists"); + assert!( + is_success(upstream.status), + "seed {seed}: started {} before {dependency} was green", + phase.execution_id + ); + } + in_flight.insert(phase.execution_id.clone()); + } + // Complete one in-flight phase per tick. + if let Some(completed) = in_flight.iter().next().cloned() { + in_flight.remove(&completed); + let index = plan + .phases + .iter() + .position(|candidate| candidate.execution_id == completed) + .expect("in flight phase exists"); + plan.phases[index].status = RunPhaseStatus::Completed; + executed += 1; + reconcile_gates(&mut plan); + } else if !has_active_phases(&plan) { + break; + } + } + assert_eq!( + executed, + plan.phases.len(), + "seed {seed}: every phase must run exactly once" + ); + assert!(plan.phases.iter().all(|phase| is_success(phase.status))); + } + } + + #[test] + fn property_every_generated_cycle_is_rejected() { + // Add one back-edge to an acyclic graph: a cycle must always be + // caught, whichever edge closes it. + for seed in 0..32 { + let mut plan = random_dag(seed, 6); + let size = plan.phases.len(); + // Force a chain so the back-edge below is guaranteed to close a + // cycle through the whole graph. + for index in 1..size { + let previous = plan.phases[index - 1].execution_id.clone(); + plan.phases[index].depends_on.push(previous); + } + // The head depends on the tail: every node is now on the cycle. + let last_id = plan.phases[size - 1].execution_id.clone(); + plan.phases[0].depends_on.push(last_id); + let error = validate_dependency_graph(&plan.phases) + .expect_err("a back-edge or self-edge must be rejected"); + let message = error.to_string(); + assert!( + message.contains("cycle") || message.contains("depends on itself"), + "seed {seed}: unexpected rejection reason: {message}" + ); + } + } + + #[test] + fn property_restart_decisions_cover_all_survivor_combinations() { + for worktree in [false, true] { + for session in [false, true] { + let expected = if worktree && session { + ResumeDecision::AutoResume + } else { + ResumeDecision::Park + }; + assert_eq!( + classify_interrupted_phase(worktree, session), + expected, + "worktree={worktree} session={session}" + ); + } + } + } + + #[test] + fn has_active_phases_follows_terminal_states() { + let mut plan = diamond(); + assert!(has_active_phases(&plan)); + for phase in &mut plan.phases { + phase.status = RunPhaseStatus::Completed; + } + assert!(!has_active_phases(&plan)); + } +} diff --git a/src-tauri/src/transition.rs b/src-tauri/src/transition.rs index 721ba08..27c25ab 100644 --- a/src-tauri/src/transition.rs +++ b/src-tauri/src/transition.rs @@ -55,7 +55,11 @@ pub fn transition_run(from: RunStatus, to: RunStatus) -> Result matches!( to, - RunStatus::Parked + // Restart reconciliation auto-resumes a recoverable in-flight + // phase by re-queuing it (`parallel-scheduling/restart-reconcile`); + // a phase that is not recoverable parks instead. + RunStatus::Queued + | RunStatus::Parked | RunStatus::Cancelled | RunStatus::Killed | RunStatus::Interrupted @@ -201,6 +205,9 @@ mod tests { ( RunStatus::Running, &[ + // Restart reconciliation re-queues a recoverable + // in-flight phase instead of interrupting it. + RunStatus::Queued, RunStatus::Parked, RunStatus::Cancelled, RunStatus::Killed, @@ -348,6 +355,30 @@ mod tests { ); } + #[test] + fn phase_restart_reconciliation_can_auto_resume_running_to_queued() { + assert_eq!( + transition_phase(RunPhaseStatus::Running, RunPhaseStatus::Queued).unwrap(), + RunPhaseStatus::Queued + ); + } + + #[test] + fn dependency_gate_edges_hold_and_release_a_phase() { + assert_eq!( + transition_phase(RunPhaseStatus::Queued, RunPhaseStatus::Waiting).unwrap(), + RunPhaseStatus::Waiting + ); + assert_eq!( + transition_phase(RunPhaseStatus::Waiting, RunPhaseStatus::Queued).unwrap(), + RunPhaseStatus::Queued + ); + assert_eq!( + transition_phase(RunPhaseStatus::Waiting, RunPhaseStatus::Running).unwrap(), + RunPhaseStatus::Running + ); + } + fn assert_invalid( result: Result, entity: &str, diff --git a/src/components/detail/PlanTonightWizard.tsx b/src/components/detail/PlanTonightWizard.tsx index f66857f..7222d12 100644 --- a/src/components/detail/PlanTonightWizard.tsx +++ b/src/components/detail/PlanTonightWizard.tsx @@ -116,6 +116,8 @@ export function PlanTonightWizard({ const [phaseTokenCap, setPhaseTokenCap] = useState("500000"); const [phaseMinutes, setPhaseMinutes] = useState("90"); const [runHours, setRunHours] = useState("8"); + /** Concurrent phase turns (`prd-parallel-scheduling` Phase 2). */ + const [maxConcurrency, setMaxConcurrency] = useState("4"); const [creating, setCreating] = useState(false); const [starting, setStarting] = useState(false); const [plan, setPlan] = useState(null); @@ -155,6 +157,7 @@ export function PlanTonightWizard({ setPendingQuestion(null); setConsentConfirmed(false); setAgentByPhase({}); + setMaxConcurrency("4"); }, [open]); const setPhaseAgent = (executionId: string, agentId: string) => { @@ -220,6 +223,10 @@ export function PlanTonightWizard({ const handleNextFromPhases = async () => { setCreating(true); try { + const concurrency = Number(maxConcurrency); + if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 16) { + throw new Error("Concurrent phases must be a whole number from 1 to 16"); + } const created = await api.createRunPlan( projectPath, selected, @@ -227,6 +234,8 @@ export function PlanTonightWizard({ draftPrAuthorized, budgetsFromInputs(), assignmentsFromSelection(), + [], + Number(maxConcurrency), ); setPlan(created); setStep(2); @@ -388,6 +397,8 @@ export function PlanTonightWizard({ setPhaseMinutes={setPhaseMinutes} runHours={runHours} setRunHours={setRunHours} + maxConcurrency={maxConcurrency} + setMaxConcurrency={setMaxConcurrency} onToggle={toggleSelected} /> )} @@ -494,6 +505,8 @@ function StepPhases({ setPhaseMinutes, runHours, setRunHours, + maxConcurrency, + setMaxConcurrency, onToggle, }: { entries: PickerEntry[]; @@ -512,6 +525,8 @@ function StepPhases({ setPhaseMinutes: (v: string) => void; runHours: string; setRunHours: (v: string) => void; + maxConcurrency: string; + setMaxConcurrency: (v: string) => void; onToggle: (id: string) => void; }) { let lastGroup = ""; @@ -637,6 +652,20 @@ function StepPhases({ className="h-7 w-12 rounded border border-border bg-background px-1.5 text-xs text-foreground" /> +

Unattended mode automatically allows safe project-scoped actions and plan execution. @@ -855,6 +884,7 @@ function StepConsent({ {Math.floor((plan.budgets.total_run_wall_clock_secs ?? 0) / 3600)} h {plan.phases.length} + {plan.max_concurrency} {hasPendingInterview && ( diff --git a/src/components/detail/RunQueuePanel.tsx b/src/components/detail/RunQueuePanel.tsx index dd9b123..69761f2 100644 --- a/src/components/detail/RunQueuePanel.tsx +++ b/src/components/detail/RunQueuePanel.tsx @@ -299,11 +299,15 @@ export function RunQueuePanel({ plan?.phases.filter((p) => ["parked", "failed", "interrupted", "killed"].includes(p.status), ).length ?? 0; - const hasQueuedPhase = plan?.phases.some((p) => p.status === "queued") ?? false; + // `waiting` phases are held by the dependency gate but still count as work + // the scheduler will run, so Start must stay enabled for them. + const isRunnable = (p: { status: string }) => + p.status === "queued" || p.status === "waiting"; + const hasQueuedPhase = plan?.phases.some(isRunnable) ?? false; const hasPendingInterview = - plan?.phases.some((p) => p.status === "queued" && p.interview_status === "pending") ?? false; + plan?.phases.some((p) => isRunnable(p) && p.interview_status === "pending") ?? false; const pendingInterviewCount = - plan?.phases.filter((p) => p.status === "queued" && p.interview_status === "pending").length ?? 0; + plan?.phases.filter((p) => isRunnable(p) && p.interview_status === "pending").length ?? 0; const canStart = !isRunning && hasQueuedPhase && !hasPendingInterview; const startDisabledReason = isRunning ? "A run is already in progress" diff --git a/src/lib/nightRun.ts b/src/lib/nightRun.ts index 9fa9975..15d0758 100644 --- a/src/lib/nightRun.ts +++ b/src/lib/nightRun.ts @@ -8,8 +8,8 @@ import type { AskUserQuestionSpec, Epic, PhaseVerdict, RunPhase, RunPhaseStatus, export const STATUS_LABEL: Record = { queued: "queued", + waiting: "waiting", running: "running", - waiting: "waiting on upstream handoff", parked: "parked", delivered: "needs linking", completed: "completed", @@ -20,8 +20,8 @@ export const STATUS_LABEL: Record = { export const STATUS_COLOR: Record = { queued: "var(--muted-foreground)", - running: "var(--primary)", waiting: "var(--muted-foreground)", + running: "var(--primary)", parked: "var(--warning)", delivered: "var(--warning)", completed: "var(--success)", diff --git a/src/lib/tauri.ts b/src/lib/tauri.ts index cefb6a7..a077bfc 100644 --- a/src/lib/tauri.ts +++ b/src/lib/tauri.ts @@ -282,7 +282,9 @@ export async function exportExecutionSummary(path: string): Promise { * `handoffTopics` (prd-agent-handoff Phase 1) opts phases into a contract * artifact; unlisted phases emit nothing. Each topic is a unique kebab-case * slug fixed at queue time. - * Rust: create_run_plan(path, execution_ids, stall_policy, draft_pr_authorized, budgets, assignments, handoff_topics) -> Result + * `maxConcurrency` (prd-parallel-scheduling Phase 2) bounds concurrent phase + * turns; omit to keep the backend's four-phase default. + * Rust: create_run_plan(path, execution_ids, stall_policy, draft_pr_authorized, budgets, assignments, handoff_topics, max_concurrency) -> Result */ export async function createRunPlan( path: string, @@ -292,6 +294,7 @@ export async function createRunPlan( budgets: RunBudgets, assignments: PhaseAgentAssignment[] = [], handoffTopics: PhaseHandoffTopic[] = [], + maxConcurrency?: number, ): Promise { return invoke("create_run_plan", { path, @@ -301,6 +304,7 @@ export async function createRunPlan( budgets, assignments, handoffTopics, + maxConcurrency, }); } diff --git a/src/types/index.ts b/src/types/index.ts index 56137be..88aeeca 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -553,10 +553,9 @@ export type StallPolicy = "continue_independent" | "halt"; /** serde `rename_all = "lowercase"`. Where a queued phase is in the run. */ export type RunPhaseStatus = | "queued" - | "running" - /** Blocked on an unmet upstream handoff dependency, not on a human - * (`prd-agent-handoff` Phase 2). Transient. */ + /** Held by the dependency gate until every `depends_on` phase lands green. */ | "waiting" + | "running" | "parked" | "delivered" | "completed" @@ -630,6 +629,15 @@ export interface RunPhase { * (`prd-agent-handoff` Phase 1). Absent = the phase has no artifact * contract; the executor emits nothing for it. */ handoff_topic?: string; + /** This phase's own isolated worktree (`prd-parallel-scheduling`). One + * worktree per in-flight phase keeps provider sessions keyed honestly. */ + worktree_path?: string; + /** Branch cut for the phase worktree from the repository default branch. */ + worktree_branch?: string; + /** Provider session id of this phase's last turn. */ + session_id?: string; + /** Session id the next turn resumes after a safe restart auto-resume. */ + resume_session_id?: string; } /** Queue-time per-phase staffing choice passed to `create_run_plan` @@ -660,6 +668,8 @@ export interface RunPlan { environment: RunEnvironment; wall_clock_secs: number; stall_policy: StallPolicy; + /** Maximum phases the executor may run at the same time. Defaults to 4. */ + max_concurrency: number; phases: RunPhase[]; }