From 9856b1fc73af441ee7c21458ca39e48a4c31e01f Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:09:07 -0400 Subject: [PATCH 01/16] Gather Claude and Codex behind Harness adapters (#434) --- src/args.rs | 2 +- src/config.rs | 2 +- src/harness.rs | 317 +++--------------- src/harness/adapter.rs | 87 +++++ src/harness/claude.rs | 161 +++++++++ src/harness/claude/stream.rs | 173 ++++++++++ src/harness/codex.rs | 168 ++++++++++ src/harness/codex/catalog.rs | 162 +++++++++ .../codex.rs => harness/codex/stream.rs} | 2 +- src/progress.rs | 190 +---------- src/session.rs | 124 +++---- src/setup.rs | 9 +- src/setup/questions.rs | 93 +---- src/skills.rs | 16 +- 14 files changed, 872 insertions(+), 634 deletions(-) create mode 100644 src/harness/adapter.rs create mode 100644 src/harness/claude.rs create mode 100644 src/harness/claude/stream.rs create mode 100644 src/harness/codex.rs create mode 100644 src/harness/codex/catalog.rs rename src/{progress/codex.rs => harness/codex/stream.rs} (99%) diff --git a/src/args.rs b/src/args.rs index 61195ec6b..b445828cf 100644 --- a/src/args.rs +++ b/src/args.rs @@ -307,7 +307,7 @@ fn ask_harness(harness: &mut Option, arg: &str, name: Option<&String>) } let Some(named) = name.and_then(|name| Harness::named(name)) else { let given = name.map(|name| format!(", not {name}")).unwrap_or_default(); - bail!("{arg} must be followed by {}{given}", harness::NAMES); + bail!("{arg} must be followed by {}{given}", harness::names()); }; *harness = Some(named); Ok(()) diff --git a/src/config.rs b/src/config.rs index 1ab016502..5bbd324d6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -155,7 +155,7 @@ impl UserConfig { Some(harness) => config.harness.default = Some(harness), None => bail!( "harness.default must be {}, not {name:?}, in {file}", - harness::NAMES + harness::names() ), }, ("harness", "default", _) => { diff --git a/src/harness.rs b/src/harness.rs index d865ccffa..fc047bb8d 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -4,14 +4,17 @@ //! child Run, and recorded with what the Command did. use std::fmt; -use std::io::Write; use std::path::Path; -use std::process::{Command, Output, Stdio}; +use std::process::Output; -use anyhow::{Context, Result, bail}; -use serde_json::Value; +use anyhow::{Result, bail}; -use crate::progress; +mod adapter; +pub(crate) mod claude; +pub(crate) mod codex; + +pub use adapter::{Adapter, Invocation}; +pub use codex::Catalog; /// The headless agent CLI a Command's sessions run on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -24,33 +27,36 @@ impl Harness { /// The Harness named `name`, as the command line and the User config /// write it, if it is one. pub fn named(name: &str) -> Option { - match name { - "claude" => Some(Harness::Claude), - "codex" => Some(Harness::Codex), - _ => None, - } + adapter::REGISTERED + .iter() + .find(|(_, adapter)| adapter.name() == name) + .map(|(harness, _)| *harness) } /// Its name, which is also its CLI's. pub fn name(self) -> &'static str { - match self { - Harness::Claude => "claude", - Harness::Codex => "codex", - } + self.adapter().name() } - /// The signals that ask a session on it to stop, in the order they are - /// sent: SIGTERM, after SIGINT for Codex, which stops cleanly only on - /// SIGINT, interrupting its turn. - pub fn stop_signals(self) -> &'static [libc::c_int] { - match self { - Harness::Claude => &[libc::SIGTERM], - Harness::Codex => &[libc::SIGINT, libc::SIGTERM], + /// Select the adapter once; session callers retain it for their lifetime. + pub fn adapter(self) -> &'static dyn Adapter { + adapter::REGISTERED + .iter() + .find(|(harness, _)| *harness == self) + .map(|(_, adapter)| *adapter) + .expect("every Harness has an adapter") + } + + /// Every Harness, in the order its adapters register for Setup. + pub const ALL: [Harness; adapter::REGISTERED.len()] = { + let mut all = [adapter::REGISTERED[0].0; adapter::REGISTERED.len()]; + let mut at = 0; + while at < all.len() { + all[at] = adapter::REGISTERED[at].0; + at += 1; } - } - - /// Every Harness, in the order Setup lists them. - pub const ALL: [Harness; 2] = [Harness::Claude, Harness::Codex]; + all + }; /// Whether sessions can run on it here: its CLI is on `PATH`. pub fn installed(self) -> bool { @@ -59,7 +65,13 @@ impl Harness { } /// The names a Harness is chosen by, for the messages that list them. -pub const NAMES: &str = "claude or codex"; +pub fn names() -> String { + adapter::REGISTERED + .iter() + .map(|(_, adapter)| adapter.name()) + .collect::>() + .join(" or ") +} /// A Model and an Effort, each none where it is left to the Harness. #[derive(Debug, Default, Clone, PartialEq, Eq)] @@ -83,18 +95,12 @@ pub struct Settings { impl Settings { /// The Model and Effort set for `harness`. pub fn of(&self, harness: Harness) -> &ModelAndEffort { - match harness { - Harness::Claude => &self.claude, - Harness::Codex => &self.codex, - } + harness.adapter().settings(self) } /// The Model and Effort set for `harness`, to set them. pub fn of_mut(&mut self, harness: Harness) -> &mut ModelAndEffort { - match harness { - Harness::Claude => &mut self.claude, - Harness::Codex => &mut self.codex, - } + harness.adapter().settings_mut(self) } } @@ -170,33 +176,15 @@ impl Choice { format!("Built with {self}") } - /// The arguments that run a Claude session on its Model and Effort, - /// where they are set. + // Compatibility for the existing argument tests; production callers + // use the adapter's complete session invocation. + #[cfg(test)] pub fn claude_args(&self) -> Vec<&str> { - let mut args = Vec::new(); - if let Some(model) = &self.model { - args.extend(["--model", model]); - } - if let Some(effort) = &self.effort { - args.extend(["--effort", effort]); - } - args + claude::model_args(self) } - - /// The arguments that run a Codex session on its Model and Effort, where - /// they are set. + #[cfg(test)] pub fn codex_args(&self) -> Vec { - let mut args = Vec::new(); - if let Some(model) = &self.model { - args.extend(["-m".to_string(), model.clone()]); - } - if let Some(effort) = &self.effort { - args.extend([ - "-c".to_string(), - format!("model_reasoning_effort=\"{effort}\""), - ]); - } - args + codex::model_args(self) } /// Check, before any work, that sessions can run on this, and settle the @@ -207,7 +195,8 @@ impl Choice { /// is matched regardless of case, a Model by its slug or display name, /// and becomes the name Codex takes. Each failure says what to change. pub fn check(&mut self) -> Result<()> { - let cli = self.harness.name(); + let adapter = self.harness.adapter(); + let cli = adapter.name(); if !on_path(cli) { bail!( "{cli} is not on PATH, and the Harness {cli} is chosen by {}: install it, \ @@ -215,61 +204,7 @@ impl Choice { self.chosen_by ); } - match self.harness { - Harness::Claude => self.test_call(), - Harness::Codex => self.check_codex(), - } - } - - /// Check that Claude takes a minimal test call on the Model, if one is - /// named, with the Effort, if any. A refusal says what Claude said. - pub fn test_call(&self) -> Result<()> { - let cli = self.harness.name(); - let Some(model) = &self.model else { - return Ok(()); - }; - progress::step(format_args!( - "checking the Model {model} with a test call to {cli}" - )); - let mut child = Command::new(cli) - .arg("-p") - .args(self.claude_args()) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .with_context(|| format!("could not run {cli} to check the Model {model}"))?; - // Dropped once written, closing stdin, so the call can end. - if let Some(mut stdin) = child.stdin.take() { - let _ = stdin.write_all(TEST_PROMPT.as_bytes()); - } - let output = child - .wait_with_output() - .with_context(|| format!("could not run {cli} to check the Model {model}"))?; - if output.status.success() { - return Ok(()); - } - let effort = match &self.effort { - Some(effort) => format!(" with the Effort {effort}"), - None => String::new(), - }; - bail!( - "{cli} refused a test call on the Model {model}{effort}: {}", - said(&output) - ) - } - - /// Settle the Model and Effort, if either is named, on Codex's names for - /// them, from its catalog. - fn check_codex(&mut self) -> Result<()> { - if self.model.is_none() && self.effort.is_none() { - return Ok(()); - } - progress::step("checking the Model and Effort against codex debug models"); - let settled = Catalog::read()?.settle(self.model.as_deref(), self.effort.as_deref())?; - self.model = settled.model; - self.effort = settled.effort; - Ok(()) + adapter.check(self) } } @@ -283,160 +218,6 @@ fn said(output: &Output) -> String { .join("\n") } -/// One Model in Codex's catalog. -#[derive(Debug, Clone, PartialEq, Eq)] -struct CatalogModel { - /// The name Codex takes. - slug: String, - display_name: Option, - /// The Efforts it supports. - efforts: Vec, - /// Whether the catalog lists it to choose from. - listed: bool, -} - -/// The Models Codex can run, as `codex debug models` prints them. -#[derive(Debug)] -pub struct Catalog { - models: Vec, -} - -impl Catalog { - /// The catalog `codex debug models` prints, which costs no tokens to - /// read. - pub fn read() -> Result { - let output = Command::new("codex") - .args(["debug", "models"]) - .stdin(Stdio::null()) - .output() - .context("could not run codex debug models to read Codex's Models")?; - if !output.status.success() { - bail!( - "codex debug models failed, so Codex's Models can't be read: {}", - said(&output) - ); - } - Catalog::parse(&String::from_utf8_lossy(&output.stdout)) - } - - /// The catalog `codex debug models` printed as `json`. - pub fn parse(json: &str) -> Result { - let parsed: Value = serde_json::from_str(json) - .context("codex debug models printed no JSON catalog of Models")?; - let models = parsed["models"] - .as_array() - .context("codex debug models printed no list of Models")? - .iter() - .filter_map(|model| { - Some(CatalogModel { - slug: model["slug"].as_str()?.to_string(), - display_name: model["display_name"].as_str().map(String::from), - efforts: model["supported_reasoning_levels"] - .as_array() - .into_iter() - .flatten() - .filter_map(|level| level["effort"].as_str().map(String::from)) - .collect(), - listed: model["visibility"] != "hide", - }) - }) - .collect(); - Ok(Catalog { models }) - } - - /// The Models it lists to choose from, each as its slug and, where it - /// has one, its display name, as in `gpt-6.1-sol (GPT-6.1-Sol)`. - pub fn listed(&self) -> Vec { - self.models - .iter() - .filter(|model| model.listed) - .map(|model| match &model.display_name { - Some(display) => format!("{} ({display})", model.slug), - None => model.slug.clone(), - }) - .collect() - } - - /// The Efforts the Model `slug` supports, or with no Model, those any - /// Model supports, each once, in the catalog's order. - pub fn efforts(&self, slug: Option<&str>) -> Vec<&str> { - let mut efforts = Vec::new(); - let models = self - .models - .iter() - .filter(|model| slug.is_none_or(|slug| model.slug == slug)); - for each in models.flat_map(|model| &model.efforts) { - if !efforts.contains(&each.as_str()) { - efforts.push(each.as_str()); - } - } - efforts - } - - /// `model` and `effort` as Codex names them: the slug of the Model whose - /// slug or display name is `model`, regardless of case, and the Effort, - /// of those that Model supports, or with no Model of those any Model - /// supports, that is `effort`, regardless of case. Fails naming the - /// valid choices for a Model or Effort the catalog doesn't have. - pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { - let found = match model { - Some(model) => Some(self.model(model)?), - None => None, - }; - let Some(effort) = effort else { - return Ok(ModelAndEffort { - model: found.map(|found| found.slug.clone()), - effort: None, - }); - }; - let efforts = self.efforts(found.map(|found| found.slug.as_str())); - let Some(settled) = efforts - .iter() - .find(|supported| supported.eq_ignore_ascii_case(effort)) - else { - let of = match found { - Some(found) => format!("the Codex Model {}", found.slug), - None => "any Codex Model".to_string(), - }; - bail!( - "the Effort {effort} is not one {of} supports: choose one of {}", - efforts.join(", ") - ); - }; - Ok(ModelAndEffort { - model: found.map(|found| found.slug.clone()), - effort: Some(settled.to_string()), - }) - } - - /// The Model whose slug or display name is `name`, regardless of case. - fn model(&self, name: &str) -> Result<&CatalogModel> { - let named = |candidate: &&CatalogModel| { - candidate.slug.eq_ignore_ascii_case(name) - || candidate - .display_name - .as_deref() - .is_some_and(|display| display.eq_ignore_ascii_case(name)) - }; - if let Some(found) = self.models.iter().find(named) { - return Ok(found); - } - let listed: Vec<&str> = self - .models - .iter() - .filter(|model| model.listed) - .map(|model| model.slug.as_str()) - .collect(); - bail!( - "the Model {name} is not in Codex's catalog: choose one of {}", - listed.join(", ") - ) - } -} - -/// What the test call that checks a Model asks. -const TEST_PROMPT: &str = "Reply with OK."; - impl fmt::Display for Choice { /// ` · · `, as in `claude · claude-opus-5-5 · /// high`, saying `default model` or `default effort` for one left to the diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs new file mode 100644 index 000000000..93ba94b36 --- /dev/null +++ b/src/harness/adapter.rs @@ -0,0 +1,87 @@ +//! Everything that varies between Harnesses, behind one interface. + +use std::path::Path; + +use anyhow::Result; + +use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex}; +use crate::progress::Stream; +use crate::setup::Outside; + +/// One Harness's session protocol and its Model and Effort rules. Callers +/// select it once, then use the same adapter for a session and its Resume. +pub trait Adapter: Sync { + fn name(&self) -> &'static str; + fn product_name(&self) -> &'static str; + fn project_skills(&self) -> &'static str; + fn skill_loading(&self) -> SkillLoading; + fn stop_signals(&self) -> &'static [libc::c_int]; + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort; + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort; + + /// Arguments for a new session or Resume, and how its prompt reaches + /// the CLI. The adapter applies its skill-loading style here. + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation; + fn check(&self, choice: &mut Choice) -> Result<()>; + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result>; + fn stream(&self, worktree: &Path) -> Box; + + /// Fixed overrides, also used by the adapter's Model and Effort check. + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[] + } + + /// Make and exclude an instruction-file fallback link where the CLI + /// needs one. Native and config-based fallbacks stay in the adapter. + fn link_instruction_fallback(&self, _worktree: &Path) -> Result<()> { + Ok(()) + } + + /// Optionally reconcile final text, usage and outcome with the CLI's + /// own session log or export after exit, before reporting the ending. + /// Returning the stream unchanged is the fallback when no read is needed. + fn read_after_exit(&self, _worktree: &Path, stream: Box) -> Box { + stream + } +} + +/// A session's arguments and optional prompt on stdin. `None` keeps stdin +/// null; `Some` is written once and closed before following the stream. +pub struct Invocation { + pub args: Vec, + pub stdin: Option, +} + +/// How a Session prompt's first line loads its Factory skill. Prompts are +/// authored in Claude's form so the generated Prompts and skills page stays +/// independent of the selected Harness. +pub enum SkillLoading { + Slash, + Dollar, + #[expect(dead_code, reason = "reserved by #434 for skill-tool Harness adapters")] + Tool, +} + +impl SkillLoading { + pub fn prompt(&self, prompt: &str) -> String { + let Some(rest) = prompt.strip_prefix("/thirdshift-") else { + return prompt.to_string(); + }; + match self { + Self::Slash => prompt.to_string(), + Self::Dollar => format!("$thirdshift-{rest}"), + Self::Tool => { + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let (skill, remaining) = rest.split_at(end); + format!("Load thirdshift-{skill} with your skill tool.{remaining}") + } + } + } +} + +/// Registration comes from the adapters, including Setup's order. +pub const REGISTERED: [(Harness, &dyn Adapter); 2] = [claude::REGISTRATION, codex::REGISTRATION]; diff --git a/src/harness/claude.rs b/src/harness/claude.rs new file mode 100644 index 000000000..6b4f119b4 --- /dev/null +++ b/src/harness/claude.rs @@ -0,0 +1,161 @@ +//! Claude Code: its launch protocol, Setup and Model check. + +pub(crate) mod stream; + +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings, said}; +use crate::progress::{self, Stream}; +use crate::setup::{Outside, questions::ask_setting}; + +pub struct Claude; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Claude, &Claude); + +impl Adapter for Claude { + fn name(&self) -> &'static str { + "claude" + } + fn product_name(&self) -> &'static str { + "Claude Code" + } + fn project_skills(&self) -> &'static str { + ".claude/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Slash + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.claude + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.claude + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let prompt = self.skill_loading().prompt(prompt); + Invocation { + args: claude_args(choice, resume, &prompt) + .into_iter() + .map(String::from) + .collect(), + stdin: None, + } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + test_call(choice) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + ask_claude(outside, current).map(Some) + } + fn stream(&self, _worktree: &Path) -> Box { + Box::new(stream::ClaudeProgress::default()) + } +} + +/// The arguments that run a Claude session on its Model and Effort, +/// where they are set. +pub fn model_args(choice: &Choice) -> Vec<&str> { + let mut args = Vec::new(); + if let Some(model) = &choice.model { + args.extend(["--model", model]); + } + if let Some(effort) = &choice.effort { + args.extend(["--effort", effort]); + } + args +} + +/// The arguments every session runs `claude` with: headless in auto mode, +/// on the Model and Effort `harness` sets, if any, streaming JSON. With +/// `resume`, the session with that id continues. The prompt comes last. +pub fn claude_args<'a>( + harness: &'a Choice, + resume: Option<&'a str>, + prompt: &'a str, +) -> Vec<&'a str> { + let mut args = vec!["-p", "--permission-mode", "auto"]; + args.extend(model_args(harness)); + args.extend(["--output-format", "stream-json", "--verbose"]); + if let Some(session_id) = resume { + args.extend(["--resume", session_id]); + } + args.push(prompt); + args +} + +/// Check that Claude takes a minimal test call on the Model, if one is +/// named, with the Effort, if any. A refusal says what Claude said. +pub fn test_call(choice: &Choice) -> Result<()> { + let cli = Claude.name(); + let Some(model) = &choice.model else { + return Ok(()); + }; + progress::step(format_args!( + "checking the Model {model} with a test call to {cli}" + )); + let mut child = Command::new(cli) + .arg("-p") + .envs(Claude.environment().iter().copied()) + .args(model_args(choice)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("could not run {cli} to check the Model {model}"))?; + // Dropped once written, closing stdin, so the call can end. + if let Some(mut stdin) = child.stdin.take() { + let _ = stdin.write_all(TEST_PROMPT.as_bytes()); + } + let output = child + .wait_with_output() + .with_context(|| format!("could not run {cli} to check the Model {model}"))?; + if output.status.success() { + return Ok(()); + } + let effort = match &choice.effort { + Some(effort) => format!(" with the Effort {effort}"), + None => String::new(), + }; + bail!( + "{cli} refused a test call on the Model {model}{effort}: {}", + said(&output) + ) +} + +/// What the minimal Model check asks. +const TEST_PROMPT: &str = "Reply with OK."; + +/// Ask Claude's Model and Effort, with `current` as the defaults. A Model is +/// checked with a test call, as a Run checks it, and on a refusal the Model +/// and Effort are asked again. +fn ask_claude(outside: &mut dyn Outside, current: &ModelAndEffort) -> Result { + loop { + let chosen = ModelAndEffort { + model: ask_setting(outside, "Model", Harness::Claude, current.model.as_deref())?, + effort: ask_setting( + outside, + "Effort", + Harness::Claude, + current.effort.as_deref(), + )?, + }; + if chosen.model.is_none() { + return Ok(chosen); + } + match outside.test_call(&chosen) { + Ok(()) => return Ok(chosen), + Err(error) => outside.say(format!("{error:#}")), + } + } +} diff --git a/src/harness/claude/stream.rs b/src/harness/claude/stream.rs new file mode 100644 index 000000000..61d836ff0 --- /dev/null +++ b/src/harness/claude/stream.rs @@ -0,0 +1,173 @@ +//! Claude Code's stream-json session protocol. + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::progress::{Stream, bash, shorten}; + +/// Condenses one Claude session's stream-json, a line at a time. Unknown and +/// malformed lines are skipped, never an error. +/// +/// A `result` event doesn't mean the session is over: a session that waits on +/// background sub-agents resumes, with another `system`/`init` event, and ends +/// each resumption with another `result`. So only the first `init` gives a +/// line, `result` events give none, and [`ClaudeProgress::summary`] reports the last +/// one once the process has exited. +/// +/// A background task still running when the session ends its turn for good is +/// killed as the process exits, so a task killed after the last `result` is +/// work the session may have been waiting on, or work it had given up on and +/// left running: see [`ClaudeProgress::killed_background_work`]. +#[derive(Default)] +pub struct ClaudeProgress { + started: bool, + cwd: Option, + session_id: Option, + /// Each background task's description, by task id. + tasks: HashMap, + /// Descriptions of the tasks killed since the last `result`, by task id, + /// in the order they were killed. + killed: Vec<(String, String)>, + /// The last `result` was an error, such as running out of turns. + failed: bool, + /// Signed in with a claude.ai subscription rather than an API key. + subscription: bool, + turns_and_cost: Option<(u64, f64)>, + /// The text of the last `result`, if it had any. + final_message: Option, +} + +impl Stream for ClaudeProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + match event["type"].as_str() { + Some("system") if event["subtype"] == "init" && !self.started => { + self.started = true; + self.cwd = event["cwd"].as_str().map(String::from); + self.session_id = event["session_id"].as_str().map(String::from); + self.subscription = event["apiKeySource"] == "none"; + vec!["session started".to_string()] + } + Some("assistant") => event["message"]["content"] + .as_array() + .into_iter() + .flatten() + .filter(|block| block["type"] == "tool_use") + .filter_map(|block| Some(self.tool_use(block["name"].as_str()?, &block["input"]))) + .collect(), + Some("system") => { + self.track_task(&event); + Vec::new() + } + Some("result") => { + self.killed.clear(); + self.failed = event["subtype"] != "success" || event["is_error"] == true; + self.final_message = event["result"].as_str().map(String::from); + // Both are cumulative across a session's result events. + if let (Some(turns), Some(cost)) = ( + event["num_turns"].as_u64(), + event["total_cost_usd"].as_f64(), + ) { + self.turns_and_cost = Some((turns, cost)); + } + Vec::new() + } + _ => Vec::new(), + } + } + + /// Turns and cost from the last `result` event that reported them. On a + /// claude.ai subscription the cost is the API-price equivalent, not a + /// charge, and says so. + fn summary(&self) -> Option { + let (turns, cost) = self.turns_and_cost?; + let basis = if self.subscription { + " at API prices" + } else { + "" + }; + Some(format!("{turns} turns, ${cost:.2}{basis}")) + } + + /// The text of the last `result` event. None if that `result` had no + /// text, as one that is an error may not. + fn final_message(&self) -> Option<&str> { + self.final_message.as_deref() + } + + /// The session's id, from its first `init` event. + fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + + /// Descriptions of the background tasks killed after the last `result`. + /// The stream doesn't say whether the session was waiting on them, or + /// had abandoned them. None if that `result` was an error. + fn killed_background_work(&self) -> Vec<&str> { + if self.failed { + return Vec::new(); + } + self.killed + .iter() + .map(|(_, description)| description.as_str()) + .collect() + } +} + +impl ClaudeProgress { + /// Track a background task's description and whether it was killed. + fn track_task(&mut self, event: &Value) { + let Some(id) = event["task_id"].as_str() else { + return; + }; + let killed = match event["subtype"].as_str() { + Some("task_started") => { + if let Some(description) = event["description"].as_str() { + self.tasks.insert(id.to_string(), description.to_string()); + } + false + } + Some("task_updated") => event["patch"]["status"] == "killed", + Some("task_notification") => event["status"] == "stopped", + _ => false, + }; + if killed && !self.killed.iter().any(|(killed_id, _)| killed_id == id) { + let description = self + .tasks + .get(id) + .map(String::as_str) + .or(event["summary"].as_str()) + .unwrap_or(id) + .to_string(); + self.killed.push((id.to_string(), description)); + } + } + + fn tool_use(&self, name: &str, input: &Value) -> String { + match name { + "Skill" => format!("skill {}", input["skill"].as_str().unwrap_or("?")), + "Bash" => bash(input["command"].as_str().unwrap_or("")), + _ => { + let detail = ["file_path", "pattern", "description", "url", "query"] + .iter() + .find_map(|key| input[key].as_str()); + match detail { + Some(detail) => format!("{name} {}", shorten(&self.relative(detail))), + None => name.to_string(), + } + } + } + } + + /// `path` relative to the session's working directory, if it is inside it. + fn relative(&self, path: &str) -> String { + self.cwd + .as_deref() + .and_then(|cwd| path.strip_prefix(cwd)?.strip_prefix('/')) + .unwrap_or(path) + .to_string() + } +} diff --git a/src/harness/codex.rs b/src/harness/codex.rs new file mode 100644 index 000000000..734b15a3d --- /dev/null +++ b/src/harness/codex.rs @@ -0,0 +1,168 @@ +//! Codex: its launch protocol, free Model catalog and Setup. + +mod catalog; +pub(crate) mod stream; +pub use catalog::Catalog; + +use std::path::Path; + +use anyhow::Result; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings}; +use crate::progress::{self, Stream}; +use crate::setup::{Outside, questions::ask_setting}; + +pub struct Codex; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Codex, &Codex); + +impl Adapter for Codex { + fn name(&self) -> &'static str { + "codex" + } + fn product_name(&self) -> &'static str { + "Codex" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Dollar + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGINT, libc::SIGTERM] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.codex + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.codex + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + Invocation { + args: codex_args(choice, resume, &self.skill_loading().prompt(prompt)), + stdin: None, + } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + check_model_and_effort(choice) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + match outside.codex_catalog() { + Ok(catalog) => ask_codex(outside, &catalog, current).map(Some), + Err(error) => { + outside.say(format!("{error:#}\nThe harness settings stay as they are; rerun `thirdshift setup` once codex debug models works.")); + Ok(None) + } + } + } + fn stream(&self, worktree: &Path) -> Box { + Box::new(stream::CodexProgress::in_worktree(worktree)) + } +} + +/// The arguments that run a Codex session on its Model and Effort, where +/// they are set. +pub fn model_args(choice: &Choice) -> Vec { + let mut args = Vec::new(); + if let Some(model) = &choice.model { + args.extend(["-m".to_string(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend([ + "-c".to_string(), + format!("model_reasoning_effort=\"{effort}\""), + ]); + } + args +} + +/// The arguments every session runs `codex` with: `exec`, streaming JSONL, +/// with no approvals and no sandbox (ADR-0012), on the Model and Effort +/// `harness` sets, if any, reading `CLAUDE.md` where a directory has no +/// `AGENTS.md`. With `resume`, the session with that id continues, given +/// every one of those again, as Codex keeps none of them. The prompt comes +/// last. +pub fn codex_args(harness: &Choice, resume: Option<&str>, prompt: &str) -> Vec { + let mut args: Vec = [ + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + ] + .map(String::from) + .to_vec(); + args.extend(model_args(harness)); + args.extend(["-c".to_string(), CLAUDE_MD_FALLBACK.to_string()]); + if let Some(session_id) = resume { + args.extend(["resume".to_string(), session_id.to_string()]); + } + args.push(prompt.to_string()); + args +} + +/// The config setting that has Codex read `CLAUDE.md` where a directory has +/// no `AGENTS.md`. +const CLAUDE_MD_FALLBACK: &str = r#"project_doc_fallback_filenames=["CLAUDE.md"]"#; + +/// Settle the Model and Effort, if either is named, on Codex's names for +/// them, from its catalog. +fn check_model_and_effort(choice: &mut Choice) -> Result<()> { + if choice.model.is_none() && choice.effort.is_none() { + return Ok(()); + } + progress::step("checking the Model and Effort against codex debug models"); + let settled = Catalog::read()?.settle(choice.model.as_deref(), choice.effort.as_deref())?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) +} +/// Ask Codex's Model, listing those in its `catalog`, then its Effort, +/// listing those the Model chosen supports, with `current` as the defaults. +/// Each is matched against the catalog as a Run matches it, and written as +/// Codex names it; one the catalog doesn't have is asked again, with the +/// valid choices. +fn ask_codex( + outside: &mut dyn Outside, + catalog: &Catalog, + current: &ModelAndEffort, +) -> Result { + outside.say(format!( + "{}'s Models: {}", + Codex.product_name(), + catalog.listed().join(", ") + )); + let model = loop { + let model = ask_setting(outside, "Model", Harness::Codex, current.model.as_deref())?; + match catalog.settle(model.as_deref(), None) { + Ok(settled) => break settled.model, + Err(error) => outside.say(format!("{error:#}")), + } + }; + let efforts = catalog.efforts(model.as_deref()).join(", "); + outside.say(match &model { + Some(model) => format!("Efforts {model} supports: {efforts}"), + None => format!( + "Efforts {}'s Models support: {efforts}", + Codex.product_name() + ), + }); + loop { + let effort = ask_setting(outside, "Effort", Harness::Codex, current.effort.as_deref())?; + match catalog.settle(model.as_deref(), effort.as_deref()) { + Ok(settled) => return Ok(settled), + Err(error) => outside.say(format!("{error:#}")), + } + } +} + +/// `prompt` as Codex takes it: a first line that loads a Factory skill, +/// `/thirdshift-` as Claude's prompts write it, loads it as +/// `$thirdshift-`. +#[cfg(test)] +pub fn codex_prompt(prompt: &str) -> String { + SkillLoading::Dollar.prompt(prompt) +} diff --git a/src/harness/codex/catalog.rs b/src/harness/codex/catalog.rs new file mode 100644 index 000000000..a0296c946 --- /dev/null +++ b/src/harness/codex/catalog.rs @@ -0,0 +1,162 @@ +//! Codex's free Model and Effort catalog. + +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; + +use serde_json::Value; + +use super::Codex; +use crate::harness::{Adapter, ModelAndEffort, said}; + +/// One Model in Codex's catalog. +#[derive(Debug, Clone, PartialEq, Eq)] +struct CatalogModel { + /// The name Codex takes. + slug: String, + display_name: Option, + /// The Efforts it supports. + efforts: Vec, + /// Whether the catalog lists it to choose from. + listed: bool, +} + +/// The Models Codex can run, as `codex debug models` prints them. +#[derive(Debug)] +pub struct Catalog { + models: Vec, +} + +impl Catalog { + /// The catalog `codex debug models` prints, which costs no tokens to + /// read. + pub fn read() -> Result { + let output = Command::new(Codex.name()) + .args(["debug", "models"]) + .envs(Codex.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run codex debug models to read Codex's Models")?; + if !output.status.success() { + bail!( + "codex debug models failed, so Codex's Models can't be read: {}", + said(&output) + ); + } + Catalog::parse(&String::from_utf8_lossy(&output.stdout)) + } + + /// The catalog `codex debug models` printed as `json`. + pub fn parse(json: &str) -> Result { + let parsed: Value = serde_json::from_str(json) + .context("codex debug models printed no JSON catalog of Models")?; + let models = parsed["models"] + .as_array() + .context("codex debug models printed no list of Models")? + .iter() + .filter_map(|model| { + Some(CatalogModel { + slug: model["slug"].as_str()?.to_string(), + display_name: model["display_name"].as_str().map(String::from), + efforts: model["supported_reasoning_levels"] + .as_array() + .into_iter() + .flatten() + .filter_map(|level| level["effort"].as_str().map(String::from)) + .collect(), + listed: model["visibility"] != "hide", + }) + }) + .collect(); + Ok(Catalog { models }) + } + + /// The Models it lists to choose from, each as its slug and, where it + /// has one, its display name, as in `gpt-6.1-sol (GPT-6.1-Sol)`. + pub fn listed(&self) -> Vec { + self.models + .iter() + .filter(|model| model.listed) + .map(|model| match &model.display_name { + Some(display) => format!("{} ({display})", model.slug), + None => model.slug.clone(), + }) + .collect() + } + + /// The Efforts the Model `slug` supports, or with no Model, those any + /// Model supports, each once, in the catalog's order. + pub fn efforts(&self, slug: Option<&str>) -> Vec<&str> { + let mut efforts = Vec::new(); + let models = self + .models + .iter() + .filter(|model| slug.is_none_or(|slug| model.slug == slug)); + for each in models.flat_map(|model| &model.efforts) { + if !efforts.contains(&each.as_str()) { + efforts.push(each.as_str()); + } + } + efforts + } + + /// `model` and `effort` as Codex names them: the slug of the Model whose + /// slug or display name is `model`, regardless of case, and the Effort, + /// of those that Model supports, or with no Model of those any Model + /// supports, that is `effort`, regardless of case. Fails naming the + /// valid choices for a Model or Effort the catalog doesn't have. + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { + let found = match model { + Some(model) => Some(self.model(model)?), + None => None, + }; + let Some(effort) = effort else { + return Ok(ModelAndEffort { + model: found.map(|found| found.slug.clone()), + effort: None, + }); + }; + let efforts = self.efforts(found.map(|found| found.slug.as_str())); + let Some(settled) = efforts + .iter() + .find(|supported| supported.eq_ignore_ascii_case(effort)) + else { + let of = match found { + Some(found) => format!("the Codex Model {}", found.slug), + None => "any Codex Model".to_string(), + }; + bail!( + "the Effort {effort} is not one {of} supports: choose one of {}", + efforts.join(", ") + ); + }; + Ok(ModelAndEffort { + model: found.map(|found| found.slug.clone()), + effort: Some(settled.to_string()), + }) + } + + /// The Model whose slug or display name is `name`, regardless of case. + fn model(&self, name: &str) -> Result<&CatalogModel> { + let named = |candidate: &&CatalogModel| { + candidate.slug.eq_ignore_ascii_case(name) + || candidate + .display_name + .as_deref() + .is_some_and(|display| display.eq_ignore_ascii_case(name)) + }; + if let Some(found) = self.models.iter().find(named) { + return Ok(found); + } + let listed: Vec<&str> = self + .models + .iter() + .filter(|model| model.listed) + .map(|model| model.slug.as_str()) + .collect(); + bail!( + "the Model {name} is not in Codex's catalog: choose one of {}", + listed.join(", ") + ) + } +} diff --git a/src/progress/codex.rs b/src/harness/codex/stream.rs similarity index 99% rename from src/progress/codex.rs rename to src/harness/codex/stream.rs index 605ac1fcd..69dddc205 100644 --- a/src/progress/codex.rs +++ b/src/harness/codex/stream.rs @@ -10,7 +10,7 @@ use std::path::Path; use serde_json::Value; -use super::{Stream, bash, shorten}; +use crate::progress::{Stream, bash, shorten}; /// Condenses one Codex session's JSONL stream, a line at a time. /// diff --git a/src/progress.rs b/src/progress.rs index 7733170c0..e0e9e74d6 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -4,17 +4,15 @@ //! printed, so a stalled Run can be told from a busy one. Each is kept in the //! Command log too, if the command keeps one. -mod codex; - -use std::collections::HashMap; use std::fmt::Display; -use std::path::Path; use chrono::Local; -use serde_json::Value; -use crate::harness::Harness; +#[cfg(test)] +use crate::harness::claude::stream::ClaudeProgress; use crate::logs; +#[cfg(test)] +use serde_json::Value; /// The longest detail a session line shows before it is cut short. const MAX_DETAIL: usize = 100; @@ -114,7 +112,7 @@ fn split_stamp(line: &str) -> Option<(&str, &str)> { /// What a session's stream shows, read a line at a time: the progress lines /// it condenses to, and once it has ended, how the session went. Each -/// Harness streams its own events: see [`for_harness`]. +/// Harness streams its own events, parsed by its adapter. pub trait Stream: Send { /// The stderr lines, without the `thirdshift: ` prefix, for one line of /// the stream: one per notable event in it, often none. Unknown and @@ -150,182 +148,8 @@ pub trait Stream: Send { } } -/// The [`Stream`] of a session on `harness` in `worktree`. -pub fn for_harness(harness: Harness, worktree: &Path) -> Box { - match harness { - Harness::Claude => Box::new(ClaudeProgress::default()), - Harness::Codex => Box::new(codex::CodexProgress::in_worktree(worktree)), - } -} - -/// Condenses one Claude session's stream-json, a line at a time. Unknown and -/// malformed lines are skipped, never an error. -/// -/// A `result` event doesn't mean the session is over: a session that waits on -/// background sub-agents resumes, with another `system`/`init` event, and ends -/// each resumption with another `result`. So only the first `init` gives a -/// line, `result` events give none, and [`ClaudeProgress::summary`] reports the last -/// one once the process has exited. -/// -/// A background task still running when the session ends its turn for good is -/// killed as the process exits, so a task killed after the last `result` is -/// work the session may have been waiting on, or work it had given up on and -/// left running: see [`ClaudeProgress::killed_background_work`]. -#[derive(Default)] -pub struct ClaudeProgress { - started: bool, - cwd: Option, - session_id: Option, - /// Each background task's description, by task id. - tasks: HashMap, - /// Descriptions of the tasks killed since the last `result`, by task id, - /// in the order they were killed. - killed: Vec<(String, String)>, - /// The last `result` was an error, such as running out of turns. - failed: bool, - /// Signed in with a claude.ai subscription rather than an API key. - subscription: bool, - turns_and_cost: Option<(u64, f64)>, - /// The text of the last `result`, if it had any. - final_message: Option, -} - -impl Stream for ClaudeProgress { - fn condense(&mut self, raw: &str) -> Vec { - let Ok(event) = serde_json::from_str::(raw) else { - return Vec::new(); - }; - match event["type"].as_str() { - Some("system") if event["subtype"] == "init" && !self.started => { - self.started = true; - self.cwd = event["cwd"].as_str().map(String::from); - self.session_id = event["session_id"].as_str().map(String::from); - self.subscription = event["apiKeySource"] == "none"; - vec!["session started".to_string()] - } - Some("assistant") => event["message"]["content"] - .as_array() - .into_iter() - .flatten() - .filter(|block| block["type"] == "tool_use") - .filter_map(|block| Some(self.tool_use(block["name"].as_str()?, &block["input"]))) - .collect(), - Some("system") => { - self.track_task(&event); - Vec::new() - } - Some("result") => { - self.killed.clear(); - self.failed = event["subtype"] != "success" || event["is_error"] == true; - self.final_message = event["result"].as_str().map(String::from); - // Both are cumulative across a session's result events. - if let (Some(turns), Some(cost)) = ( - event["num_turns"].as_u64(), - event["total_cost_usd"].as_f64(), - ) { - self.turns_and_cost = Some((turns, cost)); - } - Vec::new() - } - _ => Vec::new(), - } - } - - /// Turns and cost from the last `result` event that reported them. On a - /// claude.ai subscription the cost is the API-price equivalent, not a - /// charge, and says so. - fn summary(&self) -> Option { - let (turns, cost) = self.turns_and_cost?; - let basis = if self.subscription { - " at API prices" - } else { - "" - }; - Some(format!("{turns} turns, ${cost:.2}{basis}")) - } - - /// The text of the last `result` event. None if that `result` had no - /// text, as one that is an error may not. - fn final_message(&self) -> Option<&str> { - self.final_message.as_deref() - } - - /// The session's id, from its first `init` event. - fn session_id(&self) -> Option<&str> { - self.session_id.as_deref() - } - - /// Descriptions of the background tasks killed after the last `result`. - /// The stream doesn't say whether the session was waiting on them, or - /// had abandoned them. None if that `result` was an error. - fn killed_background_work(&self) -> Vec<&str> { - if self.failed { - return Vec::new(); - } - self.killed - .iter() - .map(|(_, description)| description.as_str()) - .collect() - } -} - -impl ClaudeProgress { - /// Track a background task's description and whether it was killed. - fn track_task(&mut self, event: &Value) { - let Some(id) = event["task_id"].as_str() else { - return; - }; - let killed = match event["subtype"].as_str() { - Some("task_started") => { - if let Some(description) = event["description"].as_str() { - self.tasks.insert(id.to_string(), description.to_string()); - } - false - } - Some("task_updated") => event["patch"]["status"] == "killed", - Some("task_notification") => event["status"] == "stopped", - _ => false, - }; - if killed && !self.killed.iter().any(|(killed_id, _)| killed_id == id) { - let description = self - .tasks - .get(id) - .map(String::as_str) - .or(event["summary"].as_str()) - .unwrap_or(id) - .to_string(); - self.killed.push((id.to_string(), description)); - } - } - - fn tool_use(&self, name: &str, input: &Value) -> String { - match name { - "Skill" => format!("skill {}", input["skill"].as_str().unwrap_or("?")), - "Bash" => bash(input["command"].as_str().unwrap_or("")), - _ => { - let detail = ["file_path", "pattern", "description", "url", "query"] - .iter() - .find_map(|key| input[key].as_str()); - match detail { - Some(detail) => format!("{name} {}", shorten(&self.relative(detail))), - None => name.to_string(), - } - } - } - } - - /// `path` relative to the session's working directory, if it is inside it. - fn relative(&self, path: &str) -> String { - self.cwd - .as_deref() - .and_then(|cwd| path.strip_prefix(cwd)?.strip_prefix('/')) - .unwrap_or(path) - .to_string() - } -} - /// Commits and pushes by name; any other command by its first line. -fn bash(command: &str) -> String { +pub(crate) fn bash(command: &str) -> String { let actions: Vec<&str> = ["commit", "push"] .into_iter() .filter(|subcommand| runs_git(command, subcommand)) @@ -345,7 +169,7 @@ fn runs_git(command: &str, subcommand: &str) -> bool { }) } -fn shorten(text: &str) -> String { +pub(crate) fn shorten(text: &str) -> String { match text.char_indices().nth(MAX_DETAIL) { Some((cut, _)) => format!("{}…", &text[..cut]), None => text.to_string(), diff --git a/src/session.rs b/src/session.rs index 5c89e33fe..779fa1a28 100644 --- a/src/session.rs +++ b/src/session.rs @@ -11,7 +11,11 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail}; -use crate::harness::{Choice, Harness}; +#[cfg(test)] +pub(crate) use crate::harness::claude::claude_args; +#[cfg(test)] +pub(crate) use crate::harness::codex::{codex_args, codex_prompt}; +use crate::harness::{Adapter, Choice, Invocation}; use crate::interrupt; use crate::issue::{IssueUrl, Repo}; use crate::logs; @@ -88,12 +92,16 @@ impl<'a> Sessions<'a> { harness: &'a Choice, steps: impl FnOnce(&Self) -> Result, ) -> (Result, Option) { - if let Err(error) = skills::link_into(worktree, harness.harness) { + let adapter = harness.harness.adapter(); + if let Err(error) = skills::link_into(worktree, adapter) + .and_then(|()| adapter.link_instruction_fallback(worktree)) + { return (Err(error), None); } let on_machine = OnMachine { worktree: worktree.to_path_buf(), harness: harness.clone(), + adapter, }; let (taken, log) = Sessions::taking(logs, Box::new(on_machine), steps); (taken, log.filter(|log| log.exists())) @@ -217,6 +225,7 @@ impl Ended { struct OnMachine { worktree: PathBuf, harness: Choice, + adapter: &'static dyn Adapter, } impl Outside for OnMachine { @@ -227,10 +236,9 @@ impl Outside for OnMachine { prompt: &str, log: &Path, ) -> Result { - let harness = self.harness.harness; - let args = session_args(&self.harness, resume, prompt); - let stream = progress::for_harness(harness, &self.worktree); - let stream = run(kind, harness, &self.worktree, &args, log, stream)?; + let invocation = self.adapter.session(&self.harness, resume, prompt); + let stream = self.adapter.stream(&self.worktree); + let stream = run(kind, self.adapter, &self.worktree, invocation, log, stream)?; Ok(Ended { session_id: stream.session_id().map(String::from), killed: stream @@ -259,7 +267,7 @@ fn ending_with(killed: &[&str]) -> String { } } -/// Run `harness`'s CLI with `args`, as [`session_args`] gives them, in +/// Run the adapter's CLI with its invocation in /// `worktree`, where it finds the Factory skills, streaming its output to /// `log` and condensing it through `stream` to progress lines on stderr, /// each labelled `kind`. Returns what the stream showed once the CLI has @@ -268,13 +276,13 @@ fn ending_with(killed: &[&str]) -> String { /// and fails with `interrupted`. fn run( kind: &str, - harness: Harness, + adapter: &'static dyn Adapter, worktree: &Path, - args: &[String], + invocation: Invocation, log: &Path, mut stream: Box, ) -> Result> { - let cli = harness.name(); + let cli = adapter.name(); if let Some(dir) = log.parent() { fs::create_dir_all(dir).with_context(|| format!("could not create {}", dir.display()))?; } @@ -282,9 +290,14 @@ fn run( File::create(log).with_context(|| format!("could not create {}", log.display()))?; let started = Instant::now(); let mut child = Command::new(cli) - .args(args) + .args(&invocation.args) + .envs(adapter.environment().iter().copied()) .current_dir(worktree) - .stdin(Stdio::null()) + .stdin(if invocation.stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) .stdout(Stdio::piped()) // Its own process group, so thirdshift decides how it is stopped and // can stop everything it started. @@ -315,9 +328,17 @@ fn run( } (followed, stream) }); + if let Some(input) = invocation.stdin + && let Some(mut stdin) = child.stdin.take() + && let Err(error) = stdin.write_all(input.as_bytes()) + { + stop(&mut child, adapter); + let _ = follower.join(); + return Err(error).with_context(|| format!("could not write the prompt to {cli}")); + } let status = loop { if interrupt::requested() { - stop(&mut child, harness); + stop(&mut child, adapter); bail!("interrupted"); } if let Some(status) = child @@ -332,6 +353,7 @@ fn run( .join() .map_err(|_| anyhow!("the session stream reader panicked"))?; + let stream = adapter.read_after_exit(worktree, stream); let elapsed = minutes_and_seconds(started.elapsed()); let summary = stream .summary() @@ -359,85 +381,17 @@ fn run( } } -/// The arguments a session runs `harness`'s CLI with, as [`claude_args`] or -/// [`codex_args`] give them, its prompt loading its skill with that -/// Harness's sigil. -fn session_args(harness: &Choice, resume: Option<&str>, prompt: &str) -> Vec { - match harness.harness { - Harness::Claude => claude_args(harness, resume, prompt) - .into_iter() - .map(String::from) - .collect(), - Harness::Codex => codex_args(harness, resume, &codex_prompt(prompt)), - } -} - -/// `prompt` as Codex takes it: a first line that loads a Factory skill, -/// `/thirdshift-` as Claude's prompts write it, loads it as -/// `$thirdshift-`. -pub fn codex_prompt(prompt: &str) -> String { - match prompt.strip_prefix("/thirdshift-") { - Some(rest) => format!("$thirdshift-{rest}"), - None => prompt.to_string(), - } -} - -/// The arguments every session runs `claude` with: headless in auto mode, -/// on the Model and Effort `harness` sets, if any, streaming JSON. With -/// `resume`, the session with that id continues. The prompt comes last. -pub fn claude_args<'a>( - harness: &'a Choice, - resume: Option<&'a str>, - prompt: &'a str, -) -> Vec<&'a str> { - let mut args = vec!["-p", "--permission-mode", "auto"]; - args.extend(harness.claude_args()); - args.extend(["--output-format", "stream-json", "--verbose"]); - if let Some(session_id) = resume { - args.extend(["--resume", session_id]); - } - args.push(prompt); - args -} - -/// The arguments every session runs `codex` with: `exec`, streaming JSONL, -/// with no approvals and no sandbox (ADR-0012), on the Model and Effort -/// `harness` sets, if any, reading `CLAUDE.md` where a directory has no -/// `AGENTS.md`. With `resume`, the session with that id continues, given -/// every one of those again, as Codex keeps none of them. The prompt comes -/// last. -pub fn codex_args(harness: &Choice, resume: Option<&str>, prompt: &str) -> Vec { - let mut args: Vec = [ - "exec", - "--json", - "--dangerously-bypass-approvals-and-sandbox", - ] - .map(String::from) - .to_vec(); - args.extend(harness.codex_args()); - args.extend(["-c".to_string(), CLAUDE_MD_FALLBACK.to_string()]); - if let Some(session_id) = resume { - args.extend(["resume".to_string(), session_id.to_string()]); - } - args.push(prompt.to_string()); - args -} - -/// The config setting that has Codex read `CLAUDE.md` where a directory has -/// no `AGENTS.md`. -const CLAUDE_MD_FALLBACK: &str = r#"project_doc_fallback_filenames=["CLAUDE.md"]"#; - const POLL: Duration = Duration::from_millis(100); /// How long a session gets to exit after each signal that asks it to stop. const STOP_GRACE: Duration = Duration::from_secs(10); -/// Stop `child`, a session on `harness`, by its process group: with the -/// signals that ask `harness` to stop, in turn, each given `STOP_GRACE`, +/// Stop `child`, a session on `adapter`, by its process group: with the +/// signals that ask it to stop, in turn, each given `STOP_GRACE`, /// then SIGKILL. -fn stop(child: &mut Child, harness: Harness) { +fn stop(child: &mut Child, adapter: &dyn Adapter) { let group = -(child.id() as libc::pid_t); - for &signal in harness.stop_signals() { + for &signal in adapter.stop_signals() { // SAFETY: kill has no memory-safety preconditions. unsafe { libc::kill(group, signal) }; let deadline = Instant::now() + STOP_GRACE; diff --git a/src/setup.rs b/src/setup.rs index 518dd6e7e..d6804fcf3 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -10,7 +10,7 @@ //! the Harnesses' CLIs, `gh`, `git`, Resend and the files under the home //! folder; `Scripted`, in tests, from a script, recording each call. -mod questions; +pub(crate) mod questions; use std::io::{BufRead, IsTerminal, Write}; use std::path::Path; @@ -351,7 +351,7 @@ fn set_email_to(document: &mut DocumentMut, to: &str) { /// What Setup does outside itself: the terminal it asks on, the Harnesses' /// CLIs, the Credentials, the lookups behind the suggested address, the User /// config file, the test email, and where its progress lines go. -trait Outside { +pub(crate) trait Outside { /// Whether there is someone to ask: stdin and stderr are both terminals. fn has_terminal(&mut self) -> bool; /// Show `prompt` and read one answer, trimmed, or `None` if stdin closed. @@ -447,13 +447,12 @@ impl Outside for OnMachine<'_> { } fn test_call(&mut self, chosen: &ModelAndEffort) -> Result<()> { - Choice { + crate::harness::claude::test_call(&Choice { harness: Harness::Claude, model: chosen.model.clone(), effort: chosen.effort.clone(), chosen_by: ChosenBy::UserConfig, - } - .test_call() + }) } fn codex_catalog(&mut self) -> Result { diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 36de315f7..4b0d489c2 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -163,7 +163,7 @@ pub fn ask( /// Ask which Harness every Run's sessions run on, listing each, marked where /// it isn't installed, and refusing that one; then its Model and Effort, /// with those `current` sets for it as the defaults, checked as a Run -/// checks them, as [`ask_claude`] and [`ask_codex`] ask them. The Harness's +/// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one /// installed, so `claude` when both are. With no /// Harness to choose, or Codex chosen and its catalog unreadable, the answer @@ -192,9 +192,10 @@ fn ask_harness( .filter(|harness| installed.contains(harness)) .or_else(|| installed.first().copied()); let Some(default) = default else { + let names = harness::names(); outside.say(format!( "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ - the harness settings stay as they are; install claude or codex, then rerun \ + the harness settings stay as they are; install {names}, then rerun \ `thirdshift setup`." )); return Ok(None); @@ -206,90 +207,22 @@ fn ask_harness( Some(default.name()), )?; match Harness::named(&name) { - None => outside.say(format!("Choose {}.", harness::NAMES)), + None => outside.say(format!("Choose {}.", harness::names())), Some(harness) if !installed.contains(&harness) => { outside.say(format!("{name} is not installed: it isn't on PATH.")) } Some(harness) => break harness, } }; - let set = current.of(harness); - let chosen = match harness { - Harness::Claude => ask_claude(outside, set)?, - Harness::Codex => match outside.codex_catalog() { - Ok(catalog) => ask_codex(outside, &catalog, set)?, - Err(error) => { - outside.say(format!( - "{error:#}\nThe harness settings stay as they are; rerun `thirdshift \ - setup` once codex debug models works." - )); - return Ok(None); - } - }, - }; - Ok(Some((harness, chosen))) -} - -/// Ask Claude's Model and Effort, with `current` as the defaults. A Model is -/// checked with a test call, as a Run checks it, and on a refusal the Model -/// and Effort are asked again. -fn ask_claude(outside: &mut impl Outside, current: &ModelAndEffort) -> Result { - loop { - let chosen = ModelAndEffort { - model: ask_setting(outside, "Model", Harness::Claude, current.model.as_deref())?, - effort: ask_setting( - outside, - "Effort", - Harness::Claude, - current.effort.as_deref(), - )?, - }; - if chosen.model.is_none() { - return Ok(chosen); - } - match outside.test_call(&chosen) { - Ok(()) => return Ok(chosen), - Err(error) => outside.say(format!("{error:#}")), - } - } -} - -/// Ask Codex's Model, listing those in its `catalog`, then its Effort, -/// listing those the Model chosen supports, with `current` as the defaults. -/// Each is matched against the catalog as a Run matches it, and written as -/// Codex names it; one the catalog doesn't have is asked again, with the -/// valid choices. -fn ask_codex( - outside: &mut impl Outside, - catalog: &harness::Catalog, - current: &ModelAndEffort, -) -> Result { - outside.say(format!("Codex's Models: {}", catalog.listed().join(", "))); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Codex, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; - let efforts = catalog.efforts(model.as_deref()).join(", "); - outside.say(match &model { - Some(model) => format!("Efforts {model} supports: {efforts}"), - None => format!("Efforts Codex's Models support: {efforts}"), - }); - loop { - let effort = ask_setting(outside, "Effort", Harness::Codex, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(settled), - Err(error) => outside.say(format!("{error:#}")), - } - } + let adapter = harness.adapter(); + let chosen = adapter.ask_settings(outside, adapter.settings(current))?; + Ok(chosen.map(|chosen| (harness, chosen))) } /// Ask for the `setting`, the Model or the Effort, of `harness`, with /// `current` as the default; `-` is none, the Harness's own default. -fn ask_setting( - outside: &mut impl Outside, +pub(crate) fn ask_setting( + outside: &mut (impl Outside + ?Sized), setting: &str, harness: Harness, current: Option<&str>, @@ -325,7 +258,11 @@ fn yes_or_no(outside: &mut impl Outside, question: &str, default: bool) -> Resul /// Ask `question` for a word, showing `default`, which nothing takes, if /// there is one. With no default, nothing is asked again. -fn answer(outside: &mut impl Outside, question: &str, default: Option<&str>) -> Result { +fn answer( + outside: &mut (impl Outside + ?Sized), + question: &str, + default: Option<&str>, +) -> Result { let prompt = match default { Some(default) => format!("{question} [{default}]: "), None => format!("{question}: "), @@ -356,7 +293,7 @@ fn ask_key(outside: &mut impl Outside, on_enter: &str) -> Result> } /// Show `prompt` and read one answer, trimmed, or end Setup if stdin closes. -fn read(outside: &mut impl Outside, prompt: &str) -> Result { +fn read(outside: &mut (impl Outside + ?Sized), prompt: &str) -> Result { match outside.read(prompt)? { Some(line) => Ok(line), None => bail!(ENDED), diff --git a/src/skills.rs b/src/skills.rs index 7a6d646fb..504534867 100644 --- a/src/skills.rs +++ b/src/skills.rs @@ -13,32 +13,24 @@ use include_dir::{Dir, include_dir}; use tempfile::TempDir; use crate::git::Git; -use crate::harness::Harness; +use crate::harness::Adapter; /// The Factory skills, as they are written out and the Prompts and skills /// page shows them, each in its `thirdshift-` directory. pub static SKILLS: Dir = include_dir!("$CARGO_MANIFEST_DIR/skills"); -/// Where in a worktree `harness` finds its project skills. -fn project_skills(harness: Harness) -> &'static str { - match harness { - Harness::Claude => ".claude/skills", - Harness::Codex => ".agents/skills", - } -} - /// The temp directory the Command wrote the Factory skills out to, once it /// has. static WRITTEN: Mutex> = Mutex::new(None); -/// Link every Factory skill into `worktree`'s project skills for `harness`, +/// Link every Factory skill into `worktree`'s project skills for `adapter`, /// `.claude/skills/` or `.agents/skills/`, writing them out first if the /// Command has not yet, and make sure the repository's `.git/info/exclude` /// keeps them out of git. The links are left in place: they go with the /// worktree. -pub fn link_into(worktree: &Path, harness: Harness) -> Result<()> { +pub fn link_into(worktree: &Path, adapter: &dyn Adapter) -> Result<()> { let written = written_out()?; - let dir = project_skills(harness); + let dir = adapter.project_skills(); exclude(&Git::new(worktree), &format!("/{dir}/thirdshift-*"))?; let project_skills = worktree.join(dir); fs::create_dir_all(&project_skills) From 0dc42af4e4db1c0c8b30ddfd434c67ecd68b7040 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:32:29 -0400 Subject: [PATCH 02/16] Stop every Harness session process tree on interruption (#435) --- README.md | 2 + src/harness.rs | 1 + src/harness/adapter.rs | 7 +++ src/harness/process_tree.rs | 110 ++++++++++++++++++++++++++++++++++ src/session.rs | 32 ++-------- tests/fakes.rs | 20 +++++++ tests/process_tree.rs | 116 ++++++++++++++++++++++++++++++++++++ tests/support/fakes.rs | 2 +- 8 files changed, 261 insertions(+), 29 deletions(-) create mode 100644 src/harness/process_tree.rs create mode 100644 tests/process_tree.rs diff --git a/README.md b/README.md index a5ffc0e51..fdc837fc9 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,8 @@ For each of the three, the command wins, then the [User config](#user-config), t With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. +Interrupting a Command stops every session's process tree on every Harness, including commands that started their own process group or session. thirdshift snapshots the descendants with `ps` on Linux and macOS, sends each process group the Harness's stop signals with a 10-second grace period each, then takes another snapshot before SIGKILL to catch commands started in the meantime. If the CLI exits on an earlier signal, thirdshift kills the remaining commands immediately without another grace period. + Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. diff --git a/src/harness.rs b/src/harness.rs index fc047bb8d..d20d5074c 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -12,6 +12,7 @@ use anyhow::{Result, bail}; mod adapter; pub(crate) mod claude; pub(crate) mod codex; +mod process_tree; pub use adapter::{Adapter, Invocation}; pub use codex::Catalog; diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 93ba94b36..10a03c07c 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -1,6 +1,7 @@ //! Everything that varies between Harnesses, behind one interface. use std::path::Path; +use std::process::Child; use anyhow::Result; @@ -30,6 +31,12 @@ pub trait Adapter: Sync { ) -> Result>; fn stream(&self, worktree: &Path) -> Box; + /// Stop the session and its descendants, including commands that made + /// their own process group or session, with this Harness's stop signals. + fn stop(&self, child: &mut Child) { + super::process_tree::stop(child, self.stop_signals()); + } + /// Fixed overrides, also used by the adapter's Model and Effort check. fn environment(&self) -> &'static [(&'static str, &'static str)] { &[] diff --git a/src/harness/process_tree.rs b/src/harness/process_tree.rs new file mode 100644 index 000000000..5039e5131 --- /dev/null +++ b/src/harness/process_tree.rs @@ -0,0 +1,110 @@ +//! The shared stop path for every Harness, including escaped process groups. + +use std::collections::{BTreeSet, HashMap}; +use std::process::{Child, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; + +use crate::progress; + +const POLL: Duration = Duration::from_millis(100); +const STOP_GRACE: Duration = Duration::from_secs(10); + +/// Give each of the Harness's stop signals the existing grace period. Once +/// the CLI exits, skip the remaining grace periods but still kill its work. +pub(super) fn stop(child: &mut Child, signals: &[libc::c_int]) { + let pid = child.id() as libc::pid_t; + let mut tree = ProcessTree { + pids: BTreeSet::from([pid]), + groups: BTreeSet::from([pid]), + }; + tree.refresh(); + 'signals: for &signal in signals { + tree.signal(signal); + let deadline = Instant::now() + STOP_GRACE; + while Instant::now() < deadline { + if let Ok(Some(_)) = child.try_wait() { + break 'signals; + } + thread::sleep(POLL); + } + } + // Retain the first snapshot: the CLI may already have exited, leaving + // its descendants reparented. Known descendants also seed this lookup + // so their new children are found even when the CLI is gone. + tree.refresh(); + tree.signal(libc::SIGKILL); + let _ = child.wait(); +} + +struct ProcessTree { + pids: BTreeSet, + groups: BTreeSet, +} + +impl ProcessTree { + fn refresh(&mut self) { + if let Err(error) = self.snapshot() { + progress::step(format_args!( + "could not snapshot the session's process tree: {error:#}; \ + stopping its known process groups" + )); + } + } + + fn snapshot(&mut self) -> Result<()> { + // Separate -o flags and empty headers work on Linux and macOS. + let output = Command::new("ps") + .args(["-A", "-o", "pid=", "-o", "ppid=", "-o", "pgid="]) + .output() + .context("could not run ps")?; + if !output.status.success() { + bail!( + "ps exited {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + let mut children = HashMap::<_, Vec<_>>::new(); + let mut groups = HashMap::new(); + for line in String::from_utf8_lossy(&output.stdout).lines() { + let mut fields = line.split_whitespace(); + let mut next = || -> Result { + fields + .next() + .context("missing ps field")? + .parse() + .context("invalid ps field") + }; + let (pid, parent, group) = (next()?, next()?, next()?); + children.entry(parent).or_default().push(pid); + groups.insert(pid, group); + } + let mut pending: Vec<_> = self.pids.iter().copied().collect(); + let mut seen = BTreeSet::new(); + while let Some(pid) = pending.pop() { + if !seen.insert(pid) { + continue; + } + self.pids.insert(pid); + if let Some(&group) = groups.get(&pid) + && group > 0 + { + self.groups.insert(group); + } + if let Some(children) = children.get(&pid) { + pending.extend(children); + } + } + Ok(()) + } + + fn signal(&self, signal: libc::c_int) { + for &group in &self.groups { + // SAFETY: kill has no memory-safety preconditions; groups are positive. + unsafe { libc::kill(-group, signal) }; + } + } +} diff --git a/src/session.rs b/src/session.rs index 779fa1a28..174af6d2f 100644 --- a/src/session.rs +++ b/src/session.rs @@ -5,7 +5,7 @@ use std::fs::{self, File}; use std::io::{BufRead, BufReader, Read, Write}; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; +use std::process::{Command, Stdio}; use std::thread; use std::time::{Duration, Instant}; @@ -272,7 +272,7 @@ fn ending_with(killed: &[&str]) -> String { /// `log` and condensing it through `stream` to progress lines on stderr, /// each labelled `kind`. Returns what the stream showed once the CLI has /// exited cleanly, its turn not failed. Otherwise fails with the error the -/// stream gave, if any. An interrupt stops the session, as [`stop`] does, +/// stream gave, if any. An interrupt stops the session, as [`Adapter::stop`] does, /// and fails with `interrupted`. fn run( kind: &str, @@ -332,13 +332,13 @@ fn run( && let Some(mut stdin) = child.stdin.take() && let Err(error) = stdin.write_all(input.as_bytes()) { - stop(&mut child, adapter); + adapter.stop(&mut child); let _ = follower.join(); return Err(error).with_context(|| format!("could not write the prompt to {cli}")); } let status = loop { if interrupt::requested() { - stop(&mut child, adapter); + adapter.stop(&mut child); bail!("interrupted"); } if let Some(status) = child @@ -383,30 +383,6 @@ fn run( const POLL: Duration = Duration::from_millis(100); -/// How long a session gets to exit after each signal that asks it to stop. -const STOP_GRACE: Duration = Duration::from_secs(10); - -/// Stop `child`, a session on `adapter`, by its process group: with the -/// signals that ask it to stop, in turn, each given `STOP_GRACE`, -/// then SIGKILL. -fn stop(child: &mut Child, adapter: &dyn Adapter) { - let group = -(child.id() as libc::pid_t); - for &signal in adapter.stop_signals() { - // SAFETY: kill has no memory-safety preconditions. - unsafe { libc::kill(group, signal) }; - let deadline = Instant::now() + STOP_GRACE; - while Instant::now() < deadline { - if let Ok(Some(_)) = child.try_wait() { - return; - } - thread::sleep(POLL); - } - } - // SAFETY: as above. - unsafe { libc::kill(group, libc::SIGKILL) }; - let _ = child.wait(); -} - /// Copy every line of `stream` to `log_file` and print the progress lines it /// condenses to, until the stream ends. fn follow( diff --git a/tests/fakes.rs b/tests/fakes.rs index 0b197d18a..496dfef4d 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -24,6 +24,7 @@ mod json; use std::fs::{self, File}; use std::io::Write; +use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -110,6 +111,25 @@ fn main() { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), + "detached-command" => { + let command = args.next().expect("no detached command"); + let mut command = Command::new(command); + command.args(args); + // SAFETY: setsid is async-signal-safe and accesses no Rust state. + unsafe { + command.pre_exec(|| { + unsafe extern "C" { + fn setsid() -> i32; + } + if setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let error = command.exec(); + die(&format!("could not start detached command: {error}"), 1); + } _ => die(&format!("fake: no fake is called {name}"), 2), } exit(0); diff --git a/tests/process_tree.rs b/tests/process_tree.rs new file mode 100644 index 000000000..473c39b3c --- /dev/null +++ b/tests/process_tree.rs @@ -0,0 +1,116 @@ +//! Interrupting a Run stops commands that escaped the session's process group. + +mod support; + +use std::fs; +use std::thread; +use std::time::{Duration, Instant}; + +use support::Scenario; + +/// Clean up by the recorded pid even when the regression assertion fails. +struct DetachedCommand(libc::pid_t); + +impl DetachedCommand { + fn exists(&self) -> bool { + // SAFETY: kill has no memory-safety preconditions; signal 0 only probes. + unsafe { libc::kill(self.0, 0) == 0 } + } +} + +impl Drop for DetachedCommand { + fn drop(&mut self) { + if self.exists() { + // SAFETY: the recorded pid is this command's process group leader. + unsafe { libc::kill(-self.0, libc::SIGKILL) }; + } + } +} + +#[test] +fn interrupting_a_claude_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("claude"); +} + +#[test] +fn interrupting_a_codex_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("codex"); +} + +fn assert_detached_command_stopped(harness: &str) { + let scenario = Scenario::new(); + let pid = scenario.path("detached-pid"); + scenario.agent_does(&format!( + r#"detached-command bash -c 'trap "" INT TERM; echo $$ > "{pid}"; exec sleep 120' >/dev/null 2>&1 & +while ! test -s "{pid}"; do sleep 0.01; done +touch "{started}" +while :; do sleep 0.1; done +"#, + pid = pid.display(), + started = scenario.path("agent-started").display(), + )); + + let mut held = scenario.run_until( + &["harness", harness, &scenario.issue_url(7)], + &[], + "agent-started", + ); + let command = DetachedCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); + assert!(command.exists(), "the detached command never started"); + let interrupted_at = Instant::now(); + held.signal("INT"); + let result = held.finish(); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!( + interrupted_at.elapsed() < Duration::from_secs(8), + "a session that exited on its first signal incurred another grace period" + ); + let deadline = Instant::now() + Duration::from_secs(2); + while command.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!( + !command.exists(), + "the detached command survived interruption" + ); +} + +#[test] +fn a_command_started_during_the_stop_grace_period_is_killed_too() { + let scenario = Scenario::new(); + let pid = scenario.path("late-pid"); + let signals = scenario.path("signals"); + scenario.agent_does(&format!( + r#"on_int() {{ + echo INT >> "{signals}" + detached-command bash -c 'trap "" INT TERM; echo $$ > "{pid}"; exec sleep 120' >/dev/null 2>&1 & +}} +trap on_int INT +trap 'echo TERM >> "{signals}"' TERM +touch "{started}" +while :; do sleep 0.1 || :; done +"#, + pid = pid.display(), + signals = signals.display(), + started = scenario.path("agent-started").display(), + )); + + let result = scenario.run_and_signal( + &["harness", "codex", &scenario.issue_url(7)], + "agent-started", + "INT", + ); + let command = DetachedCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert_eq!(fs::read_to_string(signals).unwrap(), "INT\nTERM\n"); + let deadline = Instant::now() + Duration::from_secs(2); + while command.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!( + !command.exists(), + "the command started during the grace period survived" + ); +} diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index b851c6b75..2de6a51e2 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -21,7 +21,7 @@ const SOURCES: [&str; 5] = [ /// Put the fake `gh`, `claude` and `codex` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex"] { + for name in ["gh", "claude", "codex", "detached-command"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } From 3778911b511e70cfb82365e7166b40a2a639531e Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:38:54 -0400 Subject: [PATCH 03/16] Share detached-command exit assertions after Standards review --- tests/process_tree.rs | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 473c39b3c..2c898e7dc 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -16,6 +16,14 @@ impl DetachedCommand { // SAFETY: kill has no memory-safety preconditions; signal 0 only probes. unsafe { libc::kill(self.0, 0) == 0 } } + + fn assert_stopped(&self, message: &str) { + let deadline = Instant::now() + Duration::from_secs(2); + while self.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!(!self.exists(), "{message}"); + } } impl Drop for DetachedCommand { @@ -66,14 +74,7 @@ while :; do sleep 0.1; done interrupted_at.elapsed() < Duration::from_secs(8), "a session that exited on its first signal incurred another grace period" ); - let deadline = Instant::now() + Duration::from_secs(2); - while command.exists() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(10)); - } - assert!( - !command.exists(), - "the detached command survived interruption" - ); + command.assert_stopped("the detached command survived interruption"); } #[test] @@ -105,12 +106,5 @@ while :; do sleep 0.1 || :; done assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); assert_eq!(fs::read_to_string(signals).unwrap(), "INT\nTERM\n"); - let deadline = Instant::now() + Duration::from_secs(2); - while command.exists() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(10)); - } - assert!( - !command.exists(), - "the command started during the grace period survived" - ); + command.assert_stopped("the command started during the grace period survived"); } From 3ef66d50ae1fa8b8fc964a855981d78fac91b328 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:39:00 -0400 Subject: [PATCH 04/16] Add the agy Harness and installed-only Setup (#436) --- CONTEXT.md | 2 +- README.md | 19 +- ...rness-runs-unattended-and-fully-trusted.md | 27 ++ src/args.rs | 4 +- src/asks.rs | 1 + src/config.rs | 9 +- src/harness.rs | 6 + src/harness/adapter.rs | 5 +- src/harness/agy.rs | 115 ++++++ src/harness/agy/catalog.rs | 221 +++++++++++ src/harness/agy/stream.rs | 188 +++++++++ src/harness/instructions.rs | 40 ++ src/main.rs | 6 + src/setup.rs | 101 ++++- src/setup/questions.rs | 33 +- src/skills.rs | 4 +- tests/agy_sessions.rs | 364 ++++++++++++++++++ tests/fakes.rs | 3 + tests/fakes/agy.rs | 153 ++++++++ tests/fakes/codex.rs | 2 +- tests/setup.rs | 5 +- tests/support/fakes.rs | 5 +- tests/support/mod.rs | 17 + 23 files changed, 1281 insertions(+), 49 deletions(-) create mode 100644 docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md create mode 100644 src/harness/agy.rs create mode 100644 src/harness/agy/catalog.rs create mode 100644 src/harness/agy/stream.rs create mode 100644 src/harness/instructions.rs create mode 100644 tests/agy_sessions.rs create mode 100644 tests/fakes/agy.rs diff --git a/CONTEXT.md b/CONTEXT.md index 3db2d76f9..bc699c467 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -31,7 +31,7 @@ The message thirdshift starts or resumes an agent session with. It carries the * _Avoid_: instructions, system prompt **Harness**: -The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`) or Codex (`codex`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. +The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`), Codex (`codex`), Antigravity CLI (`agy`), Grok Build (`grok`), Muse Code (`muse`) or OpenCode (`opencode`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. _Avoid_: agent, provider, backend **Model**: diff --git a/README.md b/README.md index a5ffc0e51..2530095a3 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ The user account that runs thirdshift needs: - **`claude`** (Claude Code), logged in. - **`codex`** (Codex CLI), logged in, only for `harness codex`. +- **`agy`** (Google Antigravity CLI), logged in, only for `harness agy`. - **`gh`** (GitHub CLI), logged in. - **`git`** with a global `user.name` and `user.email`, and credentials that can push to the repository (`gh auth setup-git` makes git use `gh`'s login). The agents commit as this identity; without it, an agent may borrow the author of the last commit. @@ -213,6 +214,10 @@ effort = "high" # how hard that Model reasons; blank for Claude Code's own defa [harness.codex] model = "" effort = "" + +[harness.agy] +model = "gemini-3.8-flash" +effort = "medium" ``` `thirdshift setup` writes this file for you, listing every setting at its default so the file itself shows what can be changed: @@ -245,7 +250,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or agy; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -254,11 +259,15 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.agy] +model = "" # the Model Antigravity CLI's sessions run on; default blank, for agy's own +effort = "" # how hard that Model reasons; default blank, for agy's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude` when both are, and `codex` when only it is. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. The other Harness's section is left as it is. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing only those whose CLI is installed on `PATH`. An unavailable Harness, or a name that isn't one, is refused and asked again. The default is the configured one when installed, otherwise the first installed in `claude`, `codex`, `agy` order, so `claude` when it is installed. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For agy, Setup lists `agy models` and its supported Efforts, checks each answer without a model turn, and writes the canonical names regardless of case. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -299,6 +308,12 @@ For each of the three, the command wins, then the [User config](#user-config), t With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. +With `harness agy`, every session runs Google **Antigravity CLI** as `agy -p --dangerously-skip-permissions --output-format stream-json`, with `--model` and `--effort` when set, stdin null, and `AGY_CLI_DISABLE_AUTO_UPDATE=true` (agy ignores `=1`). The sandbox is opt-in and is never enabled, so unattended sessions can write the shared `.git` ([ADR-0013](docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md)). Factory skills are linked into `.agents/skills/`, excluded from git, and loaded by the `/thirdshift-` first line. agy reads project `AGENTS.md`, `GEMINI.md` and `.agents/rules/`, and user instructions under `~/.gemini/`; it does not read `~/.claude/`. When root `CLAUDE.md` exists and neither `AGENTS.md` nor `GEMINI.md` does, thirdshift adds a `GEMINI.md` link to `CLAUDE.md` and excludes it through `.git/info/exclude`; existing instruction files always win. + +agy's `init` supplies the conversation id and `step_update` supplies tool progress. The final `result` supplies the final message, success/error status, error and cumulative usage (including on Resume). `ERROR` or a nonzero exit fails the session with agy's error. A Resume uses `--conversation ` with the same flags, Model, Effort and environment; an interrupt sends SIGTERM, then SIGKILL after the grace period. Its start-up can be slow and variable: the research measured 8–95 seconds. No speed assumption or backend retry is made. + +Before any work on agy, a named Model or Effort is checked against the free `agy models` catalog, which lists IDs, labels and effort-suffixed variants. IDs, display labels and bare aliases are matched regardless of case, as are Efforts. `model Gemini-3.8-Flash effort Medium` becomes `--model gemini-3.8-flash --effort medium`; `model gemini-3.8-flash-medium` already carries its Effort. A bare alias requires an Effort; a Model or Effort the catalog lacks fails naming valid choices. This check uses null stdin and disables self-update too, without a model turn. + Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. diff --git a/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md b/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md new file mode 100644 index 000000000..2672bdabf --- /dev/null +++ b/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md @@ -0,0 +1,27 @@ +# Every Harness runs unattended and fully trusted in the worktree + +Sessions run with no human to answer an approval prompt or a trust question. Their commits and pushes write to the main repository's shared `.git`, outside the worktree. So every **Harness** runs its sessions, and every Resume of one, with approvals off, its sandbox off, and the worktree trusted. Each Harness carries the flags and environment that do this. They are fixed, and no user setting changes them: + +| Harness | Flags and environment | +|---|---| +| Claude Code | `--permission-mode auto` | +| Codex | `--dangerously-bypass-approvals-and-sandbox` | +| agy | `--dangerously-skip-permissions`, never `--sandbox` | +| Grok Build | `--always-approve`, `GROK_FOLDER_TRUST=0` | +| Muse Code | `--yolo` | +| OpenCode | `--auto`, `--standalone` | + +The worktree is the only boundary. This generalises the Codex part of ADR 0012. Sources: `docs/research/harness-candidates.md`, `docs/research/codex-headless-harness.md`. + +## Considered Options + +- **Each CLI's own sandbox.** Rejected. On stock Ubuntu (`kernel.apparmor_restrict_unprivileged_userns = 1`), bubblewrap can't start, so every sandboxed command fails under Codex and Muse. The sandboxes of agy and Grok make the main checkout's `.git` read-only, which blocks commits. A sandboxed mode for machines where it works would be a separate issue. +- **A one-time `grok --trust` in each repository's main checkout**, done or documented by Setup. Rejected. A repository nobody had trusted would run its sessions without `AGENTS.md` or the Factory skills, and nothing would fail. A scheduled Pickup run is exactly the case where nobody would notice. +- **Using the CLI's settings for approval and trust**, such as a user config that already sets always-approve. Rejected. A session would then behave differently on every machine. Passing the flags each time makes them explicit. +- **OpenCode through the user's shared background service** (its default). Rejected. A SIGTERM leaves the turn and its shell command running in the service, and thirdshift doesn't control that service's version or environment. `--standalone` keeps the server in the session's process group, and costs 2–3 s per session. + +## Consequences + +- A Harness's self-update is turned off in each session's environment too (`MUSE_NO_AUTO_UPDATE=1`, `GROK_DISABLE_AUTOUPDATER=1`, `AGY_CLI_DISABLE_AUTO_UPDATE=true`, `OPENCODE_DISABLE_AUTOUPDATE=1`). An update in the middle of a Run could replace the binary a Resume needs. agy ignores `=1`; only `=true` works. +- Each Harness reads its own instruction file and falls back to the other. thirdshift adds the fallback where a CLI lacks one: a `GEMINI.md` link for agy, and an `AGENTS.md` link for OpenCode, both to `CLAUDE.md` at the worktree root and kept out of git. Grok reads both files when both exist, and thirdshift leaves that as it is. +- Instructions in the user's `~/.claude/` reach only the Harnesses that read them: Claude, Grok and Muse. They don't reach Codex, agy or OpenCode. diff --git a/src/args.rs b/src/args.rs index b445828cf..66272b5b1 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex", + "harness must be followed by claude or codex or agy", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex, not gemini", + "harness must be followed by claude or codex or agy, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/asks.rs b/src/asks.rs index 74f5f68af..6eeafe446 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -262,6 +262,7 @@ mod tests { default: Some(Harness::Codex), claude: model_and_effort("opus", "high"), codex: model_and_effort("gpt-6.1-sol", "max"), + ..Default::default() }, ..no_settings() } diff --git a/src/config.rs b/src/config.rs index 5bbd324d6..8c80d6a3f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude or codex or agy; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -396,6 +396,10 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.agy] +model = "" # the Model Antigravity CLI's sessions run on; default blank, for agy's own +effort = "" # how hard that Model reasons; default blank, for agy's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -577,6 +581,7 @@ mod tests { model: Some("gpt-6.1-sol".to_string()), effort: Some("max".to_string()), }, + ..Default::default() } ); assert_eq!(parse("").unwrap().harness, harness::Settings::default()); @@ -587,7 +592,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex, not \"gemini\"", + "harness.default must be claude or codex or agy, not \"gemini\"", ), ( "[harness]\ndefault = true\n", diff --git a/src/harness.rs b/src/harness.rs index fc047bb8d..eb1906dba 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -10,8 +10,10 @@ use std::process::Output; use anyhow::{Result, bail}; mod adapter; +pub(crate) mod agy; pub(crate) mod claude; pub(crate) mod codex; +mod instructions; pub use adapter::{Adapter, Invocation}; pub use codex::Catalog; @@ -21,6 +23,7 @@ pub use codex::Catalog; pub enum Harness { Claude, Codex, + Agy, } impl Harness { @@ -90,6 +93,8 @@ pub struct Settings { pub claude: ModelAndEffort, /// `[harness.codex]`. pub codex: ModelAndEffort, + /// `[harness.agy]`. + pub agy: ModelAndEffort, } impl Settings { @@ -280,6 +285,7 @@ mod tests { default: Some(Harness::Codex), claude: set("opus", "high"), codex: set("gpt-6.1-sol", "max"), + ..Default::default() } } diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 93ba94b36..643db2b1e 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -4,7 +4,7 @@ use std::path::Path; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex}; +use super::{Choice, Harness, ModelAndEffort, Settings, agy, claude, codex}; use crate::progress::Stream; use crate::setup::Outside; @@ -84,4 +84,5 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 2] = [claude::REGISTRATION, codex::REGISTRATION]; +pub const REGISTERED: [(Harness, &dyn Adapter); 3] = + [claude::REGISTRATION, codex::REGISTRATION, agy::REGISTRATION]; diff --git a/src/harness/agy.rs b/src/harness/agy.rs new file mode 100644 index 000000000..7a83e3ce0 --- /dev/null +++ b/src/harness/agy.rs @@ -0,0 +1,115 @@ +//! Antigravity CLI: launch, free Model catalog, Setup and stream protocol. +mod catalog; +pub(crate) mod stream; +pub use catalog::Catalog; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings}; +use crate::progress::{self, Stream}; +use crate::setup::{Outside, questions::ask_setting}; +use anyhow::Result; +use std::path::Path; + +pub struct Agy; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Agy, &Agy); + +impl Adapter for Agy { + fn name(&self) -> &'static str { + "agy" + } + fn product_name(&self) -> &'static str { + "Antigravity CLI" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Slash + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.agy + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.agy + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[("AGY_CLI_DISABLE_AUTO_UPDATE", "true")] + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args: Vec = [ + "-p", + "--dangerously-skip-permissions", + "--output-format", + "stream-json", + ] + .map(String::from) + .to_vec(); + if let Some(model) = &choice.model { + args.extend(["--model".to_string(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend(["--effort".to_string(), effort.clone()]); + } + if let Some(id) = resume { + args.extend(["--conversation".to_string(), id.to_string()]); + } + args.push(self.skill_loading().prompt(prompt)); + Invocation { args, stdin: None } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + if choice.model.is_none() && choice.effort.is_none() { + return Ok(()); + } + progress::step("checking the Model and Effort against agy models"); + let settled = Catalog::read()?.settle(choice.model.as_deref(), choice.effort.as_deref())?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + let catalog = match outside.agy_catalog() { + Ok(catalog) => catalog, + Err(error) => { + outside.say(format!("{error:#}\nThe harness settings stay as they are; rerun `thirdshift setup` once agy models works.")); + return Ok(None); + } + }; + outside.say(format!( + "{}'s Models: {}", + self.product_name(), + catalog.listed().join(", ") + )); + let model = loop { + let model = ask_setting(outside, "Model", Harness::Agy, current.model.as_deref())?; + match catalog.model_name(model.as_deref()) { + Ok(model) => break model, + Err(error) => outside.say(format!("{error:#}")), + } + }; + outside.say(format!( + "Efforts {} supports: {}", + model.as_deref().unwrap_or("agy"), + catalog.efforts(model.as_deref()).join(", ") + )); + loop { + let effort = ask_setting(outside, "Effort", Harness::Agy, current.effort.as_deref())?; + match catalog.settle(model.as_deref(), effort.as_deref()) { + Ok(settled) => return Ok(Some(settled)), + Err(error) => outside.say(format!("{error:#}")), + } + } + } + fn link_instruction_fallback(&self, worktree: &Path) -> Result<()> { + super::instructions::link_fallback(worktree, "GEMINI.md", &["AGENTS.md"]) + } + fn stream(&self, _worktree: &Path) -> Box { + Box::new(stream::AgyProgress::default()) + } +} diff --git a/src/harness/agy/catalog.rs b/src/harness/agy/catalog.rs new file mode 100644 index 000000000..edae5d7d0 --- /dev/null +++ b/src/harness/agy/catalog.rs @@ -0,0 +1,221 @@ +//! agy models lists model IDs and labels, with supported Efforts encoded +//! in ID suffixes. Bare model aliases require an explicit Effort. +use super::Agy; +use crate::harness::{Adapter, ModelAndEffort, said}; +use anyhow::{Context, Result, bail}; +use std::process::{Command, Stdio}; + +#[derive(Debug)] +struct Model { + id: String, + label: String, +} + +#[derive(Debug)] +pub struct Catalog { + models: Vec, +} + +/// Effort suffixes agy knows, in increasing order. +const EFFORTS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"]; + +fn alias(id: &str) -> (&str, Option<&str>) { + id.rsplit_once('-') + .filter(|(_, suffix)| EFFORTS.contains(suffix)) + .map_or((id, None), |(base, suffix)| (base, Some(suffix))) +} + +impl Catalog { + pub fn read() -> Result { + let output = Command::new(Agy.name()) + .arg("models") + .envs(Agy.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run agy models to read Antigravity CLI's Models")?; + if !output.status.success() { + bail!( + "agy models failed, so Antigravity CLI's Models can't be read: {}", + said(&output) + ); + } + Self::parse(&String::from_utf8_lossy(&output.stdout)) + } + + pub fn parse(text: &str) -> Result { + let models: Vec = text + .lines() + .filter_map(|line| { + let (id, label) = line.trim().split_once('\t')?; + (!id.is_empty() && !label.trim().is_empty()).then(|| Model { + id: id.to_string(), + label: label.trim().to_string(), + }) + }) + .collect(); + if models.is_empty() { + bail!("agy models printed no catalog of Models"); + } + Ok(Self { models }) + } + + pub fn listed(&self) -> Vec { + self.models + .iter() + .map(|model| format!("{} ({})", model.id, model.label)) + .collect() + } + + /// Canonical ID, label or bare alias, matched regardless of case. + pub fn model_name(&self, name: Option<&str>) -> Result> { + let Some(name) = name else { + return Ok(None); + }; + for model in &self.models { + if model.id.eq_ignore_ascii_case(name) || model.label.eq_ignore_ascii_case(name) { + return Ok(Some(model.id.clone())); + } + } + if let Some(base) = self + .models + .iter() + .map(|model| alias(&model.id).0) + .find(|base| base.eq_ignore_ascii_case(name)) + { + return Ok(Some(base.to_string())); + } + bail!( + "the Model {name} is not in Antigravity CLI's catalog: choose one of {}", + self.models + .iter() + .map(|model| model.id.as_str()) + .collect::>() + .join(", ") + ); + } + + pub fn efforts(&self, model: Option<&str>) -> Vec<&str> { + EFFORTS + .into_iter() + .filter(|effort| { + self.models.iter().any(|candidate| { + let (base, suffix) = alias(&candidate.id); + suffix == Some(*effort) && model.is_none_or(|model| alias(model).0 == base) + }) + }) + .collect() + } + + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { + let model = self.model_name(model)?; + let efforts = self.efforts(model.as_deref()); + let settled_effort = match effort { + Some(effort) => Some( + *efforts + .iter() + .find(|valid| valid.eq_ignore_ascii_case(effort)) + .with_context(|| { + format!( + "the Effort {effort} is not one {} supports: choose one of {}", + model.as_deref().unwrap_or("any Antigravity CLI Model"), + efforts.join(", ") + ) + })?, + ), + None => None, + }; + if let Some(model) = &model + && !self.models.iter().any(|candidate| candidate.id == *model) + && settled_effort.is_none() + { + bail!( + "the Antigravity CLI Model {model} requires an Effort: choose one of {}", + efforts.join(", ") + ); + } + Ok(ModelAndEffort { + model, + effort: settled_effort.map(String::from), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + const MODELS: &str = "Fetching available models...\ngemini-3.8-flash-high\tGemini 3.8 Flash (High)\ngemini-3.8-flash-medium\tGemini 3.8 Flash (Medium)\ngemini-3.8-flash-low\tGemini 3.8 Flash (Low)\ngemini-3.1-pro-high\tGemini 3.1 Pro (High)\ngemini-3.1-pro-low\tGemini 3.1 Pro (Low)\ngpt-oss-120b-medium\tGPT-OSS 120B (Medium)\n"; + + #[test] + fn ids_labels_and_aliases_settle_regardless_of_case() { + let catalog = Catalog::parse(MODELS).unwrap(); + for name in ["GEMINI-3.8-FLASH-HIGH", "Gemini 3.8 Flash (HIGH)"] { + assert_eq!( + catalog.settle(Some(name), None).unwrap().model.as_deref(), + Some("gemini-3.8-flash-high") + ); + } + let choice = catalog + .settle(Some("Gemini-3.8-Flash"), Some("Medium")) + .unwrap(); + assert_eq!(choice.model.as_deref(), Some("gemini-3.8-flash")); + assert_eq!(choice.effort.as_deref(), Some("medium")); + assert_eq!( + catalog + .settle(Some("GPT-OSS-120B"), Some("MEDIUM")) + .unwrap() + .model + .as_deref(), + Some("gpt-oss-120b") + ); + } + + #[test] + fn effort_choices_follow_the_model_family_and_missing_effort_is_caught() { + let catalog = Catalog::parse(MODELS).unwrap(); + assert_eq!( + catalog.efforts(Some("gemini-3.1-pro-high")), + ["low", "high"] + ); + assert!( + catalog + .settle(Some("gemini-3.1-pro"), Some("medium")) + .unwrap_err() + .to_string() + .ends_with("choose one of low, high") + ); + assert!( + catalog + .settle(Some("gemini-3.8-flash"), None) + .unwrap_err() + .to_string() + .contains("requires an Effort") + ); + assert_eq!( + catalog + .settle(None, Some("HIGH")) + .unwrap() + .effort + .as_deref(), + Some("high") + ); + assert!( + catalog + .settle(Some("gemini-99"), None) + .unwrap_err() + .to_string() + .contains("gemini-3.8-flash-high") + ); + assert!(catalog.settle(None, Some("MAX")).is_err()); + } + + #[test] + fn an_empty_or_unreadable_catalog_fails_instead_of_accepting_names() { + for text in ["", "not a catalog", "\t\n", "model\t\n"] { + assert!(Catalog::parse(text).is_err()); + } + assert_eq!( + Catalog::parse(MODELS).unwrap().listed()[0], + "gemini-3.8-flash-high (Gemini 3.8 Flash (High))" + ); + } +} diff --git a/src/harness/agy/stream.rs b/src/harness/agy/stream.rs new file mode 100644 index 000000000..64cd70132 --- /dev/null +++ b/src/harness/agy/stream.rs @@ -0,0 +1,188 @@ +//! Antigravity CLI's event envelopes. Final text, failure and cumulative +//! usage come only from the final result, never concatenated text deltas. +use crate::progress::{Stream, bash, shorten}; +use serde_json::Value; +use std::collections::{BTreeMap, HashSet}; + +#[derive(Default)] +pub struct AgyProgress { + session_id: Option, + tools: HashSet, + result: Option, + running: BTreeMap, +} + +impl Stream for AgyProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + match event["event"].as_str() { + Some("init") if self.session_id.is_none() => { + self.session_id = event["conversation_id"] + .as_str() + .filter(|id| !id.is_empty()) + .map(String::from); + vec!["session started".to_string()] + } + Some("step_update") => { + let step = &event["step_update"]; + if step["step_type"] != "tool" { + return Vec::new(); + } + let Some(index) = step["step_index"].as_u64() else { + return Vec::new(); + }; + if step["state"] == "DONE" { + self.running.remove(&index); + } + if !self.tools.insert(index) { + return Vec::new(); + } + let name = step["tool_name"].as_str().unwrap_or("tool"); + let input = &step["tool_info"]["parameters"]; + let description = match name { + "run_command" => bash(input["CommandLine"].as_str().unwrap_or("")), + _ => { + let detail = [ + "AbsolutePath", + "TargetFile", + "SearchPath", + "Query", + "Description", + ] + .iter() + .find_map(|key| input[key].as_str()); + detail.map_or_else( + || name.to_string(), + |detail| format!("{name} {}", shorten(detail)), + ) + } + }; + if step["state"] == "ACTIVE" && matches!(name, "run_command" | "invoke_subagent") { + self.running.insert( + index, + description + .strip_prefix("$ ") + .unwrap_or(&description) + .to_string(), + ); + } + vec![description] + } + Some("result") => { + self.result = Some(event["result"].clone()); + self.error() + .map(|error| format!("error: {}", shorten(error))) + .into_iter() + .collect() + } + _ => Vec::new(), + } + } + fn summary(&self) -> Option { + let result = self.result.as_ref()?; + let usage = &result["usage"]; + Some(format!( + "{} turns, {} input tokens ({} cached), {} output tokens ({} thinking), {} total tokens", + result["num_turns"].as_u64()?, + usage["input_tokens"].as_u64()?, + usage["cache_read_tokens"].as_u64().unwrap_or(0), + usage["output_tokens"].as_u64()?, + usage["thinking_tokens"].as_u64().unwrap_or(0), + usage["total_tokens"].as_u64()? + )) + } + fn final_message(&self) -> Option<&str> { + self.result.as_ref()?["response"].as_str() + } + fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed() { + return Vec::new(); + } + self.running.values().map(String::as_str).collect() + } + fn failed(&self) -> bool { + self.result + .as_ref() + .is_some_and(|result| result["status"] == "ERROR") + } + fn error(&self) -> Option<&str> { + self.result.as_ref()?["error"] + .as_str() + .filter(|error| !error.is_empty()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const INIT: &str = r#"{"event":"init","conversation_id":"recorded-conversation","init":{"cwd":"/work/widgets","permission_mode":"always-proceed"}}"#; + const TOOL: &str = r#"{"event":"step_update","step_update":{"step_index":2,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","parameters":{"CommandLine":"cargo test"}}}}"#; + const RESULT: &str = r#"{"event":"result","result":{"conversation_id":"recorded-conversation","status":"SUCCESS","response":"Final reply only.","num_turns":2,"usage":{"input_tokens":12518,"output_tokens":29,"thinking_tokens":28,"cache_read_tokens":200,"total_tokens":12547}}}"#; + + #[test] + fn recorded_events_give_progress_id_final_reply_and_cumulative_usage() { + let mut stream = AgyProgress::default(); + assert_eq!(stream.condense(INIT), ["session started"]); + assert!(stream.condense(INIT).is_empty()); + assert_eq!(stream.condense(TOOL), ["$ cargo test"]); + assert!(stream.condense(TOOL).is_empty()); + stream.condense(&TOOL.replace("ACTIVE", "DONE")); + stream.condense(r#"{"event":"step_update","step_update":{"step_index":3,"step_type":"agent_response","text_delta":"Earlier reply."}}"#); + stream.condense(RESULT); + assert_eq!(stream.session_id(), Some("recorded-conversation")); + assert_eq!(stream.final_message(), Some("Final reply only.")); + assert_eq!( + stream.summary().as_deref(), + Some( + "2 turns, 12518 input tokens (200 cached), 29 output tokens (28 thinking), 12547 total tokens" + ) + ); + assert!(!stream.failed()); + assert_eq!(stream.error(), None); + assert!(stream.killed_background_work().is_empty()); + } + + #[test] + fn the_last_result_replaces_reply_usage_and_outcome() { + let mut stream = AgyProgress::default(); + stream.condense(RESULT); + let error = + r#"{"event":"result","result":{"status":"ERROR","error":"interrupted","response":""}}"#; + assert_eq!(stream.condense(error), ["error: interrupted"]); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("interrupted")); + assert_eq!(stream.final_message(), Some("")); + assert_eq!(stream.summary(), None); + } + + #[test] + fn unfinished_commands_are_resume_work_only_on_success() { + let mut stream = AgyProgress::default(); + stream.condense(TOOL); + stream.condense(RESULT); + assert_eq!(stream.killed_background_work(), ["cargo test"]); + stream.condense(r#"{"event":"result","result":{"status":"ERROR","error":"interrupted"}}"#); + assert!(stream.killed_background_work().is_empty()); + } + + #[test] + fn unknown_and_malformed_lines_are_ignored() { + let mut stream = AgyProgress::default(); + for line in [ + "not JSON", + "{}", + r#"{"event":"future"}"#, + r#"{"event":"step_update","step_update":{}}"#, + ] { + assert!(stream.condense(line).is_empty()); + } + assert_eq!(stream.session_id(), None); + assert_eq!(stream.summary(), None); + } +} diff --git a/src/harness/instructions.rs b/src/harness/instructions.rs new file mode 100644 index 000000000..4159f465b --- /dev/null +++ b/src/harness/instructions.rs @@ -0,0 +1,40 @@ +//! Instruction-file fallback links shared by Harness adapters. Existing +//! files, including dangling links, always win; only root CLAUDE.md falls back. +use crate::{git::Git, skills}; +use anyhow::{Context, Result}; +use std::fs; +use std::io::ErrorKind; +use std::os::unix::fs::symlink; +use std::path::Path; + +/// Link `name` to root CLAUDE.md only when none of `name` and `preferred` +/// already exists. Exclude it before creating it, using the repository's +/// shared, locked exclude file. +pub(super) fn link_fallback(worktree: &Path, name: &str, preferred: &[&str]) -> Result<()> { + let exists = |name: &str| -> Result { + match fs::symlink_metadata(worktree.join(name)) { + Ok(_) => Ok(true), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(false), + Err(error) => Err(error) + .with_context(|| format!("could not inspect {}", worktree.join(name).display())), + } + }; + if exists(name)? { + return Ok(()); + } + for name in preferred { + if exists(name)? { + return Ok(()); + } + } + if !worktree.join("CLAUDE.md").is_file() { + return Ok(()); + } + skills::exclude(&Git::new(worktree), &format!("/{name}"))?; + symlink("CLAUDE.md", worktree.join(name)).with_context(|| { + format!( + "could not link {} to CLAUDE.md", + worktree.join(name).display() + ) + }) +} diff --git a/src/main.rs b/src/main.rs index bc55f95bf..aa93ecc43 100644 --- a/src/main.rs +++ b/src/main.rs @@ -94,6 +94,12 @@ Claim, the worktree and any Command log. Codex sessions run codex exec --json --dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, as in Built with claude · opus · high, and the Run notification each name all three. +On agy (Antigravity CLI), agy models checks names without a turn, regardless of case; +a bare model alias needs an Effort, and effort-suffixed model IDs are accepted. Every +session and Resume uses -p --dangerously-skip-permissions --output-format stream-json, +null stdin and AGY_CLI_DISABLE_AUTO_UPDATE=true; a Resume uses --conversation . +agy reads AGENTS.md and GEMINI.md, falling back through an excluded GEMINI.md link to +root CLAUDE.md when neither exists. It does not read ~/.claude/. --email sends one Run notification when the Run ends, whatever the outcome: ready for review, merged, failed or interrupted. --email
sends it to
; a word diff --git a/src/setup.rs b/src/setup.rs index d6804fcf3..48d3b6f6d 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -367,6 +367,8 @@ pub(crate) trait Outside { fn test_call(&mut self, chosen: &ModelAndEffort) -> Result<()>; /// Read Codex's catalog of Models. fn codex_catalog(&mut self) -> Result; + /// Read Antigravity CLI's free Model catalog. + fn agy_catalog(&mut self) -> Result; /// Read the Credentials, as strictly as a Run does: where a Resend API /// key is found, if anywhere. fn find_key(&mut self) -> Result>; @@ -459,6 +461,10 @@ impl Outside for OnMachine<'_> { Catalog::read() } + fn agy_catalog(&mut self) -> Result { + crate::harness::agy::Catalog::read() + } + fn find_key(&mut self) -> Result> { let credentials = Credentials::read()?; let found = credentials.lookup().map(|key| key.source); @@ -649,6 +655,7 @@ mod scripted { pub test_calls: VecDeque>, /// The catalog's JSON, or what reading it fails with. pub catalog: Result<&'static str, &'static str>, + pub agy_catalog: Result<&'static str, &'static str>, /// Where a key is found, or what reading the Credentials fails with. pub key: Result, &'static str>, pub github_email: Result, &'static str>, @@ -671,6 +678,9 @@ mod scripted { installed: Harness::ALL.to_vec(), test_calls: VecDeque::new(), catalog: Ok(CATALOG), + agy_catalog: Ok( + "gemini-3.8-flash-high\tGemini 3.8 Flash (High)\ngemini-3.8-flash-medium\tGemini 3.8 Flash (Medium)\ngemini-3.8-flash-low\tGemini 3.8 Flash (Low)\n", + ), key: Ok(None), github_email: Ok(None), git_email: Err("git config --global user.email: exit status 1"), @@ -793,6 +803,13 @@ mod scripted { } } + fn agy_catalog(&mut self) -> Result { + match self.agy_catalog { + Ok(text) => crate::harness::agy::Catalog::parse(text), + Err(error) => bail!("{error}"), + } + } + fn find_key(&mut self) -> Result> { self.key.clone().map_err(|error| anyhow!("{error}")) } @@ -1107,7 +1124,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or agy [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1206,6 +1223,10 @@ effort = \"high\" [harness.codex] model = \"gpt-6.1-sol\" effort = \"max\" + +[harness.agy] +model = \"\" +effort = \"\" "; let mut outside = Scripted { user_config: Some(mine.to_string()), @@ -1235,7 +1256,7 @@ effort = \"max\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or agy [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1695,18 +1716,18 @@ always = false # quiet, please // The Harness, Model and Effort. #[test] - fn the_harness_question_lists_both_harnesses_and_defaults_to_claude() { + fn the_harness_question_lists_installed_harnesses_and_defaults_to_claude() { let mut outside = Scripted::answering(&ENTER_THROUGHOUT); setup(&mut outside).unwrap(); assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex [claude]: " + "Harness for every Run's sessions, claude or codex or agy [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "agy"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some("")); } @@ -1731,13 +1752,13 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude (not installed) or codex [codex]: " + "Harness for every Run's sessions, codex [codex]: " ); assert_eq!( outside.said()[..2], [ "claude is not installed: it isn't on PATH.", - "Choose claude or codex." + "Choose codex." ] ); assert_eq!( @@ -1747,7 +1768,7 @@ always = false # quiet, please } #[test] - fn with_neither_harness_installed_the_harness_settings_are_left_as_they_are() { + fn with_no_harness_installed_the_harness_settings_are_left_as_they_are() { let mut outside = Scripted { installed: Vec::new(), user_config: Some("[harness.claude]\nmodel = \"opus\"\n".to_string()), @@ -1759,9 +1780,8 @@ always = false # quiet, please assert_eq!( outside.said(), [ - "Harness for every Run's sessions: claude (not installed) or codex (not \ - installed). Sessions can run on neither here, so the harness settings stay as \ - they are; install claude or codex, then rerun `thirdshift setup`." + "No Harness is installed here, so the harness settings stay as they are; \ + install claude or codex or agy, then rerun `thirdshift setup`." ] ); let config = table(&outside); @@ -1791,6 +1811,65 @@ always = false # quiet, please } } + #[test] + fn agy_setup_checks_names_without_a_turn_and_lists_only_installed_harnesses() { + let mut outside = Scripted { + installed: vec![Harness::Agy], + ..Scripted::answering(&then_nothing_else(&[ + (HARNESS, ""), + ("Model for agy", "gemini-99"), + ("Model for agy", "Gemini-3.8-Flash"), + ("Effort for agy", "Max"), + ("Effort for agy", "Medium"), + ])) + }; + setup(&mut outside).unwrap(); + assert_eq!( + outside.prompts()[0], + "Harness for every Run's sessions, agy [agy]: " + ); + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("agy")); + assert_eq!( + config["harness"]["agy"]["model"].as_str(), + Some("gemini-3.8-flash") + ); + assert_eq!(config["harness"]["agy"]["effort"].as_str(), Some("medium")); + assert!( + outside + .said() + .iter() + .any(|line| line.contains("the Model gemini-99 is not in")) + ); + assert!( + outside + .said() + .iter() + .any(|line| line.contains("the Effort Max is not one")) + ); + assert!(outside.test_calls().is_empty()); + } + + #[test] + fn an_unreadable_agy_catalog_keeps_existing_harness_settings() { + let mut outside = Scripted { + installed: vec![Harness::Agy], + agy_catalog: Err("not signed in"), + user_config: Some( + "[harness]\ndefault = \"agy\"\n[harness.agy]\nmodel = \"gemini-3.8-flash-high\"\n" + .to_string(), + ), + ..Scripted::answering(&then_nothing_else(&[(HARNESS, "")])) + }; + setup(&mut outside).unwrap(); + assert_eq!( + table(&outside)["harness"]["agy"]["model"].as_str(), + Some("gemini-3.8-flash-high") + ); + assert!(outside.said()[0].contains("not signed in")); + assert!(outside.said()[0].contains("once agy models works")); + } + #[test] fn claudes_model_and_effort_are_written_after_a_test_call_with_them() { let mut outside = Scripted::answering(&then_nothing_else(&[ diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 4b0d489c2..481f0c19e 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -160,14 +160,10 @@ pub fn ask( }) } -/// Ask which Harness every Run's sessions run on, listing each, marked where -/// it isn't installed, and refusing that one; then its Model and Effort, -/// with those `current` sets for it as the defaults, checked as a Run -/// checks them, through the chosen adapter. The Harness's -/// default is the `current` one if it's installed, else the first one -/// installed, so `claude` when both are. With no -/// Harness to choose, or Codex chosen and its catalog unreadable, the answer -/// is `None`. +/// Ask which installed Harness every Run's sessions run on, then its +/// Model and Effort through the chosen adapter. Keep the configured default +/// when installed, otherwise prefer Claude, then the first installed. +/// With no Harness installed, leave the Harness settings as they are. fn ask_harness( outside: &mut impl Outside, current: &harness::Settings, @@ -176,17 +172,11 @@ fn ask_harness( .into_iter() .filter(|harness| outside.installed(*harness)) .collect(); - let harnesses: Vec = Harness::ALL + let harnesses = installed .iter() - .map(|harness| { - if installed.contains(harness) { - harness.name().to_string() - } else { - format!("{} (not installed)", harness.name()) - } - }) - .collect(); - let harnesses = harnesses.join(" or "); + .map(|harness| harness.name()) + .collect::>() + .join(" or "); let default = current .default .filter(|harness| installed.contains(harness)) @@ -194,9 +184,8 @@ fn ask_harness( let Some(default) = default else { let names = harness::names(); outside.say(format!( - "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ - the harness settings stay as they are; install {names}, then rerun \ - `thirdshift setup`." + "No Harness is installed here, so the harness settings stay as they are; \ + install {names}, then rerun `thirdshift setup`." )); return Ok(None); }; @@ -207,7 +196,7 @@ fn ask_harness( Some(default.name()), )?; match Harness::named(&name) { - None => outside.say(format!("Choose {}.", harness::names())), + None => outside.say(format!("Choose {harnesses}.")), Some(harness) if !installed.contains(&harness) => { outside.say(format!("{name} is not installed: it isn't on PATH.")) } diff --git a/src/skills.rs b/src/skills.rs index 504534867..109c03ea0 100644 --- a/src/skills.rs +++ b/src/skills.rs @@ -81,7 +81,7 @@ fn written_out() -> Result { /// `git` works in, unless it is there already. The file is shared by every /// worktree, so the entry is left there, under a lock other Runs from the /// same Launch directory take too. -fn exclude(git: &Git, exclude: &str) -> Result<()> { +pub(crate) fn exclude(git: &Git, exclude: &str) -> Result<()> { let _lock = git.lock("thirdshift-exclude.lock")?; let info = git.common_dir()?.join("info"); let path = info.join("exclude"); @@ -106,5 +106,5 @@ fn exclude(git: &Git, exclude: &str) -> Result<()> { .append(true) .open(&path) .and_then(|mut file| writeln!(file, "{separator}{exclude}")) - .with_context(|| format!("could not add the Factory skills to {}", path.display())) + .with_context(|| format!("could not add an exclude pattern to {}", path.display())) } diff --git a/tests/agy_sessions.rs b/tests/agy_sessions.rs new file mode 100644 index 000000000..3a7386d40 --- /dev/null +++ b/tests/agy_sessions.rs @@ -0,0 +1,364 @@ +//! A Command's sessions on Antigravity CLI, through real Runs against fake agy. +mod support; + +use support::{FACTORY_SKILLS, Scenario}; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m 'Add feature' +gh pr create --base main --head issue-7 --title 'Add feature' --body 'Closes #7' +"#; + +#[test] +fn a_run_uses_agy_with_unattended_flags_skills_and_settled_model_and_effort() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#"{AGENT_OPENS_PR} +echo '{{"event":"step_update","step_update":{{"step_index":2,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{{"parameters":{{"CommandLine":"cargo test"}}}}}}}}' +echo '{{"event":"step_update","step_update":{{"step_index":2,"state":"DONE","step_type":"tool","tool_name":"run_command"}}}}' +"#)); + let result = scenario.run(&[ + "harness", + "agy", + "model", + "Gemini-3.8-Flash", + "effort", + "Medium", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!(scenario.claude_calls().is_empty()); + let calls = scenario.agy_calls(); + assert_eq!(calls.len(), 1); + let call = &calls[0]; + let argv: Vec<&str> = call["argv"] + .as_array() + .unwrap() + .iter() + .map(|arg| arg.as_str().unwrap()) + .collect(); + assert_eq!( + &argv[..8], + [ + "-p", + "--dangerously-skip-permissions", + "--output-format", + "stream-json", + "--model", + "gemini-3.8-flash", + "--effort", + "medium" + ] + ); + assert!(!argv.contains(&"--sandbox")); + assert_eq!(call["stdin_null"], true); + assert_eq!(call["auto_update"], "true"); + assert_eq!(call["git_status"], ""); + assert!( + call["prompt"] + .as_str() + .unwrap() + .starts_with("/thirdshift-implement ") + ); + for skill in FACTORY_SKILLS { + assert!( + call["skill_files"][format!("{skill}/SKILL.md")] + .as_str() + .is_some(), + "{skill}" + ); + } + let checks = scenario.agy_checks(); + assert_eq!(checks.len(), 1); + assert_eq!(checks[0]["argv"], serde_json::json!(["models"])); + assert_eq!(checks[0]["auto_update"], "true"); + assert_eq!(checks[0]["stdin_null"], true); + assert!( + result + .stderr + .contains("sessions run on agy · gemini-3.8-flash · medium"), + "{}", + result.stderr + ); + for line in [ + "implement: session started", + "implement: $ cargo test", + "1200 input tokens (200 cached), 300 output tokens (150 thinking), 1500 total tokens", + ] { + assert!(result.stderr.contains(line), "{}", result.stderr); + } + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with agy · gemini-3.8-flash · medium") + ); + let sessions = "home/.thirdshift/logs/acme/widgets/sessions"; + let logs = scenario.entries(sessions); + let log = std::fs::read_to_string(scenario.path(&format!("{sessions}/{}", logs[0]))).unwrap(); + assert!(log.contains("\"event\": \"init\""), "{log}"); + assert!(log.contains("\"status\": \"SUCCESS\""), "{log}"); + assert!(log.contains("step_update"), "{log}"); +} + +#[test] +fn claude_only_instructions_are_linked_as_gemini_and_kept_out_of_git() { + let scenario = Scenario::new(); + std::fs::write(scenario.launch_dir().join("CLAUDE.md"), "project rules\n").unwrap(); + scenario.launch_git(&["add", "CLAUDE.md"]); + scenario.launch_git(&["commit", "-m", "Project rules"]); + scenario.launch_git(&["push", "origin", "main"]); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&["harness", "agy", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.agy_calls(); + assert_eq!(calls[0]["gemini_link"], "CLAUDE.md"); + assert_eq!(calls[0]["gemini_text"], "project rules\n"); + assert_eq!(calls[0]["git_status"], ""); + assert_eq!(scenario.origin_file("issue-7", "GEMINI.md"), None); +} + +#[test] +fn a_session_with_unfinished_work_is_resumed_by_conversation_with_the_same_flags() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} +echo '{{"event":"step_update","step_update":{{"step_index":2,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{{"parameters":{{"CommandLine":"cargo test"}}}}}}}}' +"#)); + let result = scenario.run(&[ + "harness", + "agy", + "model", + "gemini-3.8-flash-high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.agy_calls(); + assert_eq!(calls.len(), 2, "{}", result.stderr); + let args = calls[1]["argv"].as_array().unwrap(); + assert_eq!(args[6], "--conversation"); + assert_eq!(args[7], "fake-conversation-1"); + assert!( + calls[1]["prompt"] + .as_str() + .unwrap() + .starts_with("Your background work (cargo test) was killed") + ); + for call in calls { + assert_eq!(call["auto_update"], "true"); + assert_eq!(call["stdin_null"], true); + assert_eq!(call["argv"][1], "--dangerously-skip-permissions"); + assert_eq!(call["argv"][3], "stream-json"); + assert_eq!(call["argv"][5], "gemini-3.8-flash-high"); + } +} + +#[test] +fn existing_instruction_files_win_and_no_link_is_made_without_claude() { + for existing in [None, Some("AGENTS.md"), Some("GEMINI.md")] { + let scenario = Scenario::new(); + if let Some(existing) = existing { + std::fs::write(scenario.launch_dir().join("CLAUDE.md"), "claude rules\n").unwrap(); + std::fs::write(scenario.launch_dir().join(existing), "own rules\n").unwrap(); + scenario.launch_git(&["add", "."]); + scenario.launch_git(&["commit", "-m", "Project instructions"]); + scenario.launch_git(&["push", "origin", "main"]); + } + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&["harness", "agy", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.agy_calls(); + assert_eq!(calls[0]["gemini_link"], serde_json::Value::Null); + if existing == Some("GEMINI.md") { + assert_eq!(calls[0]["gemini_text"], "own rules\n"); + } + assert!( + scenario.agy_checks().is_empty(), + "no named settings need a catalog check" + ); + } +} + +#[test] +fn bad_models_efforts_and_catalog_failure_stop_before_any_work() { + for (model, effort, error) in [ + ("gemini-99", "high", "choose one of gemini-3.8-flash-high"), + ("Gemini-3.1-Pro", "Medium", "choose one of low, high"), + ( + "gemini-3.8-flash", + "", + "requires an Effort: choose one of low, medium, high", + ), + ( + "gemini-3.8-flash-high", + "Max", + "choose one of low, medium, high", + ), + ] { + let scenario = Scenario::new(); + let mut args = vec!["harness", "agy", "model", model]; + if !effort.is_empty() { + args.extend(["effort", effort]); + } + let url = scenario.issue_url(7); + args.push(&url); + let result = scenario.run(&args); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(error), "{}", result.stderr); + assert!(scenario.agy_calls().is_empty()); + assert!(scenario.issue_labels(7).is_empty()); + assert_eq!(scenario.entries("work"), ["widgets"]); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); + } + let scenario = Scenario::new(); + let result = scenario.run_with_env( + &[ + "harness", + "agy", + "model", + "gemini-3.8-flash-high", + &scenario.issue_url(7), + ], + &[("FAKE_AGY_CATALOG_ERROR", "sign in first")], + ); + assert_eq!(result.code, Some(1)); + assert!( + result.stderr.contains( + "agy models failed, so Antigravity CLI's Models can't be read: sign in first" + ), + "{}", + result.stderr + ); + assert!(scenario.agy_calls().is_empty()); +} + +#[test] +fn an_agy_user_config_and_command_overrides_choose_its_own_settings() { + let scenario = Scenario::new(); + scenario.user_config_is("[harness]\ndefault = \"agy\"\n[harness.agy]\nmodel = \"Gemini 3.8 Flash (High)\"\neffort = \"HIGH\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[&scenario.issue_url(7), "effort", "Low"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("sessions run on agy · gemini-3.8-flash-high · low"), + "{}", + result.stderr + ); +} + +#[test] +fn an_error_result_or_nonzero_exit_fails_with_agys_own_error() { + for code in [0, 3] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#" +echo '{{"event":"result","result":{{"status":"ERROR","error":"backend unavailable","response":""}}}}' > "$FAKE_AGY_RESULT" +exit {code} +"#)); + let result = scenario.run(&["harness", "agy", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + let ending = if code == 0 { + "agy's turn failed" + } else { + "agy exited 3" + }; + assert!( + result + .stderr + .contains(&format!("{ending}: backend unavailable")), + "{}", + result.stderr + ); + assert_eq!(scenario.agy_calls().len(), 1); + } +} + +#[test] +fn an_architecture_review_uses_the_final_result_instead_of_text_deltas() { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent"]); + scenario.user_config_is("[harness]\ndefault = \"agy\"\n"); + scenario.agent_does(r#" +gh issue create --title Plan --body Plan --label needs-triage > /dev/null +echo '{"event":"step_update","step_update":{"step_index":1,"step_type":"agent_response","state":"DONE","text_delta":"Architecture review plan: https://github.com/acme/widgets/issues/99"}}' +echo 'Architecture review plan: https://github.com/acme/widgets/issues/8' > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#); + let result = scenario.run(&["architect", "--plan-only"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("published the plan https://github.com/acme/widgets/issues/8"), + "{}", + result.stderr + ); + assert_eq!(scenario.agy_calls().len(), 1); + assert!( + scenario.agy_calls()[0]["prompt"] + .as_str() + .unwrap() + .starts_with("/thirdshift-improve-codebase-architecture") + ); +} + +#[test] +fn a_repair_uses_the_commands_agy_model_effort_and_environment() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} +gh fake checks "$(git rev-parse HEAD)" '[{{"name":"test","conclusion":"failure","url":"https://ci.example/test"}}]' +"#)); + scenario.agent_does_for_in_session(7, 2, r#" +echo fix > fix.txt +git add fix.txt +git commit -q -m Fix +gh fake checks "$(git rev-parse HEAD)" '[{"name":"test","conclusion":"success","url":"https://ci.example/test"}]' +"#); + let result = scenario.run(&[ + "harness", + "agy", + "model", + "gemini-3.8-flash", + "effort", + "high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.agy_calls(); + assert_eq!(calls.len(), 2); + assert!(calls[1]["prompt"].as_str().unwrap().contains("CI")); + for call in calls { + assert_eq!(call["argv"][5], "gemini-3.8-flash"); + assert_eq!(call["argv"][7], "high"); + assert_eq!(call["auto_update"], "true"); + assert_eq!(call["stdin_null"], true); + } + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_interrupt_stops_an_agy_session_and_saves_its_work() { + let scenario = Scenario::new(); + scenario.agent_does_for( + 7, + &format!( + "echo 'half done' > wip.txt\ntouch '{}'\nsleep 900\n", + scenario.path("agy-started").display() + ), + ); + let result = scenario.run_and_signal( + &["harness", "agy", &scenario.issue_url(7)], + "agy-started", + "TERM", + ); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains("interrupted"), "{}", result.stderr); + assert_eq!( + scenario.origin_file("issue-7", "wip.txt"), + Some("half done\n".to_string()) + ); +} diff --git a/tests/fakes.rs b/tests/fakes.rs index 0b197d18a..b69827870 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -13,6 +13,8 @@ #![cfg_attr(test, allow(dead_code))] +#[path = "fakes/agy.rs"] +mod agy; #[path = "fakes/claude.rs"] mod claude; #[path = "fakes/codex.rs"] @@ -110,6 +112,7 @@ fn main() { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), + "agy" => agy::main(args.collect()), _ => die(&format!("fake: no fake is called {name}"), 2), } exit(0); diff --git a/tests/fakes/agy.rs b/tests/fakes/agy.rs new file mode 100644 index 000000000..84d38d095 --- /dev/null +++ b/tests/fakes/agy.rs @@ -0,0 +1,153 @@ +//! Fake Antigravity CLI: a free tab-separated catalog and recorded agy +//! stream envelopes, running the same per-session scripts as fake Claude. +use std::fs; +use std::io::Write; +use std::process::Command; + +use crate::claude::{exit_code, git_here, script_for, skills_snapshot}; +use crate::codex::stdin_is_null; +use crate::json::{Array, Bool, Json, Null, number, object, string}; + +const CATALOG: &str = "gemini-3.8-flash-high\tGemini 3.8 Flash (High)\ngemini-3.8-flash-medium\tGemini 3.8 Flash (Medium)\ngemini-3.8-flash-low\tGemini 3.8 Flash (Low)\ngemini-3.1-pro-high\tGemini 3.1 Pro (High)\ngemini-3.1-pro-low\tGemini 3.1 Pro (Low)\n"; + +fn emit(event: Json) { + println!("{event}"); + std::io::stdout().flush().unwrap(); +} + +pub fn main(argv: Vec) { + let catalog = argv == ["models"]; + let record_path = crate::env_path(if catalog { + "FAKE_AGY_CHECK_RECORD" + } else { + "FAKE_AGY_RECORD" + }); + let lock = crate::lock_beside(&record_path); + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let records = crate::append_record( + &record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ( + "prompt", + if catalog { + Null + } else { + argv.last().map(string).unwrap_or(Null) + }, + ), + ("branch", string(branch)), + ( + "cwd", + string(std::env::current_dir().unwrap().display().to_string()), + ), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ( + "auto_update", + string(std::env::var("AGY_CLI_DISABLE_AUTO_UPDATE").unwrap_or_default()), + ), + ( + "skill_files", + if catalog { + object([]) + } else { + skills_snapshot(".agents/skills") + }, + ), + ( + "gemini_link", + fs::read_link("GEMINI.md") + .ok() + .map(|p| string(p.display().to_string())) + .unwrap_or(Null), + ), + ( + "gemini_text", + fs::read_to_string("GEMINI.md") + .ok() + .map(string) + .unwrap_or(Null), + ), + ]), + ); + drop(lock); + if catalog { + if let Ok(error) = std::env::var("FAKE_AGY_CATALOG_ERROR") { + crate::die(&error, 1); + } + print!("{CATALOG}"); + crate::exit(0); + } + assert!(argv.iter().any(|arg| arg == "-p")); + let records = records.items(); + let session = records.len(); + let conversation = argv + .windows(2) + .find(|args| args[0] == "--conversation") + .map(|args| args[1].clone()) + .unwrap_or_else(|| format!("fake-conversation-{session}")); + emit(object([ + ("event", string("init")), + ("conversation_id", string(&conversation)), + ( + "init", + object([("permission_mode", string("always-proceed"))]), + ), + ])); + let final_message = format!("{}.final-message.{session}", record_path.display()); + let error = format!("{}.error.{session}", record_path.display()); + let result = format!("{}.result.{session}", record_path.display()); + let code = exit_code( + Command::new("bash") + .arg("-e") + .arg(script_for(records)) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_AGY_ERROR", &error) + .env("FAKE_AGY_RESULT", &result) + .status() + .unwrap(), + ); + if let Ok(result) = fs::read_to_string(result) { + emit(crate::json::parse(&result).unwrap()); + crate::exit(code); + } + let response = fs::read_to_string(final_message).unwrap_or_default(); + let error = + fs::read_to_string(error).unwrap_or_else(|_| format!("the agent's script exited {code}")); + emit(object([ + ("event", string("result")), + ( + "result", + object([ + ("conversation_id", string(conversation)), + ( + "status", + string(if code == 0 { "SUCCESS" } else { "ERROR" }), + ), + ("response", string(response)), + ( + "error", + if code == 0 { + Null + } else { + string(error.trim_end()) + }, + ), + ("num_turns", number(1)), + ( + "usage", + object([ + ("input_tokens", number(1200)), + ("cache_read_tokens", number(200)), + ("output_tokens", number(300)), + ("thinking_tokens", number(150)), + ("total_tokens", number(1500)), + ]), + ), + ]), + ), + ])); + crate::exit(code); +} diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 4afee8063..2396018a3 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -49,7 +49,7 @@ const CATALOG: &str = r#"{"models": [ ]}"#; /// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. -fn stdin_is_null() -> bool { +pub fn stdin_is_null() -> bool { let stdin = std::io::stdin() .as_fd() .try_clone_to_owned() diff --git a/tests/setup.rs b/tests/setup.rs index f830ad39c..26719a880 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -52,7 +52,8 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[pickup]", "[harness]", "[harness.claude]", - "[harness.codex]" + "[harness.codex]", + "[harness.agy]" ], "{text}" ); @@ -70,7 +71,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "agy"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index b851c6b75..7771d63aa 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -11,17 +11,18 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 5] = [ +const SOURCES: [&str; 6] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), include_str!("../fakes/codex.rs"), + include_str!("../fakes/agy.rs"), include_str!("../fakes/json.rs"), ]; /// Put the fake `gh`, `claude` and `codex` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex"] { + for name in ["gh", "claude", "codex", "agy"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index f593337b0..34ec5b166 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -603,6 +603,8 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_CLAUDE_SCRIPT", self.path("claude-script.sh")) .env("FAKE_CLAUDE_RECORD", self.path("claude-calls.json")) .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) + .env("FAKE_AGY_RECORD", self.path("agy-calls.json")) + .env("FAKE_AGY_CHECK_RECORD", self.path("agy-checks.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, @@ -815,6 +817,21 @@ test -f {root}/{COPY_REPLACED} } } + pub fn agy_calls(&self) -> Vec { + self.agy_records("agy-calls.json") + } + + pub fn agy_checks(&self) -> Vec { + self.agy_records("agy-checks.json") + } + + fn agy_records(&self, file: &str) -> Vec { + fs::read_to_string(self.path(file)) + .ok() + .map(|text| serde_json::from_str(&text).unwrap()) + .unwrap_or_default() + } + /// Assert every `claude` call found every Factory skill, by its /// `thirdshift-` name, in its worktree's `.claude/skills/`. pub fn assert_every_session_found_the_factory_skills(&self) { From e83a16a1f0ec84a5457f8a8ccb14ccb8976e0ee5 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:41:06 -0400 Subject: [PATCH 05/16] Add Muse Code Harness (#438) --- CONTEXT.md | 2 +- README.md | 16 +- ...rness-runs-unattended-and-fully-trusted.md | 27 ++ src/args.rs | 4 +- src/asks.rs | 1 + src/config.rs | 13 +- src/harness.rs | 15 +- src/harness/adapter.rs | 11 +- src/harness/claude.rs | 2 +- src/harness/codex.rs | 2 +- src/harness/muse.rs | 156 +++++++++ src/harness/muse/catalog.rs | 131 +++++++ src/harness/muse/log.rs | 192 ++++++++++ src/harness/muse/stream.rs | 198 +++++++++++ src/progress.rs | 7 +- src/session.rs | 5 +- src/setup.rs | 78 ++++- src/setup/questions.rs | 20 +- tests/fakes.rs | 5 +- tests/fakes/muse.rs | 256 ++++++++++++++ tests/muse_sessions.rs | 328 ++++++++++++++++++ tests/setup.rs | 5 +- tests/support/fakes.rs | 9 +- tests/support/mod.rs | 12 + 24 files changed, 1439 insertions(+), 56 deletions(-) create mode 100644 docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md create mode 100644 src/harness/muse.rs create mode 100644 src/harness/muse/catalog.rs create mode 100644 src/harness/muse/log.rs create mode 100644 src/harness/muse/stream.rs create mode 100644 tests/fakes/muse.rs create mode 100644 tests/muse_sessions.rs diff --git a/CONTEXT.md b/CONTEXT.md index 3db2d76f9..a2547ffff 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -31,7 +31,7 @@ The message thirdshift starts or resumes an agent session with. It carries the * _Avoid_: instructions, system prompt **Harness**: -The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`) or Codex (`codex`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. +The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`), Codex (`codex`) or Muse Code (`muse`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. _Avoid_: agent, provider, backend **Model**: diff --git a/README.md b/README.md index a5ffc0e51..e472eb3af 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or muse; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -254,11 +254,15 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.muse] +model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own +effort = "" # how hard that Model reasons; default blank, for Muse Code's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude` when both are, and `codex` when only it is. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. The other Harness's section is left as it is. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing only those installed on `PATH`. An unavailable Harness or unknown name is refused and asked again. The default is the configured Harness when installed, else the first installed, starting with `claude`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. Muse proposes `muse-spark-1.3` and checks the answers against its cache, or with a minimal call; a refusal asks for the Model and Effort again. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -289,6 +293,8 @@ A Run reads the file before any work. One that isn't valid TOML, or that has a k ### Harness, Model and Effort +The Harnesses are Claude Code (`claude`), Codex (`codex`) and Muse Code (`muse`). + Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: ```sh @@ -301,6 +307,12 @@ With `harness codex`, every session runs `codex exec --json --dangerously-bypass Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. +With `harness muse`, every session and Resume runs `muse exec --json --yolo`, with stdin null and `MUSE_NO_AUTO_UPDATE=1`. `--yolo` disables approvals and the sandbox and trusts the worktree ([ADR-0013](docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md)). Muse reads the project's `AGENTS.md`, falling back to `CLAUDE.md`, and also reads `~/.claude/CLAUDE.md`. The Factory skills are linked into `.agents/skills/` and kept out of git. The prompt's first line tells the model to load its `thirdshift-` with its skill tool. If the stream shows no `read_skill` load of that skill, thirdshift writes a warning to the progress lines and Command log; it neither fails nor retries the session. + +Muse's named Model is checked against its cached catalog under `~/.local/share/muse/model-catalog/` when present, matching its id or display label regardless of case. Without that cache, thirdshift makes a minimal test call with the same flags and environment. Effort is checked locally before any call: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` or `ultra`. Model and Effort reach Muse as `--model` and `--reasoning-effort`. Setup proposes `muse-spark-1.3`; leaving the Model unset uses Muse's own default, `muse-spark-1.3-contributor`, which allows content to be used for product improvement. + +Muse's session id comes from `stream.id`, and a Resume uses `--session-id ` with every flag and environment override again. Progress lines name session starts, tools and errors. After exit, thirdshift reads the last reply and token totals from Muse's own `sessions/////session.jsonl`. Architecture reviews see that last reply alone, rather than Muse's terminal event's joined replies. A missing or unreadable log falls back to the stream text with no usage. `run.terminal.failed` or a non-zero exit fails the Run with Muse's error. An interrupt sends SIGTERM, then SIGKILL after ten seconds if needed. + The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. ### Email diff --git a/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md b/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md new file mode 100644 index 000000000..2672bdabf --- /dev/null +++ b/docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md @@ -0,0 +1,27 @@ +# Every Harness runs unattended and fully trusted in the worktree + +Sessions run with no human to answer an approval prompt or a trust question. Their commits and pushes write to the main repository's shared `.git`, outside the worktree. So every **Harness** runs its sessions, and every Resume of one, with approvals off, its sandbox off, and the worktree trusted. Each Harness carries the flags and environment that do this. They are fixed, and no user setting changes them: + +| Harness | Flags and environment | +|---|---| +| Claude Code | `--permission-mode auto` | +| Codex | `--dangerously-bypass-approvals-and-sandbox` | +| agy | `--dangerously-skip-permissions`, never `--sandbox` | +| Grok Build | `--always-approve`, `GROK_FOLDER_TRUST=0` | +| Muse Code | `--yolo` | +| OpenCode | `--auto`, `--standalone` | + +The worktree is the only boundary. This generalises the Codex part of ADR 0012. Sources: `docs/research/harness-candidates.md`, `docs/research/codex-headless-harness.md`. + +## Considered Options + +- **Each CLI's own sandbox.** Rejected. On stock Ubuntu (`kernel.apparmor_restrict_unprivileged_userns = 1`), bubblewrap can't start, so every sandboxed command fails under Codex and Muse. The sandboxes of agy and Grok make the main checkout's `.git` read-only, which blocks commits. A sandboxed mode for machines where it works would be a separate issue. +- **A one-time `grok --trust` in each repository's main checkout**, done or documented by Setup. Rejected. A repository nobody had trusted would run its sessions without `AGENTS.md` or the Factory skills, and nothing would fail. A scheduled Pickup run is exactly the case where nobody would notice. +- **Using the CLI's settings for approval and trust**, such as a user config that already sets always-approve. Rejected. A session would then behave differently on every machine. Passing the flags each time makes them explicit. +- **OpenCode through the user's shared background service** (its default). Rejected. A SIGTERM leaves the turn and its shell command running in the service, and thirdshift doesn't control that service's version or environment. `--standalone` keeps the server in the session's process group, and costs 2–3 s per session. + +## Consequences + +- A Harness's self-update is turned off in each session's environment too (`MUSE_NO_AUTO_UPDATE=1`, `GROK_DISABLE_AUTOUPDATER=1`, `AGY_CLI_DISABLE_AUTO_UPDATE=true`, `OPENCODE_DISABLE_AUTOUPDATE=1`). An update in the middle of a Run could replace the binary a Resume needs. agy ignores `=1`; only `=true` works. +- Each Harness reads its own instruction file and falls back to the other. thirdshift adds the fallback where a CLI lacks one: a `GEMINI.md` link for agy, and an `AGENTS.md` link for OpenCode, both to `CLAUDE.md` at the worktree root and kept out of git. Grok reads both files when both exist, and thirdshift leaves that as it is. +- Instructions in the user's `~/.claude/` reach only the Harnesses that read them: Claude, Grok and Muse. They don't reach Codex, agy or OpenCode. diff --git a/src/args.rs b/src/args.rs index b445828cf..86f9aed90 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex", + "harness must be followed by claude or codex or muse", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex, not gemini", + "harness must be followed by claude or codex or muse, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/asks.rs b/src/asks.rs index 74f5f68af..c1773c2a5 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -262,6 +262,7 @@ mod tests { default: Some(Harness::Codex), claude: model_and_effort("opus", "high"), codex: model_and_effort("gpt-6.1-sol", "max"), + muse: ModelAndEffort::default(), }, ..no_settings() } diff --git a/src/config.rs b/src/config.rs index 5bbd324d6..667f19809 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or muse; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -396,6 +396,10 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.muse] +model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own +effort = "" # how hard that Model reasons; default blank, for Muse Code's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -527,7 +531,9 @@ mod tests { "harness.claude.model", "harness.claude.effort", "harness.codex.model", - "harness.codex.effort" + "harness.codex.effort", + "harness.muse.model", + "harness.muse.effort" ] ); let commented_out: Vec<&str> = DEFAULTS @@ -577,6 +583,7 @@ mod tests { model: Some("gpt-6.1-sol".to_string()), effort: Some("max".to_string()), }, + muse: ModelAndEffort::default(), } ); assert_eq!(parse("").unwrap().harness, harness::Settings::default()); @@ -587,7 +594,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex, not \"gemini\"", + "harness.default must be claude or codex or muse, not \"gemini\"", ), ( "[harness]\ndefault = true\n", diff --git a/src/harness.rs b/src/harness.rs index fc047bb8d..4d431f0cb 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -12,6 +12,7 @@ use anyhow::{Result, bail}; mod adapter; pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod muse; pub use adapter::{Adapter, Invocation}; pub use codex::Catalog; @@ -21,6 +22,7 @@ pub use codex::Catalog; pub enum Harness { Claude, Codex, + Muse, } impl Harness { @@ -90,6 +92,8 @@ pub struct Settings { pub claude: ModelAndEffort, /// `[harness.codex]`. pub codex: ModelAndEffort, + /// `[harness.muse]`. + pub muse: ModelAndEffort, } impl Settings { @@ -187,13 +191,9 @@ impl Choice { codex::model_args(self) } - /// Check, before any work, that sessions can run on this, and settle the - /// Model and Effort on the names the Harness knows them by. The - /// Harness's CLI must be on `PATH`. A Model named for Claude must take a - /// minimal test call, with the Effort, if any. A Model or Effort named - /// for Codex must be in its catalog, which costs no tokens to read: each - /// is matched regardless of case, a Model by its slug or display name, - /// and becomes the name Codex takes. Each failure says what to change. + /// Check, before any work, that the CLI is on `PATH`, then settle the + /// Model and Effort through its adapter's catalog or minimal test call. + /// Each failure says what to change. pub fn check(&mut self) -> Result<()> { let adapter = self.harness.adapter(); let cli = adapter.name(); @@ -280,6 +280,7 @@ mod tests { default: Some(Harness::Codex), claude: set("opus", "high"), codex: set("gpt-6.1-sol", "max"), + muse: ModelAndEffort::default(), } } diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 93ba94b36..5dabaaf26 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -4,7 +4,7 @@ use std::path::Path; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex}; +use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex, muse}; use crate::progress::Stream; use crate::setup::Outside; @@ -28,7 +28,7 @@ pub trait Adapter: Sync { outside: &mut dyn Outside, current: &ModelAndEffort, ) -> Result>; - fn stream(&self, worktree: &Path) -> Box; + fn stream(&self, worktree: &Path, prompt: &str) -> Box; /// Fixed overrides, also used by the adapter's Model and Effort check. fn environment(&self) -> &'static [(&'static str, &'static str)] { @@ -62,7 +62,6 @@ pub struct Invocation { pub enum SkillLoading { Slash, Dollar, - #[expect(dead_code, reason = "reserved by #434 for skill-tool Harness adapters")] Tool, } @@ -84,4 +83,8 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 2] = [claude::REGISTRATION, codex::REGISTRATION]; +pub const REGISTERED: [(Harness, &dyn Adapter); 3] = [ + claude::REGISTRATION, + codex::REGISTRATION, + muse::REGISTRATION, +]; diff --git a/src/harness/claude.rs b/src/harness/claude.rs index 6b4f119b4..731d51211 100644 --- a/src/harness/claude.rs +++ b/src/harness/claude.rs @@ -58,7 +58,7 @@ impl Adapter for Claude { ) -> Result> { ask_claude(outside, current).map(Some) } - fn stream(&self, _worktree: &Path) -> Box { + fn stream(&self, _worktree: &Path, _prompt: &str) -> Box { Box::new(stream::ClaudeProgress::default()) } } diff --git a/src/harness/codex.rs b/src/harness/codex.rs index 734b15a3d..e46782693 100644 --- a/src/harness/codex.rs +++ b/src/harness/codex.rs @@ -60,7 +60,7 @@ impl Adapter for Codex { } } } - fn stream(&self, worktree: &Path) -> Box { + fn stream(&self, worktree: &Path, _prompt: &str) -> Box { Box::new(stream::CodexProgress::in_worktree(worktree)) } } diff --git a/src/harness/muse.rs b/src/harness/muse.rs new file mode 100644 index 000000000..e1c136057 --- /dev/null +++ b/src/harness/muse.rs @@ -0,0 +1,156 @@ +//! Muse Code: unattended sessions, skill-tool loading and the read after exit. + +mod catalog; +mod log; +mod stream; + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings, said}; +use crate::progress::Stream; +use crate::setup::{Outside, questions::ask_setting}; + +pub struct Muse; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Muse, &Muse); + +impl Adapter for Muse { + fn name(&self) -> &'static str { + "muse" + } + fn product_name(&self) -> &'static str { + "Muse Code" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Tool + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[("MUSE_NO_AUTO_UPDATE", "1")] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.muse + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.muse + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args = ["exec", "--json", "--yolo"].map(String::from).to_vec(); + if let Some(model) = &choice.model { + args.extend(["--model".into(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend(["--reasoning-effort".into(), effort.clone()]); + } + if let Some(id) = resume { + args.extend(["--session-id".into(), id.into()]); + } + args.push(self.skill_loading().prompt(prompt)); + Invocation { args, stdin: None } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + let settled = check_model_and_effort(&ModelAndEffort { + model: choice.model.clone(), + effort: choice.effort.clone(), + })?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + loop { + let chosen = ModelAndEffort { + model: ask_setting( + outside, + "Model", + Harness::Muse, + current.model.as_deref().or(Some("muse-spark-1.3")), + )?, + effort: ask_setting(outside, "Effort", Harness::Muse, current.effort.as_deref())?, + }; + let chosen = match catalog::settle_effort(chosen.effort.as_deref()) { + Ok(effort) => ModelAndEffort { effort, ..chosen }, + Err(error) => { + outside.say(format!("{error:#}")); + continue; + } + }; + match outside.muse_check(&chosen) { + Ok(settled) => return Ok(Some(settled)), + Err(error) => outside.say(format!("{error:#}")), + } + } + } + fn stream(&self, _worktree: &Path, prompt: &str) -> Box { + Box::new(stream::MuseProgress::for_prompt(prompt)) + } + fn read_after_exit(&self, _worktree: &Path, stream: Box) -> Box { + log::read_after_exit(stream) + } +} + +/// Muse's per-user data, where its catalog and session logs live. +fn data_dir() -> Option { + std::env::var_os("XDG_DATA_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".local/share"))) + .map(|dir| dir.join("muse")) +} + +/// The same check used by a Command and Setup: validate Effort first, then +/// use the cached catalog, falling back to a minimal call only without it. +pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result { + let mut settled = ModelAndEffort { + model: chosen.model.clone(), + effort: catalog::settle_effort(chosen.effort.as_deref())?, + }; + if let Some(model) = &settled.model { + if let Some(catalog) = catalog::Catalog::read()? { + crate::progress::step("checking the Model against Muse's cached catalog"); + settled.model = Some(catalog.settle_model(model)?); + } else { + let choice = Choice { + harness: Harness::Muse, + model: settled.model.clone(), + effort: settled.effort.clone(), + chosen_by: super::ChosenBy::UserConfig, + }; + crate::progress::step(format_args!( + "checking the Model {model} with a test call to muse" + )); + let invocation = Muse.session(&choice, None, "Reply with OK."); + let output = Command::new(Muse.name()) + .args(invocation.args) + .envs(Muse.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run muse to check the Model")?; + let mut stream = Muse.stream(Path::new("."), "Reply with OK."); + for line in String::from_utf8_lossy(&output.stdout).lines() { + stream.condense(line); + } + if !output.status.success() || stream.failed() { + bail!( + "muse refused a test call on the Model {model}: {}", + stream + .error() + .map(String::from) + .unwrap_or_else(|| said(&output)) + ); + } + } + } + Ok(settled) +} diff --git a/src/harness/muse/catalog.rs b/src/harness/muse/catalog.rs new file mode 100644 index 000000000..147b3c715 --- /dev/null +++ b/src/harness/muse/catalog.rs @@ -0,0 +1,131 @@ +//! Muse's cached model catalog costs no model call to read. + +use anyhow::{Context, Result, bail}; +use serde_json::Value; +use std::fs; + +pub struct Catalog { + models: Vec<(String, String)>, +} + +impl Catalog { + pub fn parse(text: &str) -> Result { + let json: Value = + serde_json::from_str(text).context("Muse's cached catalog is not JSON")?; + let rows = json["rows"] + .as_array() + .context("Muse's cached catalog has no rows")?; + let mut models = Vec::new(); + for row in rows.iter().filter(|row| row["visibility"] != "hidden") { + let id = row["model_id"] + .as_str() + .context("Muse's cached catalog has a row with no model_id")?; + models.push(( + id.to_string(), + row["display_label"].as_str().unwrap_or(id).to_string(), + )); + } + Ok(Self { models }) + } + + pub fn read() -> Result> { + let Some(dir) = super::data_dir().map(|dir| dir.join("model-catalog")) else { + return Ok(None); + }; + let entries = match fs::read_dir(&dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("could not read {}", dir.display())); + } + }; + let mut catalog = None; + for entry in entries { + let path = entry?.path(); + if path.extension().is_none_or(|extension| extension != "json") { + continue; + } + let parsed = Self::parse( + &fs::read_to_string(&path) + .with_context(|| format!("could not read {}", path.display()))?, + )?; + catalog + .get_or_insert_with(|| Self { models: Vec::new() }) + .models + .extend(parsed.models); + } + Ok(catalog) + } + + pub fn settle_model(&self, name: &str) -> Result { + self.models + .iter() + .find(|(id, label)| id.eq_ignore_ascii_case(name) || label.eq_ignore_ascii_case(name)) + .map(|(id, _)| id.clone()) + .with_context(|| { + format!( + "the Model {name} is not in Muse's cached catalog: choose one of {}", + self.models + .iter() + .map(|(id, _)| id.as_str()) + .collect::>() + .join(", ") + ) + }) + } +} + +pub const EFFORTS: [&str; 8] = [ + "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", +]; + +/// Muse rejects unknown Efforts locally; do so before any test call here too. +pub fn settle_effort(effort: Option<&str>) -> Result> { + let Some(effort) = effort else { + return Ok(None); + }; + if let Some(known) = EFFORTS + .iter() + .find(|known| known.eq_ignore_ascii_case(effort)) + { + return Ok(Some((*known).to_string())); + } + bail!( + "the Effort {effort} is not supported by Muse: choose one of {}", + EFFORTS.join(", ") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn visible_models_match_by_id_or_label_regardless_of_case() { + let catalog = Catalog::parse( + r#"{"rows":[ + {"model_id":"muse-spark-1.3","display_label":"Muse Spark 1.3","visibility":"visible"}, + {"model_id":"muse-image-1.0","visibility":"hidden"} + ]}"#, + ) + .unwrap(); + for name in ["MUSE-SPARK-1.3", "muse spark 1.3"] { + assert_eq!(catalog.settle_model(name).unwrap(), "muse-spark-1.3"); + } + assert!(catalog.settle_model("muse-image-1.0").is_err()); + assert!( + catalog + .settle_model("bad-model") + .unwrap_err() + .to_string() + .contains("choose one of muse-spark-1.3") + ); + } + + #[test] + fn malformed_catalogs_are_rejected() { + for text in ["broken", "{}", "{\"rows\":[{}]}"] { + assert!(Catalog::parse(text).is_err()); + } + } +} diff --git a/src/harness/muse/log.rs b/src/harness/muse/log.rs new file mode 100644 index 000000000..7c328ea84 --- /dev/null +++ b/src/harness/muse/log.rs @@ -0,0 +1,192 @@ +//! Read Muse's retained session records for the last reply and token usage. + +use crate::progress::Stream; +use anyhow::Result; +use serde_json::Value; +use std::fs; +use std::path::{Path, PathBuf}; + +#[derive(Default)] +pub struct SessionLog { + message: Option, + tokens: Option, +} + +#[derive(Default)] +struct Tokens { + input: u64, + cached: u64, + output: u64, +} + +impl SessionLog { + pub fn parse(text: &str, session_id: &str) -> Result { + let mut log = Self::default(); + for line in text.lines().filter(|line| !line.trim().is_empty()) { + let record: Value = serde_json::from_str(line)?; + if let Some(children) = record["children"].as_array() { + for child in children { + let record: Value = + serde_json::from_str(child["record_json"].as_str().unwrap_or(""))?; + log.record(&record, session_id); + } + } else { + log.record(&record, session_id); + } + } + Ok(log) + } + fn record(&mut self, record: &Value, session_id: &str) { + if record["stream"]["id"].as_str() != Some(session_id) || record["payload"]["kind"] != "run" + { + return; + } + let event = &record["payload"]["event"]; + match event["kind"].as_str() { + Some("assistant_message_committed") => { + self.message = event["text"].as_str().map(String::from); + } + Some("model_completed") => { + if let Some(usage) = event["usage"].as_object() { + let tokens = self.tokens.get_or_insert_default(); + tokens.input += usage + .get("input_tokens") + .and_then(Value::as_u64) + .unwrap_or(0); + tokens.cached += usage + .get("cached_tokens") + .and_then(Value::as_u64) + .unwrap_or(0); + tokens.output += usage + .get("output_tokens") + .and_then(Value::as_u64) + .unwrap_or(0); + } + } + _ => {} + } + } + fn summary(&self) -> Option { + self.tokens.as_ref().map(|tokens| { + format!( + "{} input tokens ({} cached), {} output tokens", + tokens.input, tokens.cached, tokens.output + ) + }) + } +} + +/// Date folders are year/month/day; never descend into sub-agent logs. +fn find_log(root: &Path, id: &str, depth: usize) -> Option { + let log = root.join(id).join("session.jsonl"); + if log.is_file() { + return Some(log); + } + if depth == 0 { + return None; + } + fs::read_dir(root) + .ok()? + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .find_map(|entry| find_log(&entry.path(), id, depth - 1)) +} + +pub fn read_after_exit(stream: Box) -> Box { + let log = stream + .session_id() + .filter(|id| !id.contains(['/', '\\']) && *id != "..") + .and_then(|id| { + let dir = super::data_dir()?.join("sessions"); + let path = find_log(&dir, id, 3)?; + SessionLog::parse(&fs::read_to_string(path).ok()?, id).ok() + }); + Box::new(AfterExit { stream, log }) +} + +struct AfterExit { + stream: Box, + log: Option, +} + +impl Stream for AfterExit { + fn condense(&mut self, raw: &str) -> Vec { + self.stream.condense(raw) + } + fn summary(&self) -> Option { + self.log.as_ref().and_then(SessionLog::summary) + } + fn final_message(&self) -> Option<&str> { + self.log + .as_ref() + .and_then(|log| log.message.as_deref()) + .or_else(|| self.stream.final_message()) + } + fn session_id(&self) -> Option<&str> { + self.stream.session_id() + } + fn killed_background_work(&self) -> Vec<&str> { + self.stream.killed_background_work() + } + fn failed(&self) -> bool { + self.stream.failed() + } + fn error(&self) -> Option<&str> { + self.stream.error() + } + fn warnings(&self) -> Vec { + self.stream.warnings() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // Retained records observed in Muse 1.4.2, trimmed to the fields read. + fn record(event: Value) -> Value { + json!({"stream":{"kind":"session","id":"s1"},"payload_type":"runtime.session", + "payload":{"kind":"run","event":event}}) + } + + #[test] + fn the_last_reply_is_kept_and_usage_is_summed_without_child_session_usage() { + let lines = [ + record(json!({"kind":"assistant_message_committed","text":"first reply"})), + record(json!({"kind":"model_completed","usage":{"input_tokens":26000,"cached_tokens":12000,"output_tokens":40}})), + json!({"stream":{"id":"child"},"payload":{"kind":"run","event":{"kind":"model_completed","usage":{"input_tokens":999999}}}}), + record(json!({"kind":"model_completed","usage":{"input_tokens":24000,"cached_tokens":20000,"output_tokens":30}})), + record(json!({"kind":"assistant_message_committed","text":"Architecture review idea: https://github.com/acme/widgets/issues/8"})), + ].map(|line| line.to_string()).join("\n"); + let log = SessionLog::parse(&lines, "s1").unwrap(); + assert_eq!( + log.message.as_deref(), + Some("Architecture review idea: https://github.com/acme/widgets/issues/8") + ); + assert_eq!( + log.summary().as_deref(), + Some("50000 input tokens (32000 cached), 70 output tokens") + ); + } + + #[test] + fn retained_permission_transactions_are_unwrapped() { + let message = record(json!({"kind":"assistant_message_committed","text":"last reply"})); + let frame = json!({"retained_frame":"session_permission_transaction","children":[{"record_json":message.to_string()}]}); + let log = SessionLog::parse(&frame.to_string(), "s1").unwrap(); + assert_eq!(log.message.as_deref(), Some("last reply")); + assert_eq!(log.summary(), None); + } + + #[test] + fn a_truncated_or_non_json_log_is_unreadable() { + for text in [ + "not JSON", + "{", + "{\"children\":[{\"record_json\":\"bad\"}]}", + ] { + assert!(SessionLog::parse(text, "s1").is_err()); + } + } +} diff --git a/src/harness/muse/stream.rs b/src/harness/muse/stream.rs new file mode 100644 index 000000000..721c216de --- /dev/null +++ b/src/harness/muse/stream.rs @@ -0,0 +1,198 @@ +//! Muse's JSONL event envelopes. Usage is read from its session log after exit. + +use crate::progress::{Stream, bash, shorten}; +use serde_json::Value; + +#[derive(Default)] +pub struct MuseProgress { + session_id: Option, + started: bool, + final_message: Option, + failure: Option, + expected_skill: Option, + skill_loaded: bool, + tasks: Vec<(String, String)>, + killed: Vec, +} + +impl MuseProgress { + pub fn for_prompt(prompt: &str) -> Self { + Self { + expected_skill: prompt + .strip_prefix('/') + .and_then(|rest| rest.split_whitespace().next()) + .filter(|name| name.starts_with("thirdshift-")) + .map(String::from), + ..Self::default() + } + } + + fn tool(&mut self, payload: &Value) -> Vec { + let facts = &payload["correlation_facts"]; + let name = facts["tool_name"].as_str().unwrap_or(""); + let text = payload["text"].as_str().unwrap_or(""); + if name == "read_skill" { + let header = text.lines().next().unwrap_or(""); + let skill = header + .strip_prefix(" Vec { + let Ok(event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if event["stream"]["kind"] == "session" + && let Some(id) = event["stream"]["id"].as_str() + { + self.session_id = Some(id.to_string()); + } + let payload = &event["payload"]; + match event["payload_type"].as_str() { + Some("run.lifecycle.started") if !self.started => { + self.started = true; + vec!["session started".into()] + } + Some("task.lifecycle.proposed") + if payload["event"]["task_kind"] + .as_str() + .is_some_and(|kind| kind.starts_with("model.")) => + { + self.final_message = None; + Vec::new() + } + Some("task.lifecycle.proposed") => { + let event = &payload["event"]; + if let (Some(id), Some(kind)) = + (event["task_id"].as_str(), event["task_kind"].as_str()) + && kind.starts_with("tool.") + { + self.tasks.push((id.into(), kind.into())); + } + Vec::new() + } + Some("task.lifecycle.cancelled") => { + let event = &payload["event"]; + if event["reason"] + .as_str() + .is_some_and(|reason| reason.contains("session ended")) + && let Some(id) = event["task_id"].as_str() + && let Some((_, description)) = self.tasks.iter().find(|(task, _)| task == id) + { + self.killed.push(description.clone()); + } + Vec::new() + } + Some("run.output.delta") => { + if let Some(text) = payload["text"].as_str() { + self.final_message.get_or_insert_default().push_str(text); + } + Vec::new() + } + Some("run.terminal.completed") => { + if self.final_message.is_none() { + self.final_message = payload["text"].as_str().map(String::from); + } + Vec::new() + } + Some("run.terminal.failed") => { + self.failure = Some( + payload["reason"] + .as_str() + .unwrap_or("Muse's turn failed") + .to_string(), + ); + vec![format!( + "error: {}", + shorten(self.failure.as_deref().unwrap()) + )] + } + Some("tool.result") => self.tool(payload), + _ => Vec::new(), + } + } + fn summary(&self) -> Option { + None + } + fn final_message(&self) -> Option<&str> { + self.final_message.as_deref() + } + fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed() { + Vec::new() + } else { + self.killed.iter().map(String::as_str).collect() + } + } + fn failed(&self) -> bool { + self.failure.is_some() + } + fn error(&self) -> Option<&str> { + self.failure.as_deref() + } + fn warnings(&self) -> Vec { + self.expected_skill + .as_ref() + .filter(|_| !self.skill_loaded) + .map(|skill| { + vec![format!( + "warning: the session never loaded {skill} with its skill tool" + )] + }) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn a_successful_read_skill_result_loads_only_the_named_skill() { + let mut stream = MuseProgress::for_prompt( + "/thirdshift-implement https://github.com/acme/widgets/issues/7", + ); + let result = |name: &str| { + json!({"stream":{"kind":"session","id":"s1"},"payload_type":"tool.result", "payload":{ + "correlation_facts":{"tool_name":"read_skill","outcome":"success"}, + "text":format!("\npath: /skill/SKILL.md\n") + }}).to_string() + }; + stream.condense(&result("thirdshift-tdd")); + assert_eq!(stream.warnings().len(), 1); + assert_eq!( + stream.condense(&result("thirdshift-implement")), + ["skill thirdshift-implement"] + ); + assert!(stream.warnings().is_empty()); + assert_eq!(stream.session_id(), Some("s1")); + } +} diff --git a/src/progress.rs b/src/progress.rs index e0e9e74d6..c670d841f 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1,5 +1,5 @@ //! Progress lines on stderr: thirdshift's own steps, and a session's -//! stream, Claude's stream-json or Codex's JSONL, condensed to one short +//! stream, condensed by its Harness adapter to one short //! line per notable event. Each line is stamped with the local time it was //! printed, so a stalled Run can be told from a busy one. Each is kept in the //! Command log too, if the command keeps one. @@ -119,6 +119,11 @@ pub trait Stream: Send { /// malformed lines give none, never an error. fn condense(&mut self, raw: &str) -> Vec; + /// Warnings reported once after exit, without failing or retrying the session. + fn warnings(&self) -> Vec { + Vec::new() + } + /// What the session-ended line says the session took, if the stream /// told. fn summary(&self) -> Option; diff --git a/src/session.rs b/src/session.rs index 779fa1a28..6d74324b9 100644 --- a/src/session.rs +++ b/src/session.rs @@ -237,7 +237,7 @@ impl Outside for OnMachine { log: &Path, ) -> Result { let invocation = self.adapter.session(&self.harness, resume, prompt); - let stream = self.adapter.stream(&self.worktree); + let stream = self.adapter.stream(&self.worktree, prompt); let stream = run(kind, self.adapter, &self.worktree, invocation, log, stream)?; Ok(Ended { session_id: stream.session_id().map(String::from), @@ -354,6 +354,9 @@ fn run( .map_err(|_| anyhow!("the session stream reader panicked"))?; let stream = adapter.read_after_exit(worktree, stream); + for warning in stream.warnings() { + progress::step(format_args!("{kind}: {warning}")); + } let elapsed = minutes_and_seconds(started.elapsed()); let summary = stream .summary() diff --git a/src/setup.rs b/src/setup.rs index d6804fcf3..1b3b5a2a1 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -367,6 +367,8 @@ pub(crate) trait Outside { fn test_call(&mut self, chosen: &ModelAndEffort) -> Result<()>; /// Read Codex's catalog of Models. fn codex_catalog(&mut self) -> Result; + /// Check Muse's settings against its cache, or with its minimal test call. + fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result; /// Read the Credentials, as strictly as a Run does: where a Resend API /// key is found, if anywhere. fn find_key(&mut self) -> Result>; @@ -459,6 +461,10 @@ impl Outside for OnMachine<'_> { Catalog::read() } + fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result { + crate::harness::muse::check_model_and_effort(chosen) + } + fn find_key(&mut self) -> Result> { let credentials = Credentials::read()?; let found = credentials.lookup().map(|key| key.source); @@ -609,6 +615,7 @@ mod scripted { Say(String), /// It made Claude's test call on this Model and Effort. Tested(ModelAndEffort), + MuseChecked(ModelAndEffort), /// It wrote this new User config. WriteNew(String), /// It replaced the User config with this. @@ -649,6 +656,7 @@ mod scripted { pub test_calls: VecDeque>, /// The catalog's JSON, or what reading it fails with. pub catalog: Result<&'static str, &'static str>, + pub muse_checks: VecDeque>, /// Where a key is found, or what reading the Credentials fails with. pub key: Result, &'static str>, pub github_email: Result, &'static str>, @@ -662,7 +670,7 @@ mod scripted { impl Scripted { /// A terminal answering `answers` in order, then closing stdin, - /// with both Harnesses installed, no key, no email to suggest and no + /// with every Harness installed, no key, no email to suggest and no /// User config. pub fn answering(answers: &[(&'static str, &'static str)]) -> Self { Scripted { @@ -671,6 +679,7 @@ mod scripted { installed: Harness::ALL.to_vec(), test_calls: VecDeque::new(), catalog: Ok(CATALOG), + muse_checks: VecDeque::new(), key: Ok(None), github_email: Ok(None), git_email: Err("git config --global user.email: exit status 1"), @@ -793,6 +802,14 @@ mod scripted { } } + fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result { + self.calls.push(Call::MuseChecked(chosen.clone())); + self.muse_checks + .pop_front() + .unwrap_or_else(|| Ok(chosen.clone())) + .map_err(|error| anyhow!("{error}")) + } + fn find_key(&mut self) -> Result> { self.key.clone().map_err(|error| anyhow!("{error}")) } @@ -1107,7 +1124,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or muse [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1206,6 +1223,10 @@ effort = \"high\" [harness.codex] model = \"gpt-6.1-sol\" effort = \"max\" + +[harness.muse] +model = \"\" +effort = \"\" "; let mut outside = Scripted { user_config: Some(mine.to_string()), @@ -1235,7 +1256,7 @@ effort = \"max\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or muse [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1692,21 +1713,56 @@ always = false # quiet, please } } + #[test] + fn muse_setup_proposes_its_non_contributor_model_and_retries_a_refused_check() { + let mut outside = Scripted { + installed: vec![Harness::Muse], + muse_checks: [ + Err("Muse refused the Model"), + Ok(ModelAndEffort { + model: Some("muse-spark-1.3".to_string()), + effort: Some("high".to_string()), + }), + ] + .into(), + ..Scripted::answering(&then_nothing_else(&[ + (HARNESS, "muse"), + ("Model for muse", "bad-model"), + ("Effort for muse", "low"), + ("Model for muse", ""), + ("Effort for muse", "High"), + ])) + }; + setup(&mut outside).unwrap(); + assert_eq!( + outside.prompts()[1], + "Model for muse, - for muse's own default [muse-spark-1.3]: " + ); + assert!(outside.said().contains(&"Muse refused the Model")); + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("muse")); + assert_eq!( + config["harness"]["muse"]["model"].as_str(), + Some("muse-spark-1.3") + ); + assert_eq!(config["harness"]["muse"]["effort"].as_str(), Some("high")); + } + // The Harness, Model and Effort. #[test] - fn the_harness_question_lists_both_harnesses_and_defaults_to_claude() { + fn the_harness_question_lists_installed_harnesses_and_defaults_to_claude() { let mut outside = Scripted::answering(&ENTER_THROUGHOUT); setup(&mut outside).unwrap(); assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex [claude]: " + "Harness for every Run's sessions, claude or codex or muse [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "muse"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some("")); } @@ -1731,13 +1787,13 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude (not installed) or codex [codex]: " + "Harness for every Run's sessions, codex [codex]: " ); assert_eq!( outside.said()[..2], [ "claude is not installed: it isn't on PATH.", - "Choose claude or codex." + "Choose claude or codex or muse." ] ); assert_eq!( @@ -1747,7 +1803,7 @@ always = false # quiet, please } #[test] - fn with_neither_harness_installed_the_harness_settings_are_left_as_they_are() { + fn with_no_harness_installed_the_harness_settings_are_left_as_they_are() { let mut outside = Scripted { installed: Vec::new(), user_config: Some("[harness.claude]\nmodel = \"opus\"\n".to_string()), @@ -1759,9 +1815,7 @@ always = false # quiet, please assert_eq!( outside.said(), [ - "Harness for every Run's sessions: claude (not installed) or codex (not \ - installed). Sessions can run on neither here, so the harness settings stay as \ - they are; install claude or codex, then rerun `thirdshift setup`." + "No Harness is installed here, so the harness settings stay as they are; install claude or codex or muse, then rerun `thirdshift setup`." ] ); let config = table(&outside); diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 4b0d489c2..7ad9bb688 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -165,7 +165,7 @@ pub fn ask( /// with those `current` sets for it as the defaults, checked as a Run /// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one -/// installed, so `claude` when both are. With no +/// installed, starting with `claude`. With no /// Harness to choose, or Codex chosen and its catalog unreadable, the answer /// is `None`. fn ask_harness( @@ -176,17 +176,11 @@ fn ask_harness( .into_iter() .filter(|harness| outside.installed(*harness)) .collect(); - let harnesses: Vec = Harness::ALL + let harnesses = installed .iter() - .map(|harness| { - if installed.contains(harness) { - harness.name().to_string() - } else { - format!("{} (not installed)", harness.name()) - } - }) - .collect(); - let harnesses = harnesses.join(" or "); + .map(|harness| harness.name()) + .collect::>() + .join(" or "); let default = current .default .filter(|harness| installed.contains(harness)) @@ -194,9 +188,7 @@ fn ask_harness( let Some(default) = default else { let names = harness::names(); outside.say(format!( - "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ - the harness settings stay as they are; install {names}, then rerun \ - `thirdshift setup`." + "No Harness is installed here, so the harness settings stay as they are; install {names}, then rerun `thirdshift setup`." )); return Ok(None); }; diff --git a/tests/fakes.rs b/tests/fakes.rs index 0b197d18a..06f13d79c 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh`, `claude` and `codex` the end-to-end tests put first on +//! The fake `gh`, `claude`, `codex` and `muse` the end-to-end tests put first on //! PATH: one executable, which acts as whichever its name says. See //! `fakes/gh.rs`, `fakes/claude.rs` and `fakes/codex.rs` for what each does, //! and `support/fakes.rs` for how the harness builds it. @@ -21,6 +21,8 @@ mod codex; mod gh; #[path = "fakes/json.rs"] mod json; +#[path = "fakes/muse.rs"] +mod muse; use std::fs::{self, File}; use std::io::Write; @@ -110,6 +112,7 @@ fn main() { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), + "muse" => muse::main(args.collect()), _ => die(&format!("fake: no fake is called {name}"), 2), } exit(0); diff --git a/tests/fakes/muse.rs b/tests/fakes/muse.rs new file mode 100644 index 000000000..44139d847 --- /dev/null +++ b/tests/fakes/muse.rs @@ -0,0 +1,256 @@ +//! Fake Muse: runs the scenario script, emits envelopes, and writes its session log. + +use std::fs; +use std::io::Write; +use std::os::fd::AsFd; +use std::os::unix::fs::MetadataExt; +use std::process::Command; + +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::json::{Array, Bool, Json, Null, object, string}; + +/// Where Muse finds a worktree's project skills. +const SKILLS: &str = ".agents/skills"; + +/// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. +pub(crate) fn stdin_is_null() -> bool { + let stdin = std::io::stdin() + .as_fd() + .try_clone_to_owned() + .map(fs::File::from) + .and_then(|file| file.metadata()); + let (Ok(stdin), Ok(null)) = (stdin, fs::metadata("/dev/null")) else { + return false; + }; + stdin.rdev() == null.rdev() && stdin.ino() == null.ino() +} + +/// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts +/// them until its script ends. A caught signal, unlike an ignored one, is +/// back to its default in the script, which can trap it. +fn outlast_interrupts() { + // From the C library, which the standard library links: SIGINT and + // SIGTERM are 2 and 15 on Linux and macOS alike. + unsafe extern "C" { + fn signal(signum: i32, handler: extern "C" fn(i32)) -> usize; + } + extern "C" fn caught(_: i32) {} + for signum in [2, 15] { + // SAFETY: the handler does nothing, so it is async-signal-safe. + unsafe { signal(signum, caught) }; + } +} + +/// Print one line of the stream. +fn emit(event: Json) { + println!("{event}"); + std::io::stdout().flush().unwrap(); +} + +pub fn main(argv: Vec) { + match argv.first().map(String::as_str) { + Some("exec") if argv.iter().any(|arg| arg == "--json") => {} + _ => crate::die( + &format!( + "fake muse: unexpected arguments {}", + crate::python_list(&argv) + ), + 2, + ), + } + if argv.last().is_some_and(|prompt| prompt == "Reply with OK.") { + let path = crate::env_path("FAKE_MUSE_RECORD").with_extension("checks.json"); + crate::append_record( + &path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ("stdin_null", Bool(stdin_is_null())), + ( + "no_auto_update", + string(std::env::var("MUSE_NO_AUTO_UPDATE").unwrap_or_default()), + ), + ]), + ); + if argv + .windows(2) + .any(|pair| pair[0] == "--model" && pair[1] == "bad-model") + { + emit_event( + "fake-check", + "run.terminal.failed", + object([("reason", string("model does not exist or you lack access"))]), + ); + crate::exit(1); + } + emit_event( + "fake-check", + "run.terminal.completed", + object([("text", string("OK"))]), + ); + crate::exit(0); + } + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let record_path = &crate::env_path("FAKE_MUSE_RECORD"); + + let lock = crate::lock_beside(record_path); + let cwd = std::env::current_dir().unwrap(); + let cwd = cwd.to_str().unwrap(); + let records = crate::append_record( + record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ("prompt", argv.last().map(string).unwrap_or(Null)), + ("cwd", string(cwd)), + ("branch", string(branch)), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ( + "no_auto_update", + string(std::env::var("MUSE_NO_AUTO_UPDATE").unwrap_or_default()), + ), + ("skill_files", skills_snapshot(SKILLS)), + ("beside_skills", beside_skills_snapshot(SKILLS)), + ]), + ); + let records = records.items(); + let script = script_for(records); + drop(lock); + + let session = records.len(); + outlast_interrupts(); + let id = argv + .windows(2) + .find(|pair| pair[0] == "--session-id") + .map(|pair| pair[1].clone()) + .unwrap_or_else(|| format!("fake-muse-{session}")); + emit_event(&id, "run.lifecycle.started", object([])); + let prompt = argv.last().unwrap(); + if let Some(rest) = prompt.strip_prefix("Load ") { + let skill = rest.split_whitespace().next().unwrap(); + if std::env::var("FAKE_MUSE_SKIP_SKILL").is_err() { + emit_event( + &id, + "tool.result", + object([ + ( + "correlation_facts", + object([ + ("tool_name", string("read_skill")), + ("outcome", string("success")), + ]), + ), + ( + "text", + string(format!( + "\n" + )), + ), + ]), + ); + } + } + // Beside the record, not in $TMPDIR, which tests expect to be left empty. + let final_message = format!("{}.final-message.{session}", record_path.display()); + let error = format!("{}.error.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(&script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_MUSE_ERROR", &error) + .status() + .unwrap(); + let code = exit_code(status); + let message = fs::read_to_string(&final_message).unwrap_or_else(|_| "done".to_string()); + emit_event( + &id, + "run.output.delta", + object([("text", string(&message))]), + ); + if std::env::var("FAKE_MUSE_NO_LOG").is_err() { + let dir = crate::env_path("HOME") + .join(".local/share/muse/sessions/2026/10/06") + .join(&id); + fs::create_dir_all(&dir).unwrap(); + let log = dir.join("session.jsonl"); + let previous = object([ + ("kind", string("assistant_message_committed")), + ("text", string("earlier reply")), + ]); + let last = object([ + ("kind", string("assistant_message_committed")), + ("text", string(&message)), + ]); + let usage = object([ + ("kind", string("model_completed")), + ( + "usage", + object([ + ("input_tokens", crate::json::number(1200)), + ("cached_tokens", crate::json::number(200)), + ("output_tokens", crate::json::number(300)), + ("reasoning_tokens", crate::json::number(100)), + ]), + ), + ]); + let record = |event| { + object([ + ( + "stream", + object([("kind", string("session")), ("id", string(&id))]), + ), + ("payload_type", string("runtime.session")), + ( + "payload", + object([("kind", string("run")), ("event", event)]), + ), + ]) + }; + fs::write( + &log, + format!( + "{}\n{}\n{}\n", + record(previous), + record(usage), + record(last) + ), + ) + .unwrap(); + } + if std::env::var("FAKE_MUSE_CORRUPT_LOG").is_ok() { + let log = crate::env_path("HOME") + .join(".local/share/muse/sessions/2026/10/06") + .join(&id) + .join("session.jsonl"); + fs::write(log, "truncated {").unwrap(); + } + if code == 0 { + // Muse's terminal text joins all replies; the session log doesn't. + emit_event( + &id, + "run.terminal.completed", + object([("text", string(format!("earlier reply{message}")))]), + ); + crate::exit(0); + } + let message = fs::read_to_string(&error) + .map(|message| message.trim_end().to_string()) + .unwrap_or_else(|_| format!("the agent's script exited {code}")); + emit_event( + &id, + "run.terminal.failed", + object([("reason", string(message))]), + ); + crate::exit(1); +} + +fn emit_event(id: &str, kind: &str, payload: Json) { + emit(object([ + ( + "stream", + object([("kind", string("session")), ("id", string(id))]), + ), + ("payload_type", string(kind)), + ("payload", payload), + ])); +} diff --git a/tests/muse_sessions.rs b/tests/muse_sessions.rs new file mode 100644 index 000000000..a4e74f1bc --- /dev/null +++ b/tests/muse_sessions.rs @@ -0,0 +1,328 @@ +//! Muse sessions through the scenario harness. +mod support; + +use support::Scenario; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +#[test] +fn muse_runs_unattended_with_linked_skills_and_reports_usage_from_its_log() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let url = scenario.issue_url(7); + let result = scenario.run(&["harness", "muse", &url]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.muse_calls(); + assert_eq!(calls.len(), 1); + let call = &calls[0]; + assert_eq!(call["argv"][0], "exec"); + assert_eq!(call["argv"][1], "--json"); + assert_eq!(call["argv"][2], "--yolo"); + assert_eq!(call["stdin_null"], true); + assert_eq!(call["no_auto_update"], "1"); + assert_eq!(call["git_status"], ""); + assert_eq!( + call["prompt"].as_str().unwrap().lines().next().unwrap(), + format!("Load thirdshift-implement with your skill tool. {url}") + ); + scenario.assert_every_muse_session_found_the_factory_skills(); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); + assert!( + result + .stderr + .contains("1200 input tokens (200 cached), 300 output tokens"), + "{}", + result.stderr + ); + assert!(!result.stderr.contains("warning:"), "{}", result.stderr); +} + +const CATALOG: &str = r#"{"rows":[ + {"model_id":"muse-spark-1.3","display_label":"Muse Spark 1.3","visibility":"visible"}, + {"model_id":"muse-spark-1.3-contributor","display_label":"Contributor","visibility":"visible","is_default":true} +]}"#; + +fn cache_catalog(scenario: &Scenario) { + let dir = scenario.path("home/.local/share/muse/model-catalog"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("meta.json"), CATALOG).unwrap(); +} + +fn assert_no_work(scenario: &Scenario, result: &support::RunResult, error: &str) { + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(error), "{}", result.stderr); + assert!(scenario.muse_calls().is_empty()); + assert!(scenario.issue_labels(7).is_empty()); + assert_eq!(scenario.entries("work"), vec!["widgets"]); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); +} + +#[test] +fn an_unknown_model_in_muses_cached_catalog_fails_before_any_call_or_work() { + let scenario = Scenario::new(); + cache_catalog(&scenario); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "muse", + "model", + "bad-model", + &scenario.issue_url(7), + ]); + assert_no_work( + &scenario, + &result, + "the Model bad-model is not in Muse's cached catalog", + ); + assert!(!scenario.path("muse-calls.checks.json").exists()); +} + +#[test] +fn without_a_catalog_muse_checks_the_model_with_an_unattended_minimal_call() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "muse", + "model", + "bad-model", + &scenario.issue_url(7), + ]); + assert_no_work( + &scenario, + &result, + "model does not exist or you lack access", + ); + let checks: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(scenario.path("muse-calls.checks.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + checks[0]["argv"], + serde_json::json!([ + "exec", + "--json", + "--yolo", + "--model", + "bad-model", + "Reply with OK." + ]) + ); + assert_eq!(checks[0]["stdin_null"], true); + assert_eq!(checks[0]["no_auto_update"], "1"); +} + +#[test] +fn invalid_effort_fails_locally_even_without_a_catalog() { + let scenario = Scenario::new(); + let result = scenario.run(&[ + "harness", + "muse", + "model", + "bad-model", + "effort", + "bogus", + &scenario.issue_url(7), + ]); + assert_no_work( + &scenario, + &result, + "the Effort bogus is not supported by Muse", + ); + assert!(!scenario.path("muse-calls.checks.json").exists()); +} + +#[test] +fn the_user_config_selects_muse_and_its_cached_model_and_effort_are_settled() { + let scenario = Scenario::new(); + cache_catalog(&scenario); + scenario.user_config_is("[harness]\ndefault = \"muse\"\n[harness.muse]\nmodel = \"Muse Spark 1.3\"\neffort = \"Max\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[&scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.muse_calls(); + assert_eq!( + calls[0]["argv"].as_array().unwrap()[..7], + serde_json::json!([ + "exec", + "--json", + "--yolo", + "--model", + "muse-spark-1.3", + "--reasoning-effort", + "max" + ]) + .as_array() + .unwrap()[..] + ); + assert!(!scenario.path("muse-calls.checks.json").exists()); + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with muse · muse-spark-1.3 · max") + ); +} + +#[test] +fn missing_log_uses_stream_text_without_usage_and_a_missing_skill_warns_once_in_both_logs() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run_with_env( + &["harness", "muse", &scenario.issue_url(7)], + &[("FAKE_MUSE_NO_LOG", "1"), ("FAKE_MUSE_SKIP_SKILL", "1")], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert_eq!(scenario.muse_calls().len(), 1); + let warning = "warning: the session never loaded thirdshift-implement with its skill tool"; + assert_eq!(result.stderr.matches(warning).count(), 1); + assert!(!result.stderr.contains("input tokens")); + let dir = "home/.thirdshift/logs/acme/widgets/commands/issue"; + let name = scenario.entries(dir).pop().unwrap(); + let log = std::fs::read_to_string(scenario.path(&format!("{dir}/{name}"))).unwrap(); + assert_eq!(log.matches(warning).count(), 1); +} + +#[test] +fn a_failed_turn_or_nonzero_exit_fails_with_muses_error() { + for script in [ + "echo 'catalog unavailable' > \"$FAKE_MUSE_ERROR\"\nexit 1", + r#"echo '{"stream":{"kind":"session","id":"fake-muse-1"},"payload_type":"run.terminal.failed","payload":{"reason":"catalog unavailable"}}'"#, + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, script); + let result = scenario.run(&["harness", "muse", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result.stderr.contains("catalog unavailable"), + "{}", + result.stderr + ); + assert_eq!(scenario.muse_calls().len(), 1); + } +} + +#[test] +fn muse_resumes_the_stream_session_id_with_every_flag_and_environment_again() { + let scenario = Scenario::new(); + cache_catalog(&scenario); + scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} +echo '{{"stream":{{"kind":"session","id":"fake-muse-1"}},"payload_type":"task.lifecycle.proposed","payload":{{"event":{{"task_id":"task-1","task_kind":"tool.bash"}}}}}}' +echo '{{"stream":{{"kind":"session","id":"fake-muse-1"}},"payload_type":"task.lifecycle.cancelled","payload":{{"event":{{"task_id":"task-1","reason":"session ended"}}}}}}' +"#)); + let result = scenario.run(&[ + "harness", + "muse", + "model", + "muse-spark-1.3", + "effort", + "high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.muse_calls(); + assert_eq!(calls.len(), 2, "{}", result.stderr); + let args = calls[1]["argv"].as_array().unwrap(); + assert_eq!( + args[..9], + serde_json::json!([ + "exec", + "--json", + "--yolo", + "--model", + "muse-spark-1.3", + "--reasoning-effort", + "high", + "--session-id", + "fake-muse-1" + ]) + .as_array() + .unwrap()[..] + ); + assert_eq!(calls[1]["stdin_null"], true); + assert_eq!(calls[1]["no_auto_update"], "1"); +} + +#[test] +fn architecture_reviews_receive_only_the_last_reply_with_or_without_the_session_log() { + for log_setting in [ + None, + Some("FAKE_MUSE_NO_LOG"), + Some("FAKE_MUSE_CORRUPT_LOG"), + ] { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does_in_session( + 1, + r#" +url=$(gh issue create --title "An idea" --body "The idea" --label needs-triage) +printf 'Architecture review idea: %s\n' "$url" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let env = log_setting + .map(|setting| vec![(setting, "1")]) + .unwrap_or_default(); + let result = scenario.run_with_env(&["architect", "harness", "muse"], &env); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert_eq!( + result.stdout.trim(), + "https://github.com/acme/widgets/issues/8" + ); + assert_eq!( + result.stderr.contains("1200 input tokens"), + log_setting.is_none() + ); + } +} + +#[test] +fn setup_checks_muses_proposed_model_with_the_same_flags_and_environment() { + let scenario = Scenario::new(); + let result = scenario.run_on_terminal( + &["setup"], + &[], + &[ + ("Harness for every Run", "muse"), + ("Model for muse", ""), + ("Effort for muse", "low"), + ("Every Run a Merge run", ""), + ("Every Run first fast-forwards", ""), + ("Run notifications", ""), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let config: toml::Table = result.user_config.unwrap().parse().unwrap(); + assert_eq!( + config["harness"]["muse"]["model"].as_str(), + Some("muse-spark-1.3") + ); + let checks: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(scenario.path("muse-calls.checks.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + checks[0]["argv"], + serde_json::json!([ + "exec", + "--json", + "--yolo", + "--model", + "muse-spark-1.3", + "--reasoning-effort", + "low", + "Reply with OK." + ]) + ); + assert_eq!(checks[0]["stdin_null"], true); + assert_eq!(checks[0]["no_auto_update"], "1"); +} diff --git a/tests/setup.rs b/tests/setup.rs index f830ad39c..874fa6107 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -52,7 +52,8 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[pickup]", "[harness]", "[harness.claude]", - "[harness.codex]" + "[harness.codex]", + "[harness.muse]" ], "{text}" ); @@ -70,7 +71,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "muse"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index b851c6b75..bf54f5608 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh`, `claude` and `codex`, built from `tests/fakes.rs` with plain +//! The fake `gh`, `claude`, `codex` and `muse`, built from `tests/fakes.rs` with plain //! `rustc` the first time a test needs them and kept in Cargo's scratch //! directory for tests. They aren't a binary target, so neither the crate nor //! a release ships them. @@ -11,17 +11,18 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 5] = [ +const SOURCES: [&str; 6] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), include_str!("../fakes/codex.rs"), + include_str!("../fakes/muse.rs"), include_str!("../fakes/json.rs"), ]; -/// Put the fake `gh`, `claude` and `codex` in `bin`. +/// Put the fake `gh`, `claude`, `codex` and `muse` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex"] { + for name in ["gh", "claude", "codex", "muse"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index f593337b0..484c0e9ba 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -603,6 +603,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_CLAUDE_SCRIPT", self.path("claude-script.sh")) .env("FAKE_CLAUDE_RECORD", self.path("claude-calls.json")) .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) + .env("FAKE_MUSE_RECORD", self.path("muse-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, @@ -815,6 +816,17 @@ test -f {root}/{COPY_REPLACED} } } + /// Every call the fake Muse received. + pub fn muse_calls(&self) -> Vec { + fs::read_to_string(self.path("muse-calls.json")) + .map(|text| serde_json::from_str(&text).unwrap()) + .unwrap_or_default() + } + + pub fn assert_every_muse_session_found_the_factory_skills(&self) { + assert_found_the_factory_skills(self.muse_calls()); + } + /// Assert every `claude` call found every Factory skill, by its /// `thirdshift-` name, in its worktree's `.claude/skills/`. pub fn assert_every_session_found_the_factory_skills(&self) { From 5ac1f719f46a2f59efd6cb43238f57b9a5b7091a Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:43:45 -0400 Subject: [PATCH 06/16] Avoid inferring killed agy work from active tool steps (#436) --- src/harness/agy.rs | 40 +++++++++++++++++++++++++++++++++++++++ src/harness/agy/stream.rs | 26 ++++++------------------- tests/agy_sessions.rs | 34 --------------------------------- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/src/harness/agy.rs b/src/harness/agy.rs index 7a83e3ce0..9199b1fd8 100644 --- a/src/harness/agy.rs +++ b/src/harness/agy.rs @@ -113,3 +113,43 @@ impl Adapter for Agy { Box::new(stream::AgyProgress::default()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::harness::ChosenBy; + + /// Resume uses the same public invocation interface as a new session. + #[test] + fn a_resume_reuses_every_flag_and_environment_and_the_conversation_id() { + let chosen = Choice { + harness: Harness::Agy, + model: Some("gemini-3.8-flash".to_string()), + effort: Some("medium".to_string()), + chosen_by: ChosenBy::Command, + }; + let adapter = chosen.harness.adapter(); + let resume = adapter.session(&chosen, Some("recorded-conversation"), "Continue."); + assert_eq!( + resume.args, + [ + "-p", + "--dangerously-skip-permissions", + "--output-format", + "stream-json", + "--model", + "gemini-3.8-flash", + "--effort", + "medium", + "--conversation", + "recorded-conversation", + "Continue." + ] + ); + assert_eq!(resume.stdin, None); + assert_eq!( + adapter.environment(), + [("AGY_CLI_DISABLE_AUTO_UPDATE", "true")] + ); + } +} diff --git a/src/harness/agy/stream.rs b/src/harness/agy/stream.rs index 64cd70132..df6ee94d0 100644 --- a/src/harness/agy/stream.rs +++ b/src/harness/agy/stream.rs @@ -2,14 +2,13 @@ //! usage come only from the final result, never concatenated text deltas. use crate::progress::{Stream, bash, shorten}; use serde_json::Value; -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; #[derive(Default)] pub struct AgyProgress { session_id: Option, tools: HashSet, result: Option, - running: BTreeMap, } impl Stream for AgyProgress { @@ -33,9 +32,6 @@ impl Stream for AgyProgress { let Some(index) = step["step_index"].as_u64() else { return Vec::new(); }; - if step["state"] == "DONE" { - self.running.remove(&index); - } if !self.tools.insert(index) { return Vec::new(); } @@ -59,15 +55,6 @@ impl Stream for AgyProgress { ) } }; - if step["state"] == "ACTIVE" && matches!(name, "run_command" | "invoke_subagent") { - self.running.insert( - index, - description - .strip_prefix("$ ") - .unwrap_or(&description) - .to_string(), - ); - } vec![description] } Some("result") => { @@ -100,10 +87,9 @@ impl Stream for AgyProgress { self.session_id.as_deref() } fn killed_background_work(&self) -> Vec<&str> { - if self.failed() { - return Vec::new(); - } - self.running.values().map(String::as_str).collect() + // ACTIVE describes a tool step, not background work killed at exit. + // The recorded protocol has no explicit killed-work signal. + Vec::new() } fn failed(&self) -> bool { self.result @@ -162,11 +148,11 @@ mod tests { } #[test] - fn unfinished_commands_are_resume_work_only_on_success() { + fn an_active_tool_step_does_not_prove_background_work_was_killed() { let mut stream = AgyProgress::default(); stream.condense(TOOL); stream.condense(RESULT); - assert_eq!(stream.killed_background_work(), ["cargo test"]); + assert!(stream.killed_background_work().is_empty()); stream.condense(r#"{"event":"result","result":{"status":"ERROR","error":"interrupted"}}"#); assert!(stream.killed_background_work().is_empty()); } diff --git a/tests/agy_sessions.rs b/tests/agy_sessions.rs index 3a7386d40..49deb0aad 100644 --- a/tests/agy_sessions.rs +++ b/tests/agy_sessions.rs @@ -118,40 +118,6 @@ fn claude_only_instructions_are_linked_as_gemini_and_kept_out_of_git() { assert_eq!(scenario.origin_file("issue-7", "GEMINI.md"), None); } -#[test] -fn a_session_with_unfinished_work_is_resumed_by_conversation_with_the_same_flags() { - let scenario = Scenario::new(); - scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} -echo '{{"event":"step_update","step_update":{{"step_index":2,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{{"parameters":{{"CommandLine":"cargo test"}}}}}}}}' -"#)); - let result = scenario.run(&[ - "harness", - "agy", - "model", - "gemini-3.8-flash-high", - &scenario.issue_url(7), - ]); - assert_eq!(result.code, Some(0), "{}", result.stderr); - let calls = scenario.agy_calls(); - assert_eq!(calls.len(), 2, "{}", result.stderr); - let args = calls[1]["argv"].as_array().unwrap(); - assert_eq!(args[6], "--conversation"); - assert_eq!(args[7], "fake-conversation-1"); - assert!( - calls[1]["prompt"] - .as_str() - .unwrap() - .starts_with("Your background work (cargo test) was killed") - ); - for call in calls { - assert_eq!(call["auto_update"], "true"); - assert_eq!(call["stdin_null"], true); - assert_eq!(call["argv"][1], "--dangerously-skip-permissions"); - assert_eq!(call["argv"][3], "stream-json"); - assert_eq!(call["argv"][5], "gemini-3.8-flash-high"); - } -} - #[test] fn existing_instruction_files_win_and_no_link_is_made_without_claude() { for existing in [None, Some("AGENTS.md"), Some("GEMINI.md")] { From 6fc043a8967ef0e80b8c763661b7647d857cfe31 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:46:32 -0400 Subject: [PATCH 07/16] Include agy settings in the config key regression check (#436) --- src/config.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 8c80d6a3f..533a6920d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -531,7 +531,9 @@ mod tests { "harness.claude.model", "harness.claude.effort", "harness.codex.model", - "harness.codex.effort" + "harness.codex.effort", + "harness.agy.model", + "harness.agy.effort" ] ); let commented_out: Vec<&str> = DEFAULTS From b6830267e55ff783796739c84e7fa6fc1d939bcf Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 12:47:45 -0400 Subject: [PATCH 08/16] Cover Muse's multi-reply stream fallback (#438) --- src/harness/muse/stream.rs | 17 +++++++++++++++++ tests/fakes/muse.rs | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/harness/muse/stream.rs b/src/harness/muse/stream.rs index 721c216de..e0a77a166 100644 --- a/src/harness/muse/stream.rs +++ b/src/harness/muse/stream.rs @@ -175,6 +175,23 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn the_stream_fallback_keeps_the_last_reply_instead_of_muses_joined_terminal_text() { + // Muse's recorded two-reply sequence, with irrelevant envelope fields removed. + let lines = [ + r#"{"stream":{"kind":"session","id":"s1"},"payload_type":"run.output.delta","payload":{"text":"MANGO"}}"#, + r#"{"stream":{"kind":"session","id":"s1"},"payload_type":"task.lifecycle.proposed","payload":{"event":{"task_kind":"model.meta.response"}}}"#, + r#"{"stream":{"kind":"session","id":"s1"},"payload_type":"run.output.delta","payload":{"text":"OK MANGO"}}"#, + r#"{"stream":{"kind":"session","id":"s1"},"payload_type":"run.terminal.completed","payload":{"text":"MANGOOK MANGO"}}"#, + ]; + let mut stream = MuseProgress::default(); + for line in lines { + stream.condense(line); + } + assert_eq!(stream.final_message(), Some("OK MANGO")); + assert_eq!(stream.summary(), None); + } + #[test] fn a_successful_read_skill_result_loads_only_the_named_skill() { let mut stream = MuseProgress::for_prompt( diff --git a/tests/fakes/muse.rs b/tests/fakes/muse.rs index 44139d847..3c3654d34 100644 --- a/tests/fakes/muse.rs +++ b/tests/fakes/muse.rs @@ -162,6 +162,22 @@ pub fn main(argv: Vec) { .unwrap(); let code = exit_code(status); let message = fs::read_to_string(&final_message).unwrap_or_else(|_| "done".to_string()); + emit_event( + &id, + "run.output.delta", + object([("text", string("earlier reply"))]), + ); + emit_event( + &id, + "task.lifecycle.proposed", + object([( + "event", + object([ + ("task_id", string("model-final")), + ("task_kind", string("model.meta.response")), + ]), + )]), + ); emit_event( &id, "run.output.delta", From 347dfe772f1b490d092f26ec1bfdce044bf54cf3 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:02:45 -0400 Subject: [PATCH 09/16] Add Grok Build Harness (#437) --- CONTEXT.md | 2 +- README.md | 20 ++- src/args.rs | 4 +- src/asks.rs | 1 + src/config.rs | 15 +- src/harness.rs | 8 +- src/harness/adapter.rs | 8 +- src/harness/grok.rs | 115 ++++++++++++++ src/harness/grok/catalog.rs | 198 ++++++++++++++++++++++++ src/harness/grok/stream.rs | 156 +++++++++++++++++++ src/main.rs | 7 +- src/progress.rs | 2 +- src/session.rs | 2 +- src/setup.rs | 92 +++++++++-- src/setup/questions.rs | 4 +- tests/fakes.rs | 3 + tests/fakes/codex.rs | 4 +- tests/fakes/grok.rs | 131 ++++++++++++++++ tests/fixtures/grok-models.json | 6 + tests/fixtures/grok-models.txt | 9 ++ tests/grok_sessions.rs | 261 ++++++++++++++++++++++++++++++++ tests/process_tree.rs | 5 + tests/setup.rs | 5 +- tests/support/fakes.rs | 7 +- tests/support/mod.rs | 18 +++ 25 files changed, 1046 insertions(+), 37 deletions(-) create mode 100644 src/harness/grok.rs create mode 100644 src/harness/grok/catalog.rs create mode 100644 src/harness/grok/stream.rs create mode 100644 tests/fakes/grok.rs create mode 100644 tests/fixtures/grok-models.json create mode 100644 tests/fixtures/grok-models.txt create mode 100644 tests/grok_sessions.rs diff --git a/CONTEXT.md b/CONTEXT.md index 3db2d76f9..1c8881191 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -31,7 +31,7 @@ The message thirdshift starts or resumes an agent session with. It carries the * _Avoid_: instructions, system prompt **Harness**: -The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`) or Codex (`codex`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. +The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`), Codex (`codex`) or Grok Build (`grok`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. _Avoid_: agent, provider, backend **Model**: diff --git a/README.md b/README.md index fdc837fc9..74181775f 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or grok; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -254,11 +254,15 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude` when both are, and `codex` when only it is. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. The other Harness's section is left as it is. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing Claude Code, Codex and Grok Build, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude`, then `codex`, then `grok`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For Grok Build, Setup lists `grok models` and the chosen Model's Efforts from its refreshed cache, and checks each answer against them. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -289,19 +293,25 @@ A Run reads the file before any work. One that isn't valid TOML, or that has a k ### Harness, Model and Effort +The available Harnesses are Claude Code (`claude`), Codex (`codex`) and Grok Build (`grok`). + Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: ```sh thirdshift https://github.com///issues/ model claude-opus-5-5 effort high ``` -For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. +For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Grok Build takes `-m ` and `--reasoning-effort `. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. +With `harness grok`, every session runs `grok -p --always-approve --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag is always explicit. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. + +Grok reads both `AGENTS.md` and `CLAUDE.md` when both exist, along with `AGENT.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md` and project rules under `.grok`, `.claude` and `.cursor`. It also reads `~/.grok/rules`, `~/.claude/CLAUDE.md` and `~/.claude/rules`; thirdshift writes none of these user-level files. Grok Build uses `~/.grok` for sign-in and sessions (or `GROK_HOME` when set). + Interrupting a Command stops every session's process tree on every Harness, including commands that started their own process group or session. thirdshift snapshots the descendants with `ps` on Linux and macOS, sends each process group the Harness's stop signals with a 10-second grace period each, then takes another snapshot before SIGKILL to catch commands started in the meantime. If the CLI exits on an earlier signal, thirdshift kills the remaining commands immediately without another grace period. -Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. +Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. On Grok Build, `grok models` lists the available Models without a model call and refreshes `models_cache.json` under `GROK_HOME` or `~/.grok`; thirdshift reads that cache for each Model's supported Efforts. Both are matched regardless of case and passed as Grok names them. With no Model named, a named Effort is checked against Grok's current default Model. Unknown Models and unsupported Efforts fail with the valid choices, and an unreadable catalog fails before work. The check has the same null stdin and fixed environment as sessions. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. @@ -859,7 +869,7 @@ Editing a skill in `skills/` has no effect until you rebuild and reinstall ([ADR The [Prompts and skills page](https://thirdshift.app/prompts/) is generated from the prompts, the `claude` arguments and the skills, and each Session prompt is also generated as a Markdown file in [`prompts/`](prompts/), with a [`README.md`](prompts/README.md) that lists them with the `claude` command lines. `cargo test` fails until they are regenerated after a change to any of them. Regenerate the page and `prompts/` with `UPDATE_PROMPTS=1 cargo test prompts_page`, which also deletes any file in `prompts/` that it doesn't generate. -The integration tests swap in a fake `gh` and `claude`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. +The integration tests swap in a fake `gh`, `claude`, `codex` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. ### The Rust version diff --git a/src/args.rs b/src/args.rs index b445828cf..ec34ac380 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex", + "harness must be followed by claude or codex or grok", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex, not gemini", + "harness must be followed by claude or codex or grok, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/asks.rs b/src/asks.rs index 74f5f68af..c03951ddf 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -262,6 +262,7 @@ mod tests { default: Some(Harness::Codex), claude: model_and_effort("opus", "high"), codex: model_and_effort("gpt-6.1-sol", "max"), + ..harness::Settings::default() }, ..no_settings() } diff --git a/src/config.rs b/src/config.rs index 5bbd324d6..2d13e7a3f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or grok; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -396,6 +396,10 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -527,7 +531,9 @@ mod tests { "harness.claude.model", "harness.claude.effort", "harness.codex.model", - "harness.codex.effort" + "harness.codex.effort", + "harness.grok.model", + "harness.grok.effort" ] ); let commented_out: Vec<&str> = DEFAULTS @@ -577,6 +583,7 @@ mod tests { model: Some("gpt-6.1-sol".to_string()), effort: Some("max".to_string()), }, + ..harness::Settings::default() } ); assert_eq!(parse("").unwrap().harness, harness::Settings::default()); @@ -587,7 +594,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex, not \"gemini\"", + "harness.default must be claude or codex or grok, not \"gemini\"", ), ( "[harness]\ndefault = true\n", @@ -868,7 +875,7 @@ mod tests { let config: toml::Table = completed.parse().unwrap(); let harness = config["harness"].as_table().unwrap(); assert!(harness.contains_key("default"), "{text:?}:\n{completed}"); - for name in ["claude", "codex"] { + for name in ["claude", "codex", "grok"] { let settings = harness[name].as_table().unwrap(); for key in ["model", "effort"] { assert!(settings.contains_key(key), "{text:?}:\n{completed}"); diff --git a/src/harness.rs b/src/harness.rs index d20d5074c..f835fcb81 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -12,6 +12,7 @@ use anyhow::{Result, bail}; mod adapter; pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod grok; mod process_tree; pub use adapter::{Adapter, Invocation}; @@ -22,6 +23,7 @@ pub use codex::Catalog; pub enum Harness { Claude, Codex, + Grok, } impl Harness { @@ -91,6 +93,8 @@ pub struct Settings { pub claude: ModelAndEffort, /// `[harness.codex]`. pub codex: ModelAndEffort, + /// `[harness.grok]`. + pub grok: ModelAndEffort, } impl Settings { @@ -194,7 +198,8 @@ impl Choice { /// minimal test call, with the Effort, if any. A Model or Effort named /// for Codex must be in its catalog, which costs no tokens to read: each /// is matched regardless of case, a Model by its slug or display name, - /// and becomes the name Codex takes. Each failure says what to change. + /// and becomes the name Codex takes. Grok Build checks its free model + /// list and refreshed effort cache. Each failure says what to change. pub fn check(&mut self) -> Result<()> { let adapter = self.harness.adapter(); let cli = adapter.name(); @@ -281,6 +286,7 @@ mod tests { default: Some(Harness::Codex), claude: set("opus", "high"), codex: set("gpt-6.1-sol", "max"), + ..Settings::default() } } diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 10a03c07c..0a1f60b90 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -5,7 +5,7 @@ use std::process::Child; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex}; +use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex, grok}; use crate::progress::Stream; use crate::setup::Outside; @@ -91,4 +91,8 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 2] = [claude::REGISTRATION, codex::REGISTRATION]; +pub const REGISTERED: [(Harness, &dyn Adapter); 3] = [ + claude::REGISTRATION, + codex::REGISTRATION, + grok::REGISTRATION, +]; diff --git a/src/harness/grok.rs b/src/harness/grok.rs new file mode 100644 index 000000000..442652d42 --- /dev/null +++ b/src/harness/grok.rs @@ -0,0 +1,115 @@ +//! Grok Build: unattended sessions and its Claude-shaped Messages stream. + +mod catalog; +mod stream; +pub use catalog::Catalog; + +use std::path::Path; + +use anyhow::Result; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings}; +use crate::progress::Stream; +use crate::setup::{Outside, questions::ask_setting}; + +pub struct Grok; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Grok, &Grok); + +impl Adapter for Grok { + fn name(&self) -> &'static str { + "grok" + } + fn product_name(&self) -> &'static str { + "Grok Build" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Slash + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[ + ("GROK_DISABLE_AUTOUPDATER", "1"), + ("GROK_FOLDER_TRUST", "0"), + ] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.grok + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.grok + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args: Vec = [ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + ] + .map(String::from) + .to_vec(); + if let Some(model) = &choice.model { + args.extend(["-m".to_string(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend(["--reasoning-effort".to_string(), effort.clone()]); + } + if let Some(id) = resume { + args.extend(["-r".to_string(), id.to_string()]); + } + args.push(self.skill_loading().prompt(prompt)); + Invocation { args, stdin: None } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + crate::progress::step("checking the Model and Effort against grok models"); + let settled = Catalog::read()?.settle(choice.model.as_deref(), choice.effort.as_deref())?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + let catalog = match outside.grok_catalog() { + Ok(catalog) => catalog, + Err(error) => { + outside.say(format!("{error:#}\nThe harness settings stay as they are; rerun `thirdshift setup` once grok models works.")); + return Ok(None); + } + }; + outside.say(format!( + "{}'s Models: {}", + self.product_name(), + catalog.listed().join(", ") + )); + let model = loop { + let model = ask_setting(outside, "Model", Harness::Grok, current.model.as_deref())?; + match catalog.settle(model.as_deref(), None) { + Ok(settled) => break settled.model, + Err(error) => outside.say(format!("{error:#}")), + } + }; + outside.say(format!( + "Efforts {} supports: {}", + model.as_deref().unwrap_or("Grok Build's default Model"), + catalog.efforts(model.as_deref()).join(", ") + )); + loop { + let effort = ask_setting(outside, "Effort", Harness::Grok, current.effort.as_deref())?; + match catalog.settle(model.as_deref(), effort.as_deref()) { + Ok(settled) => return Ok(Some(settled)), + Err(error) => outside.say(format!("{error:#}")), + } + } + } + fn stream(&self, _worktree: &Path) -> Box { + Box::new(stream::GrokProgress::default()) + } +} diff --git a/src/harness/grok/catalog.rs b/src/harness/grok/catalog.rs new file mode 100644 index 000000000..7712df051 --- /dev/null +++ b/src/harness/grok/catalog.rs @@ -0,0 +1,198 @@ +//! Grok's free model list and per-model Efforts from the cache it refreshes. + +use std::fs; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use serde_json::Value; + +use super::Grok; +use crate::harness::{Adapter, ModelAndEffort, said}; + +#[derive(Debug)] +struct Model { + name: String, + efforts: Vec, +} + +#[derive(Debug)] +pub struct Catalog { + models: Vec, + default: String, +} + +impl Catalog { + pub fn read() -> Result { + let output = Command::new(Grok.name()) + .arg("models") + .envs(Grok.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run grok models to read Grok Build's Models")?; + if !output.status.success() { + bail!( + "grok models failed, so Grok Build's Models can't be read: {}", + said(&output) + ); + } + // `grok models` prints names, but refreshes the effort metadata here. + let home = match std::env::var_os("GROK_HOME").filter(|home| !home.is_empty()) { + Some(home) => std::path::PathBuf::from(home), + None => crate::config::home()?.join(".grok"), + }; + let path = home.join("models_cache.json"); + let cache = fs::read_to_string(&path).with_context(|| { + format!( + "could not read Grok Build's Model and Effort catalog {} after grok models", + path.display() + ) + })?; + Self::parse(&String::from_utf8_lossy(&output.stdout), &cache) + } + + /// The text list and JSON cache recorded from Grok Build 1.0.46. + pub fn parse(list: &str, cache: &str) -> Result { + let cache: Value = + serde_json::from_str(cache).context("Grok Build's Model cache is not JSON")?; + let metadata = cache["models"] + .as_object() + .context("Grok Build's cache has no Models")?; + let default = list + .lines() + .find_map(|line| line.trim().strip_prefix("Default model: ")) + .context("grok models printed no default Model")? + .trim() + .to_string(); + let available = list + .split_once("Available models:") + .context("grok models printed no list of Models")? + .1; + let mut models = Vec::new(); + for name in available.lines().filter_map(|line| { + let line = line.trim(); + line.strip_prefix("* ") + .or_else(|| line.strip_prefix("- "))? + .split_whitespace() + .next() + }) { + let info = metadata.get(name).context(format!( + "Grok Build's cache has no metadata for Model {name}" + ))?; + let efforts = info["info"]["reasoning_efforts"] + .as_array() + .into_iter() + .flatten() + .filter_map(|effort| effort["value"].as_str()) + .map(String::from) + .collect(); + models.push(Model { + name: name.to_string(), + efforts, + }); + } + if models.is_empty() || !models.iter().any(|model| model.name == default) { + bail!("grok models printed no available default Model"); + } + Ok(Self { models, default }) + } + + pub fn listed(&self) -> Vec<&str> { + self.models + .iter() + .map(|model| model.name.as_str()) + .collect() + } + + fn model(&self, name: &str) -> Result<&Model> { + self.models + .iter() + .find(|model| model.name.eq_ignore_ascii_case(name)) + .with_context(|| { + format!( + "the Model {name} is not in Grok Build's catalog: choose one of {}", + self.listed().join(", ") + ) + }) + } + + pub fn efforts(&self, model: Option<&str>) -> Vec<&str> { + self.model(model.unwrap_or(&self.default)) + .map(|model| model.efforts.iter().map(String::as_str).collect()) + .unwrap_or_default() + } + + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { + let found = self.model(model.unwrap_or(&self.default))?; + let effort = match effort { + None => None, + Some(effort) => { + let Some(valid) = found + .efforts + .iter() + .find(|valid| valid.eq_ignore_ascii_case(effort)) + else { + bail!( + "the Effort {effort} is not one the Grok Build Model {} supports: choose one of {}", + found.name, + found.efforts.join(", ") + ); + }; + Some(valid.clone()) + } + }; + Ok(ModelAndEffort { + model: model.map(|_| found.name.clone()), + effort, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIST: &str = include_str!("../../../tests/fixtures/grok-models.txt"); + const CACHE: &str = include_str!("../../../tests/fixtures/grok-models.json"); + + #[test] + fn recorded_models_have_their_own_efforts_and_default_model_is_checked_when_no_model_is_set() { + let catalog = Catalog::parse(LIST, CACHE).unwrap(); + assert_eq!( + catalog.listed(), + ["grok-4.7", "grok-4.7-build-fast", "grok-4.6", "grok-4.5"] + ); + assert_eq!(catalog.efforts(Some("grok-4.5")), ["high", "medium", "low"]); + assert_eq!( + catalog.settle(None, Some("XHIGH")).unwrap(), + ModelAndEffort { + model: None, + effort: Some("xhigh".to_string()) + } + ); + let older_default = LIST.replace("Default model: grok-4.7", "Default model: grok-4.5"); + let older = Catalog::parse(&older_default, CACHE).unwrap(); + assert!( + older + .settle(None, Some("xhigh")) + .unwrap_err() + .to_string() + .contains("choose one of high, medium, low") + ); + } + + #[test] + fn an_unreadable_catalog_fails_instead_of_accepting_unchecked_settings() { + for (list, cache) in [ + ("", CACHE), + (LIST, "not json"), + (LIST, "{}"), + (LIST, "{\"models\": {}}"), + ( + "Default model: missing\nAvailable models:\n - missing", + CACHE, + ), + ] { + assert!(Catalog::parse(list, cache).is_err(), "{list:?} {cache:?}"); + } + } +} diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs new file mode 100644 index 000000000..59a000cc7 --- /dev/null +++ b/src/harness/grok/stream.rs @@ -0,0 +1,156 @@ +//! Grok's Messages stream shares Claude's framing, with different tool names. + +use serde_json::Value; + +use crate::harness::claude::stream::ClaudeProgress; +use crate::progress::Stream; + +#[derive(Default)] +pub struct GrokProgress { + claude: ClaudeProgress, + failed: bool, + error: Option, + session_id: Option, + usage: Option, +} + +impl Stream for GrokProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(mut event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if let Some(id) = event["session_id"].as_str() { + self.session_id.get_or_insert_with(|| id.to_string()); + } + match event["type"].as_str() { + Some("error") => { + self.failed = true; + self.error = event["message"].as_str().map(String::from); + } + Some("result") => { + self.failed = event["is_error"] == true || event["subtype"] != "success"; + let errors: Vec<_> = event["errors"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + if !errors.is_empty() { + self.error = Some(errors.join("\n")); + } else if self.failed + && let Some(text) = event["result"].as_str().filter(|text| !text.is_empty()) + { + self.error = Some(text.to_string()); + } + if let (Some(input), Some(output)) = ( + event["usage"]["input_tokens"].as_u64(), + event["usage"]["output_tokens"].as_u64(), + ) { + let cached = event["usage"]["cache_read_input_tokens"] + .as_u64() + .unwrap_or(0); + let created = event["usage"]["cache_creation_input_tokens"] + .as_u64() + .unwrap_or(0); + self.usage = Some(format!( + "{input} input tokens, {cached} cache read tokens, {created} cache creation tokens, {output} output tokens" + )); + } + } + _ => {} + } + if let Some(content) = event + .get_mut("message") + .and_then(|message| message.get_mut("content")) + .and_then(Value::as_array_mut) + { + for block in content + .iter_mut() + .filter(|block| block["type"] == "tool_use") + { + if matches!( + block["name"].as_str(), + Some("bash" | "run_terminal_command") + ) { + block["name"] = Value::String("Bash".to_string()); + } + if let Some(path) = block["input"]["path"].as_str().map(String::from) { + block["input"]["file_path"] = Value::String(path); + } + } + } + self.claude.condense(&event.to_string()) + } + fn summary(&self) -> Option { + self.claude.summary().or_else(|| self.usage.clone()) + } + fn final_message(&self) -> Option<&str> { + self.claude.final_message() + } + fn session_id(&self) -> Option<&str> { + self.claude.session_id().or(self.session_id.as_deref()) + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed { + Vec::new() + } else { + self.claude.killed_background_work() + } + } + fn failed(&self) -> bool { + self.failed + } + fn error(&self) -> Option<&str> { + self.error.as_deref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn messages_frames_reuse_claude_progress_and_keep_groks_id_final_text_and_spend() { + let mut stream = GrokProgress::default(); + assert_eq!(stream.condense(r#"{"type":"system","subtype":"init","session_id":"abc123","apiKeySource":"oauth","cwd":"/repo"}"#), ["session started"]); + assert_eq!(stream.condense(r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"read_file","input":{"path":"/repo/src/main.rs"}},{"type":"tool_use","name":"bash","input":{"command":"cargo test"}}]}}"#), ["read_file src/main.rs", "$ cargo test"]); + stream.condense(r#"{"type":"result","subtype":"success","is_error":false,"num_turns":7,"result":"Here's a summary...","total_cost_usd":0.0127,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":0,"cache_creation_input_tokens":0},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!(stream.final_message(), Some("Here's a summary...")); + assert_eq!(stream.summary().as_deref(), Some("7 turns, $0.01")); + assert!(!stream.failed()); + } + + #[test] + fn usage_without_a_complete_cost_and_a_result_without_init_are_still_read() { + let mut stream = GrokProgress::default(); + stream.condense(r#"{"type":"result","subtype":"success","result":"done","usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!( + stream.summary().as_deref(), + Some( + "812 input tokens, 45 cache read tokens, 12 cache creation tokens, 210 output tokens" + ) + ); + } + + #[test] + fn malformed_and_unrelated_lines_are_ignored_and_error_frames_report_the_failure() { + let mut stream = GrokProgress::default(); + for line in [ + "not json", + "null", + "[]", + "{}", + r#"{"type":"assistant","message":"bad"}"#, + ] { + assert!(stream.condense(line).is_empty()); + } + stream.condense(r#"{"type":"error","message":"backend unavailable"}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("backend unavailable")); + stream.condense(r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("quota exceeded")); + } +} diff --git a/src/main.rs b/src/main.rs index bc55f95bf..2f790f7cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,7 +73,7 @@ merge, --no-merge, --email, --no-email, base-fix, --no-base-fix, parallel (o --parallel ), harness , model and effort (or --harness, --model and --effort) go before or after the Issue URL, in any order. -harness, model and effort choose the Harness every session of the command runs on, and its +harness (claude, codex or grok), model and effort choose the Harness every session runs on, and its Model and Effort, for each Ticket's Run and a Base fix too. For each, the command wins, then the User config, then the default: claude, with its own Model and Effort. A Model and Effort in the User config come from the chosen Harness's own section, so harness claude over a codex @@ -89,7 +89,10 @@ default takes [harness.claude]: Before any work, the Harness's CLI must be on PATH. On claude, a named Model gets a minimal test call with its Effort, which must succeed. On codex, a named Model and Effort must be in codex debug models, matched regardless of case, a Model by slug or display name, and are -passed on as Codex names them. A failure stops the command naming what to fix, before the +passed on as Codex names them. Grok Build checks grok models and its refreshed effort cache, +then runs grok -p --always-approve --output-format streaming-messages-json with null stdin, +GROK_DISABLE_AUTOUPDATER=1 and GROK_FOLDER_TRUST=0. Resume uses -r . +A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json --dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, as in Built with claude · opus · diff --git a/src/progress.rs b/src/progress.rs index e0e9e74d6..3b583fe10 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1,5 +1,5 @@ //! Progress lines on stderr: thirdshift's own steps, and a session's -//! stream, Claude's stream-json or Codex's JSONL, condensed to one short +//! stream, parsed by its Harness adapter, condensed to one short //! line per notable event. Each line is stamped with the local time it was //! printed, so a stalled Run can be told from a busy one. Each is kept in the //! Command log too, if the command keeps one. diff --git a/src/session.rs b/src/session.rs index 174af6d2f..9f3f5f0a7 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,4 +1,4 @@ -//! Headless agent sessions, on Claude Code or Codex, and their logs. +//! Headless agent sessions on the chosen Harness, and their logs. use std::cell::RefCell; use std::fs::{self, File}; diff --git a/src/setup.rs b/src/setup.rs index d6804fcf3..dc6f4a6ef 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -367,6 +367,8 @@ pub(crate) trait Outside { fn test_call(&mut self, chosen: &ModelAndEffort) -> Result<()>; /// Read Codex's catalog of Models. fn codex_catalog(&mut self) -> Result; + /// Read Grok Build's catalog, refreshed by grok models. + fn grok_catalog(&mut self) -> Result; /// Read the Credentials, as strictly as a Run does: where a Resend API /// key is found, if anywhere. fn find_key(&mut self) -> Result>; @@ -459,6 +461,10 @@ impl Outside for OnMachine<'_> { Catalog::read() } + fn grok_catalog(&mut self) -> Result { + crate::harness::grok::Catalog::read() + } + fn find_key(&mut self) -> Result> { let credentials = Credentials::read()?; let found = credentials.lookup().map(|key| key.source); @@ -649,6 +655,8 @@ mod scripted { pub test_calls: VecDeque>, /// The catalog's JSON, or what reading it fails with. pub catalog: Result<&'static str, &'static str>, + /// Grok's text model list and refreshed effort cache, or a read failure. + pub grok_catalog: Result<(&'static str, &'static str), &'static str>, /// Where a key is found, or what reading the Credentials fails with. pub key: Result, &'static str>, pub github_email: Result, &'static str>, @@ -662,7 +670,7 @@ mod scripted { impl Scripted { /// A terminal answering `answers` in order, then closing stdin, - /// with both Harnesses installed, no key, no email to suggest and no + /// with every Harness installed, no key, no email to suggest and no /// User config. pub fn answering(answers: &[(&'static str, &'static str)]) -> Self { Scripted { @@ -671,6 +679,10 @@ mod scripted { installed: Harness::ALL.to_vec(), test_calls: VecDeque::new(), catalog: Ok(CATALOG), + grok_catalog: Ok(( + include_str!("../tests/fixtures/grok-models.txt"), + include_str!("../tests/fixtures/grok-models.json"), + )), key: Ok(None), github_email: Ok(None), git_email: Err("git config --global user.email: exit status 1"), @@ -793,6 +805,13 @@ mod scripted { } } + fn grok_catalog(&mut self) -> Result { + match self.grok_catalog { + Ok((list, cache)) => crate::harness::grok::Catalog::parse(list, cache), + Err(error) => bail!("{error}"), + } + } + fn find_key(&mut self) -> Result> { self.key.clone().map_err(|error| anyhow!("{error}")) } @@ -1107,7 +1126,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or grok [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1206,6 +1225,10 @@ effort = \"high\" [harness.codex] model = \"gpt-6.1-sol\" effort = \"max\" + +[harness.grok] +model = \"\" +effort = \"\" "; let mut outside = Scripted { user_config: Some(mine.to_string()), @@ -1235,7 +1258,7 @@ effort = \"max\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or grok [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1695,14 +1718,14 @@ always = false # quiet, please // The Harness, Model and Effort. #[test] - fn the_harness_question_lists_both_harnesses_and_defaults_to_claude() { + fn the_harness_question_lists_every_harness_and_defaults_to_claude() { let mut outside = Scripted::answering(&ENTER_THROUGHOUT); setup(&mut outside).unwrap(); assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex [claude]: " + "Harness for every Run's sessions, claude or codex or grok [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); @@ -1731,13 +1754,13 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude (not installed) or codex [codex]: " + "Harness for every Run's sessions, claude (not installed) or codex or grok (not installed) [codex]: " ); assert_eq!( outside.said()[..2], [ "claude is not installed: it isn't on PATH.", - "Choose claude or codex." + "Choose claude or codex or grok." ] ); assert_eq!( @@ -1747,7 +1770,7 @@ always = false # quiet, please } #[test] - fn with_neither_harness_installed_the_harness_settings_are_left_as_they_are() { + fn with_no_harness_installed_the_harness_settings_are_left_as_they_are() { let mut outside = Scripted { installed: Vec::new(), user_config: Some("[harness.claude]\nmodel = \"opus\"\n".to_string()), @@ -1760,8 +1783,8 @@ always = false # quiet, please outside.said(), [ "Harness for every Run's sessions: claude (not installed) or codex (not \ - installed). Sessions can run on neither here, so the harness settings stay as \ - they are; install claude or codex, then rerun `thirdshift setup`." + installed) or grok (not installed). Sessions can run on none here, so the harness \ + settings stay as they are; install claude or codex or grok, then rerun `thirdshift setup`." ] ); let config = table(&outside); @@ -1888,6 +1911,55 @@ always = false # quiet, please ); } + #[test] + fn setup_offers_grok_when_it_is_the_only_installed_harness_and_checks_its_model_and_effort() { + let mut outside = Scripted { + installed: vec![Harness::Grok], + ..Scripted::answering(&then_nothing_else(&[ + (HARNESS, ""), + ("Model for grok", "bogus"), + ("Model for grok", "GROK-4.5"), + ("Effort for grok", "xhigh"), + ("Effort for grok", "High"), + ])) + }; + + setup(&mut outside).unwrap(); + + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("grok")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("high")); + let said = outside.said().join("\n"); + assert!( + said.contains("choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5"), + "{said}" + ); + assert!(said.contains("choose one of high, medium, low"), "{said}"); + assert!(outside.test_calls().is_empty()); + } + + #[test] + fn when_groks_catalog_cannot_be_read_setup_preserves_the_harness_settings() { + let mut outside = Scripted { + grok_catalog: Err("catalog unavailable"), + user_config: Some("[harness]\ndefault = \"codex\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n".to_string()), + ..Scripted::answering(&then_nothing_else(&[(HARNESS, "grok")])) + }; + setup(&mut outside).unwrap(); + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("medium")); + assert!(outside.said().join("\n").contains("once grok models works")); + } + #[test] fn codex_is_offered_with_its_models_and_the_chosen_models_efforts() { let mut outside = Scripted::answering(&then_nothing_else(&[ diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 4b0d489c2..af44b79e1 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -165,7 +165,7 @@ pub fn ask( /// with those `current` sets for it as the defaults, checked as a Run /// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one -/// installed, so `claude` when both are. With no +/// installed, so `claude` when it is. With no /// Harness to choose, or Codex chosen and its catalog unreadable, the answer /// is `None`. fn ask_harness( @@ -194,7 +194,7 @@ fn ask_harness( let Some(default) = default else { let names = harness::names(); outside.say(format!( - "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ + "Harness for every Run's sessions: {harnesses}. Sessions can run on none here, so \ the harness settings stay as they are; install {names}, then rerun \ `thirdshift setup`." )); diff --git a/tests/fakes.rs b/tests/fakes.rs index 496dfef4d..33d132a8b 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -19,6 +19,8 @@ mod claude; mod codex; #[path = "fakes/gh.rs"] mod gh; +#[path = "fakes/grok.rs"] +mod grok; #[path = "fakes/json.rs"] mod json; @@ -111,6 +113,7 @@ fn main() { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), + "grok" => grok::main(args.collect()), "detached-command" => { let command = args.next().expect("no detached command"); let mut command = Command::new(command); diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 4afee8063..3fb7c7a9e 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -49,7 +49,7 @@ const CATALOG: &str = r#"{"models": [ ]}"#; /// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. -fn stdin_is_null() -> bool { +pub fn stdin_is_null() -> bool { let stdin = std::io::stdin() .as_fd() .try_clone_to_owned() @@ -64,7 +64,7 @@ fn stdin_is_null() -> bool { /// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts /// them until its script ends. A caught signal, unlike an ignored one, is /// back to its default in the script, which can trap it. -fn outlast_interrupts() { +pub fn outlast_interrupts() { // From the C library, which the standard library links: SIGINT and // SIGTERM are 2 and 15 on Linux and macOS alike. unsafe extern "C" { diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs new file mode 100644 index 000000000..94dd5b03d --- /dev/null +++ b/tests/fakes/grok.rs @@ -0,0 +1,131 @@ +//! Fake Grok Build: its text catalog, cache and Claude-shaped Messages stream. + +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::Command; + +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::codex::{outlast_interrupts, stdin_is_null}; +use crate::json::{Array, Bool, Null, object, string}; + +pub const MODELS: &str = include_str!("../fixtures/grok-models.txt"); +const CACHE: &str = include_str!("../fixtures/grok-models.json"); + +pub fn main(argv: Vec) { + let catalog = argv.first().is_some_and(|arg| arg == "models"); + if !catalog && !argv.iter().any(|arg| arg == "-p") { + crate::die("fake grok: expected models or -p", 2); + } + let record_path = crate::env_path("FAKE_GROK_RECORD"); + let lock = crate::lock_beside(&record_path); + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let cwd = std::env::current_dir().unwrap(); + let records = crate::append_record( + &record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ( + "prompt", + if catalog { + Null + } else { + argv.last().map(string).unwrap_or(Null) + }, + ), + ("cwd", string(cwd.to_str().unwrap())), + ("branch", string(branch)), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ("skill_files", skills_snapshot(".agents/skills")), + ("beside_skills", beside_skills_snapshot(".agents/skills")), + ( + "GROK_DISABLE_AUTOUPDATER", + string(std::env::var("GROK_DISABLE_AUTOUPDATER").unwrap_or_default()), + ), + ( + "GROK_FOLDER_TRUST", + string(std::env::var("GROK_FOLDER_TRUST").unwrap_or_default()), + ), + ]), + ); + let sessions: Vec<_> = records + .items() + .iter() + .filter(|call| call.at("prompt") != &Null) + .cloned() + .collect(); + drop(lock); + if catalog { + let home = crate::env_path("HOME"); + let cache = home.join(".grok/models_cache.json"); + fs::create_dir_all(cache.parent().unwrap()).unwrap(); + fs::write(cache, CACHE).unwrap(); + println!("{MODELS}"); + crate::exit(0); + } + outlast_interrupts(); + let session = sessions.len(); + let script = script_for(&sessions); + let id = argv + .iter() + .position(|arg| arg == "-r") + .and_then(|at| argv.get(at + 1)) + .cloned() + .unwrap_or_else(|| format!("fake-grok-{session}")); + println!( + "{}", + object([ + ("type", string("system")), + ("subtype", string("init")), + ("session_id", string(&id)), + ("cwd", string(cwd.to_str().unwrap())), + ]) + ); + std::io::stdout().flush().unwrap(); + let final_message = format!("{}.final-message.{session}", record_path.display()); + let after_result = format!("{}.after-result.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_CLAUDE_AFTER_RESULT", &after_result) + .status() + .unwrap(); + let code = exit_code(status); + println!( + "{}", + object([ + ("type", string("result")), + ( + "subtype", + string(if code == 0 { + "success" + } else { + "error_during_execution" + }) + ), + ("is_error", Bool(code != 0)), + ("session_id", string(id)), + ( + "result", + string(fs::read_to_string(&final_message).unwrap_or_default()) + ), + ("num_turns", crate::json::number(2)), + ("total_cost_usd", crate::json::number(0.0127)), + ( + "usage", + object([ + ("input_tokens", crate::json::number(812)), + ("output_tokens", crate::json::number(210)), + ("cache_read_input_tokens", crate::json::number(0)), + ]) + ), + ]) + ); + if let Ok(lines) = fs::read_to_string(Path::new(&after_result)) { + print!("{lines}"); + } + crate::exit(code); +} diff --git a/tests/fixtures/grok-models.json b/tests/fixtures/grok-models.json new file mode 100644 index 000000000..7787a2d15 --- /dev/null +++ b/tests/fixtures/grok-models.json @@ -0,0 +1,6 @@ +{"models": { + "grok-4.7": {"info": {"id": "grok-4.7", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.7-build-fast": {"info": {"id": "grok-4.7-build-fast", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.6": {"info": {"id": "grok-4.6", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.5": {"info": {"id": "grok-4.5", "reasoning_efforts": [{"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}} +}} diff --git a/tests/fixtures/grok-models.txt b/tests/fixtures/grok-models.txt new file mode 100644 index 000000000..2a84b9e2b --- /dev/null +++ b/tests/fixtures/grok-models.txt @@ -0,0 +1,9 @@ +You are logged in with grok.com. + +Default model: grok-4.7 + +Available models: + * grok-4.7 (default) + - grok-4.7-build-fast + - grok-4.6 + - grok-4.5 diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs new file mode 100644 index 000000000..05f38d606 --- /dev/null +++ b/tests/grok_sessions.rs @@ -0,0 +1,261 @@ +//! Commands on Grok Build through the scenario harness. + +mod support; + +use support::Scenario; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +#[test] +fn a_run_can_choose_grok_build() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert!(result.stderr.contains("sessions run on grok")); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_invalid_model_or_effort_fails_before_any_work_and_names_the_valid_choices() { + for (model, effort, error) in [ + ( + "grok-9.9-bogus", + "high", + "the Model grok-9.9-bogus is not in Grok Build's catalog: choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5", + ), + ( + "grok-4.5", + "xhigh", + "the Effort xhigh is not one the Grok Build Model grok-4.5 supports: choose one of high, medium, low", + ), + ( + "grok-4.7", + "max", + "the Effort max is not one the Grok Build Model grok-4.7 supports: choose one of xhigh, high, medium, low", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "grok", + "model", + model, + "effort", + effort, + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!(result.stderr.contains(error), "{}", result.stderr); + assert_eq!(scenario.entries("work"), ["widgets"]); + assert!(scenario.issue_labels(7).is_empty()); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0]["argv"], serde_json::json!(["models"])); + assert_eq!(calls[0]["stdin_null"], true); + assert_eq!(calls[0]["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(calls[0]["GROK_FOLDER_TRUST"], "0"); + } +} + +#[test] +fn grok_sessions_are_unattended_load_the_factory_skill_and_report_tools_and_usage() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#" +echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"run_terminal_command","input":{{"command":"cargo test"}}}}]}}}}' +{AGENT_OPENS_PR} +"#)); + let url = scenario.issue_url(7); + let result = scenario.run_with_env( + &[ + "harness", "grok", "model", "GROK-4.7", "effort", "High", &url, + ], + &[ + ("GROK_DISABLE_AUTOUPDATER", "0"), + ("GROK_FOLDER_TRUST", "1"), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + for line in [ + "implement: $ cargo test", + "2 turns, $0.01", + "sessions run on grok · grok-4.7 · high", + ] { + assert!( + result.stderr.contains(line), + "missing {line:?}: {}", + result.stderr + ); + } + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 2); + let session = &calls[1]; + let prompt = session["prompt"].as_str().unwrap(); + assert_eq!( + prompt.lines().next().unwrap(), + format!("/thirdshift-implement {url}") + ); + assert_eq!( + session["argv"], + serde_json::json!([ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "high", + prompt + ]) + ); + assert_eq!(session["git_status"], ""); + for call in calls { + assert_eq!(call["stdin_null"], true); + assert_eq!(call["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(call["GROK_FOLDER_TRUST"], "0"); + } + scenario.assert_every_grok_session_found_the_factory_skills(); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + assert!(log.contains("fake-grok-1")); + assert!(log.contains("\"type\": \"result\"")); + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with grok · grok-4.7 · high") + ); +} + +#[test] +fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { + for (event, exit, cause) in [ + ( + r#"{"type":"error","message":"backend unavailable"}"#, + "exit 1", + "grok exited 1: backend unavailable", + ), + ( + r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#, + "", + "grok's turn failed: quota exceeded", + ), + ] { + let scenario = Scenario::new(); + let script = if exit.is_empty() { + format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_CLAUDE_AFTER_RESULT\"\n") + } else { + format!("echo '{event}'\n{exit}") + }; + scenario.agent_does_for(7, &script); + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(cause), "{}", result.stderr); + } +} + +#[test] +fn a_resume_uses_the_streams_id_and_repeats_the_unattended_flags_and_environment() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!(r#" +{AGENT_OPENS_PR} +echo '{{"type":"system","subtype":"task_started","task_id":"task-1","description":"cargo test"}}' +echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status":"stopped"}}' > "$FAKE_CLAUDE_AFTER_RESULT" +"#)); + let result = scenario.run(&[ + "harness", + "grok", + "model", + "grok-4.7", + "effort", + "low", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 3); + let prompt = calls[2]["prompt"].as_str().unwrap(); + assert!(prompt.starts_with("Your background work (cargo test) was killed")); + assert_eq!( + calls[2]["argv"], + serde_json::json!([ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "low", + "-r", + "fake-grok-1", + prompt + ]) + ); + for call in &calls { + assert_eq!(call["stdin_null"], true); + assert_eq!(call["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(call["GROK_FOLDER_TRUST"], "0"); + } + scenario.assert_every_grok_session_found_the_factory_skills(); +} + +#[test] +fn the_user_config_can_choose_grok_and_its_own_model_and_effort() { + let scenario = Scenario::new(); + scenario.user_config_is("[harness]\ndefault = \"grok\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[&scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("sessions run on grok · grok-4.5 · medium") + ); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_architecture_reviews_final_message_reaches_the_command() { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does( + r#" +idea=$(gh issue create --title "Deepen sessions" --body "An idea" --label needs-triage) +printf 'Architecture review idea: %s\n' "$idea" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let result = scenario.run(&["architect", "harness", "grok"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + scenario + .issue_labels(8) + .contains(&"architect-idea".to_string()) + ); + let calls = scenario.grok_calls(); + assert_eq!( + calls[1]["prompt"].as_str().unwrap().lines().next().unwrap(), + "/thirdshift-improve-codebase-architecture" + ); + scenario.assert_every_grok_session_found_the_factory_skills(); +} diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 2c898e7dc..7c4cb606b 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -45,6 +45,11 @@ fn interrupting_a_codex_run_stops_a_command_in_its_own_session() { assert_detached_command_stopped("codex"); } +#[test] +fn interrupting_a_grok_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("grok"); +} + fn assert_detached_command_stopped(harness: &str) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); diff --git a/tests/setup.rs b/tests/setup.rs index f830ad39c..d89797123 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -52,7 +52,8 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[pickup]", "[harness]", "[harness.claude]", - "[harness.codex]" + "[harness.codex]", + "[harness.grok]" ], "{text}" ); @@ -70,7 +71,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "grok"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 2de6a51e2..db7f25d32 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -11,17 +11,20 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 5] = [ +const SOURCES: [&str; 8] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), include_str!("../fakes/codex.rs"), include_str!("../fakes/json.rs"), + include_str!("../fakes/grok.rs"), + include_str!("../fixtures/grok-models.json"), + include_str!("../fixtures/grok-models.txt"), ]; /// Put the fake `gh`, `claude` and `codex` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex", "detached-command"] { + for name in ["gh", "claude", "codex", "grok", "detached-command"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index f593337b0..df6a8fdd9 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -603,6 +603,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_CLAUDE_SCRIPT", self.path("claude-script.sh")) .env("FAKE_CLAUDE_RECORD", self.path("claude-calls.json")) .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) + .env("FAKE_GROK_RECORD", self.path("grok-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, @@ -815,6 +816,23 @@ test -f {root}/{COPY_REPLACED} } } + /// Every call the fake received on Grok, including its catalog check. + pub fn grok_calls(&self) -> Vec { + match fs::read_to_string(self.path("grok-calls.json")) { + Ok(text) => serde_json::from_str(&text).unwrap(), + Err(_) => Vec::new(), + } + } + + pub fn assert_every_grok_session_found_the_factory_skills(&self) { + assert_found_the_factory_skills( + self.grok_calls() + .into_iter() + .filter(|call| call["prompt"].is_string()) + .collect(), + ); + } + /// Assert every `claude` call found every Factory skill, by its /// `thirdshift-` name, in its worktree's `.claude/skills/`. pub fn assert_every_session_found_the_factory_skills(&self) { From ab32200b11a6c9714c1d06cd8f25819490370490 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:07:58 -0400 Subject: [PATCH 10/16] Address Grok Harness Standards and Spec findings --- README.md | 2 +- src/harness/codex.rs | 35 +++++++++++++++++++-------------- src/harness/grok.rs | 40 ++++++++++++++++++++++---------------- src/harness/grok/stream.rs | 14 +++++++++++-- src/main.rs | 2 +- src/setup/questions.rs | 20 ++++++++++++++++++- tests/fakes/grok.rs | 9 +++++++++ tests/grok_sessions.rs | 9 +++++++-- 8 files changed, 92 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 74181775f..91ebc0980 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,7 @@ For each of the three, the command wins, then the [User config](#user-config), t With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. -With `harness grok`, every session runs `grok -p --always-approve --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag is always explicit. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. +With `harness grok`, every session runs `grok -p --always-approve --sandbox off --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag and disabled sandbox are always explicit, overriding the CLI’s own config and `GROK_SANDBOX`. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. Grok reads both `AGENTS.md` and `CLAUDE.md` when both exist, along with `AGENT.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md` and project rules under `.grok`, `.claude` and `.cursor`. It also reads `~/.grok/rules`, `~/.claude/CLAUDE.md` and `~/.claude/rules`; thirdshift writes none of these user-level files. Grok Build uses `~/.grok` for sign-in and sessions (or `GROK_HOME` when set). diff --git a/src/harness/codex.rs b/src/harness/codex.rs index 734b15a3d..c65dc837c 100644 --- a/src/harness/codex.rs +++ b/src/harness/codex.rs @@ -11,7 +11,7 @@ use anyhow::Result; use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings}; use crate::progress::{self, Stream}; -use crate::setup::{Outside, questions::ask_setting}; +use crate::setup::{Outside, questions::ask_checked_setting}; pub struct Codex; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Codex, &Codex); @@ -135,13 +135,13 @@ fn ask_codex( Codex.product_name(), catalog.listed().join(", ") )); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Codex, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; + let model = ask_checked_setting( + outside, + "Model", + Harness::Codex, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; let efforts = catalog.efforts(model.as_deref()).join(", "); outside.say(match &model { Some(model) => format!("Efforts {model} supports: {efforts}"), @@ -150,13 +150,18 @@ fn ask_codex( Codex.product_name() ), }); - loop { - let effort = ask_setting(outside, "Effort", Harness::Codex, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(settled), - Err(error) => outside.say(format!("{error:#}")), - } - } + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Codex, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(ModelAndEffort { model, effort }) } /// `prompt` as Codex takes it: a first line that loads a Factory skill, diff --git a/src/harness/grok.rs b/src/harness/grok.rs index 442652d42..1c81b8ccd 100644 --- a/src/harness/grok.rs +++ b/src/harness/grok.rs @@ -11,7 +11,7 @@ use anyhow::Result; use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings}; use crate::progress::Stream; -use crate::setup::{Outside, questions::ask_setting}; +use crate::setup::{Outside, questions::ask_checked_setting}; pub struct Grok; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Grok, &Grok); @@ -46,8 +46,9 @@ impl Adapter for Grok { } fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { let mut args: Vec = [ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", ] @@ -62,7 +63,7 @@ impl Adapter for Grok { if let Some(id) = resume { args.extend(["-r".to_string(), id.to_string()]); } - args.push(self.skill_loading().prompt(prompt)); + args.extend(["-p".to_string(), self.skill_loading().prompt(prompt)]); Invocation { args, stdin: None } } fn check(&self, choice: &mut Choice) -> Result<()> { @@ -89,25 +90,30 @@ impl Adapter for Grok { self.product_name(), catalog.listed().join(", ") )); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Grok, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; + let model = ask_checked_setting( + outside, + "Model", + Harness::Grok, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; outside.say(format!( "Efforts {} supports: {}", model.as_deref().unwrap_or("Grok Build's default Model"), catalog.efforts(model.as_deref()).join(", ") )); - loop { - let effort = ask_setting(outside, "Effort", Harness::Grok, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(Some(settled)), - Err(error) => outside.say(format!("{error:#}")), - } - } + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Grok, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(Some(ModelAndEffort { model, effort })) } fn stream(&self, _worktree: &Path) -> Box { Box::new(stream::GrokProgress::default()) diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs index 59a000cc7..a172ed6e5 100644 --- a/src/harness/grok/stream.rs +++ b/src/harness/grok/stream.rs @@ -12,6 +12,7 @@ pub struct GrokProgress { error: Option, session_id: Option, usage: Option, + cost_known: bool, } impl Stream for GrokProgress { @@ -28,6 +29,11 @@ impl Stream for GrokProgress { self.error = event["message"].as_str().map(String::from); } Some("result") => { + // Messages streams use zero as a placeholder for unknown cost. + self.cost_known = event["total_cost_usd"] + .as_f64() + .is_some_and(|cost| cost > 0.0); + self.usage = None; self.failed = event["is_error"] == true || event["subtype"] != "success"; let errors: Vec<_> = event["errors"] .as_array() @@ -82,7 +88,11 @@ impl Stream for GrokProgress { self.claude.condense(&event.to_string()) } fn summary(&self) -> Option { - self.claude.summary().or_else(|| self.usage.clone()) + if self.cost_known { + self.claude.summary().or_else(|| self.usage.clone()) + } else { + self.usage.clone() + } } fn final_message(&self) -> Option<&str> { self.claude.final_message() @@ -124,7 +134,7 @@ mod tests { #[test] fn usage_without_a_complete_cost_and_a_result_without_init_are_still_read() { let mut stream = GrokProgress::default(); - stream.condense(r#"{"type":"result","subtype":"success","result":"done","usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); + stream.condense(r#"{"type":"result","subtype":"success","result":"done","num_turns":1,"total_cost_usd":0,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); assert_eq!(stream.session_id(), Some("abc123")); assert_eq!( stream.summary().as_deref(), diff --git a/src/main.rs b/src/main.rs index 2f790f7cf..e3809357f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -90,7 +90,7 @@ Before any work, the Harness's CLI must be on PATH. On claude, a named Model get test call with its Effort, which must succeed. On codex, a named Model and Effort must be in codex debug models, matched regardless of case, a Model by slug or display name, and are passed on as Codex names them. Grok Build checks grok models and its refreshed effort cache, -then runs grok -p --always-approve --output-format streaming-messages-json with null stdin, +then runs grok -p --always-approve --sandbox off --output-format streaming-messages-json with null stdin, GROK_DISABLE_AUTOUPDATER=1 and GROK_FOLDER_TRUST=0. Resume uses -r . A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json diff --git a/src/setup/questions.rs b/src/setup/questions.rs index af44b79e1..bfabbe708 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -166,7 +166,7 @@ pub fn ask( /// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one /// installed, so `claude` when it is. With no -/// Harness to choose, or Codex chosen and its catalog unreadable, the answer +/// Harness to choose, or a chosen Harness with an unreadable catalog, the answer /// is `None`. fn ask_harness( outside: &mut impl Outside, @@ -240,6 +240,24 @@ pub(crate) fn ask_setting( Ok(Some(answer).filter(|answer| !answer.is_empty() && answer != "-")) } +/// Ask one catalog-backed setting, retrying refusals with the catalog's +/// valid choices and returning the name the Harness takes. +pub(crate) fn ask_checked_setting( + outside: &mut (impl Outside + ?Sized), + setting: &str, + harness: Harness, + current: Option<&str>, + settle: impl Fn(Option<&str>) -> Result>, +) -> Result> { + loop { + let answer = ask_setting(outside, setting, harness, current)?; + match settle(answer.as_deref()) { + Ok(settled) => return Ok(settled), + Err(error) => outside.say(format!("{error:#}")), + } + } +} + /// Ask `question` until the answer is yes, no or nothing, which is `default`. fn yes_or_no(outside: &mut impl Outside, question: &str, default: bool) -> Result { let choices = if default { "[Y/n]" } else { "[y/N]" }; diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs index 94dd5b03d..a9cb20612 100644 --- a/tests/fakes/grok.rs +++ b/tests/fakes/grok.rs @@ -17,6 +17,15 @@ pub fn main(argv: Vec) { if !catalog && !argv.iter().any(|arg| arg == "-p") { crate::die("fake grok: expected models or -p", 2); } + if !catalog { + let prompt = argv + .iter() + .position(|arg| arg == "-p") + .and_then(|at| argv.get(at + 1)); + if prompt.is_none_or(|prompt| prompt.starts_with('-')) { + crate::die("a value is required for --single ", 2); + } + } let record_path = crate::env_path("FAKE_GROK_RECORD"); let lock = crate::lock_beside(&record_path); let (branch, _) = git_here(&["branch", "--show-current"]); diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs index 05f38d606..b8966dc85 100644 --- a/tests/grok_sessions.rs +++ b/tests/grok_sessions.rs @@ -87,6 +87,7 @@ echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"ru &[ ("GROK_DISABLE_AUTOUPDATER", "0"), ("GROK_FOLDER_TRUST", "1"), + ("GROK_SANDBOX", "workspace"), ], ); assert_eq!(result.code, Some(0), "{}", result.stderr); @@ -112,14 +113,16 @@ echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"ru assert_eq!( session["argv"], serde_json::json!([ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", "-m", "grok-4.7", "--reasoning-effort", "high", + "-p", prompt ]) ); @@ -198,8 +201,9 @@ echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status assert_eq!( calls[2]["argv"], serde_json::json!([ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", "-m", @@ -208,6 +212,7 @@ echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status "low", "-r", "fake-grok-1", + "-p", prompt ]) ); From 78403c0797d64ef3563eaccbc5f27a55187df007 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:23:58 -0400 Subject: [PATCH 11/16] Emit one terminal result in Grok failure scenarios --- README.md | 2 +- tests/fakes/grok.rs | 70 +++++++++++++++++++++--------------------- tests/grok_sessions.rs | 14 ++++++++- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e6c92a6cd..154f803dc 100644 --- a/README.md +++ b/README.md @@ -884,7 +884,7 @@ Editing a skill in `skills/` has no effect until you rebuild and reinstall ([ADR The [Prompts and skills page](https://thirdshift.app/prompts/) is generated from the prompts, the `claude` arguments and the skills, and each Session prompt is also generated as a Markdown file in [`prompts/`](prompts/), with a [`README.md`](prompts/README.md) that lists them with the `claude` command lines. `cargo test` fails until they are regenerated after a change to any of them. Regenerate the page and `prompts/` with `UPDATE_PROMPTS=1 cargo test prompts_page`, which also deletes any file in `prompts/` that it doesn't generate. -The integration tests swap in a fake `gh`, `claude`, `codex` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. +The integration tests swap in a fake `gh`, `claude`, `codex`, `agy` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. ### The Rust version diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs index a9cb20612..76b7b6d3a 100644 --- a/tests/fakes/grok.rs +++ b/tests/fakes/grok.rs @@ -2,7 +2,6 @@ use std::fs; use std::io::Write; -use std::path::Path; use std::process::Command; use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; @@ -94,47 +93,48 @@ pub fn main(argv: Vec) { ); std::io::stdout().flush().unwrap(); let final_message = format!("{}.final-message.{session}", record_path.display()); - let after_result = format!("{}.after-result.{session}", record_path.display()); + let result = format!("{}.result.{session}", record_path.display()); let status = Command::new("bash") .arg("-e") .arg(script) .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) - .env("FAKE_CLAUDE_AFTER_RESULT", &after_result) + .env("FAKE_GROK_RESULT", &result) .status() .unwrap(); let code = exit_code(status); - println!( - "{}", - object([ - ("type", string("result")), - ( - "subtype", - string(if code == 0 { - "success" - } else { - "error_during_execution" - }) - ), - ("is_error", Bool(code != 0)), - ("session_id", string(id)), - ( - "result", - string(fs::read_to_string(&final_message).unwrap_or_default()) - ), - ("num_turns", crate::json::number(2)), - ("total_cost_usd", crate::json::number(0.0127)), - ( - "usage", - object([ - ("input_tokens", crate::json::number(812)), - ("output_tokens", crate::json::number(210)), - ("cache_read_input_tokens", crate::json::number(0)), - ]) - ), - ]) - ); - if let Ok(lines) = fs::read_to_string(Path::new(&after_result)) { - print!("{lines}"); + if let Ok(result) = fs::read_to_string(result) { + print!("{result}"); + } else { + println!( + "{}", + object([ + ("type", string("result")), + ( + "subtype", + string(if code == 0 { + "success" + } else { + "error_during_execution" + }) + ), + ("is_error", Bool(code != 0)), + ("session_id", string(id)), + ( + "result", + string(fs::read_to_string(&final_message).unwrap_or_default()) + ), + ("num_turns", crate::json::number(2)), + ("total_cost_usd", crate::json::number(0.0127)), + ( + "usage", + object([ + ("input_tokens", crate::json::number(812)), + ("output_tokens", crate::json::number(210)), + ("cache_read_input_tokens", crate::json::number(0)), + ]) + ), + ]) + ); } crate::exit(code); } diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs index 9e9047dd1..71a149a93 100644 --- a/tests/grok_sessions.rs +++ b/tests/grok_sessions.rs @@ -165,7 +165,7 @@ fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { ] { let scenario = Scenario::new(); let script = if exit.is_empty() { - format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_CLAUDE_AFTER_RESULT\"\n") + format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_GROK_RESULT\"\n") } else { format!("echo '{event}'\n{exit}") }; @@ -173,6 +173,18 @@ fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); assert_eq!(result.code, Some(1), "{}", result.stderr); assert!(result.stderr.contains(cause), "{}", result.stderr); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + let results = log + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["type"] == "result") + .count(); + assert_eq!(results, 1, "{log}"); } } From f4a25dbb378f8c32d4f040d0598f163059e5e53d Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:39:04 -0400 Subject: [PATCH 12/16] Add the OpenCode Harness with standalone sessions and export recovery (#439) --- README.md | 16 +- src/args.rs | 4 +- src/config.rs | 12 +- src/harness.rs | 4 + src/harness/adapter.rs | 5 +- src/harness/opencode.rs | 160 +++++++++++++ src/harness/opencode/export.rs | 182 ++++++++++++++ src/harness/opencode/stream.rs | 183 ++++++++++++++ src/main.rs | 7 + src/setup.rs | 24 +- tests/fakes.rs | 3 + tests/fakes/muse.rs | 4 +- tests/fakes/opencode.rs | 246 +++++++++++++++++++ tests/opencode_sessions.rs | 426 +++++++++++++++++++++++++++++++++ tests/process_tree.rs | 5 + tests/setup.rs | 2 +- tests/support/fakes.rs | 13 +- tests/support/mod.rs | 1 + 18 files changed, 1277 insertions(+), 20 deletions(-) create mode 100644 src/harness/opencode.rs create mode 100644 src/harness/opencode/export.rs create mode 100644 src/harness/opencode/stream.rs create mode 100644 tests/fakes/opencode.rs create mode 100644 tests/opencode_sessions.rs diff --git a/README.md b/README.md index 2c308ccbe..f246324bd 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude, codex, agy or muse; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex, agy, muse or opencode; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -267,11 +267,15 @@ effort = "" # how hard that Model reasons; default blank, for agy's own [harness.muse] model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own effort = "" # how hard that Model reasons; default blank, for Muse Code's own + +[harness.opencode] +model = "" # the Model OpenCode's sessions run on, provider/model; default blank, for OpenCode's own +effort = "" # the Model's variant, passed as #effort; default blank, for OpenCode's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing only those installed on `PATH`. An unavailable Harness or unknown name is refused and asked again. The default is the configured Harness when installed, else the first installed, starting with `claude`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For agy, Setup lists `agy models` and its supported Efforts, checks each answer without a model turn, and writes the canonical names regardless of case. Muse proposes `muse-spark-1.3` and checks the answers against its cache, or with a minimal call; a refusal asks for the Model and Effort again. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing only those installed on `PATH`. An unavailable Harness or unknown name is refused and asked again. The default is the configured Harness when installed, else the first installed, starting with `claude`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For agy, Setup lists `agy models` and its supported Efforts, checks each answer without a model turn, and writes the canonical names regardless of case. Muse proposes `muse-spark-1.3` and checks the answers against its cache, or with a minimal call; a refusal asks for the Model and Effort again. OpenCode proposes no Model and checks the Model and Effort with a minimal standalone call, asking again on a refusal. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -302,7 +306,7 @@ A Run reads the file before any work. One that isn't valid TOML, or that has a k ### Harness, Model and Effort -The Harnesses are Claude Code (`claude`), Codex (`codex`), Antigravity CLI (`agy`) and Muse Code (`muse`). +The Harnesses are Claude Code (`claude`), Codex (`codex`), Antigravity CLI (`agy`) Muse Code (`muse`) and OpenCode (`opencode`). Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: @@ -330,6 +334,12 @@ Muse's named Model is checked against its cached catalog under `~/.local/share/m Muse's session id comes from `stream.id`, and a Resume uses `--session-id ` with every flag and environment override again. Progress lines name session starts, tools and errors. After exit, thirdshift reads the last reply and token totals from Muse's own `sessions/////session.jsonl`. Architecture reviews see that last reply alone, rather than Muse's terminal event's joined replies. A missing or unreadable log falls back to the stream text with no usage. `run.terminal.failed` or a non-zero exit fails the Run with Muse's error. An interrupt sends SIGTERM, then SIGKILL after ten seconds if needed. +With `harness opencode`, every session, Resume and preflight check runs `opencode run --standalone --format json --auto`, with the prompt on stdin and `OPENCODE_DISABLE_AUTOUPDATE=1`. Standalone starts a private server in the session's process tree and adds about 2–3 seconds per session; it keeps the work and its commands under thirdshift's interruption handling rather than in the shared service ([ADR-0013](docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md)). OpenCode reads project `AGENTS.md` and `~/.config/opencode/AGENTS.md`; it never reads `CLAUDE.md` or `~/.claude/CLAUDE.md` itself. When root `CLAUDE.md` exists and `AGENTS.md` does not, thirdshift links `AGENTS.md` to it and excludes that link from git. Factory skills are linked into `.agents/skills/`, and the prompt tells the model to load its skill with the skill tool. A missing successful `skill` load produces a warning without failing or retrying the session. + +OpenCode takes the Model id from your provider config, as `/`, including MiMo-V2.6-Pro when your provider routes it. Model and Effort are passed together as `-m /#`; an explicit Effort replaces a variant already in the Model id, and needs a Model set too. Before any work, thirdshift makes a minimal standalone test call with the same stdin, flags and environment. `provider.no-route` identifies a bad Model or Effort without a model turn; a successful check uses a model call because OpenCode has no free catalog without its service. Setup checks its answers the same way, proposing no default Model. + +The session id comes from `sessionID` in the stream, and a Resume uses `-s ` with every flag and environment override again. OpenCode creates a new session for an unknown id, so thirdshift only resumes an id it read from the stream. Progress comes from `step_start`, `tool_use`, `step_finish` and `error`. An aborted tool in a successful session gets the usual Resume. After exit, thirdshift reads `opencode session export --standalone `: an exported `failed` outcome fails the Run even with exit 0, the final message is the last assistant message's last text part, and usage is summed over assistant messages to exclude the automatic title call. If export fails or is unreadable, the last stream `text` supplies the final message and no usage is shown. A stream error or nonzero exit also fails the Run with OpenCode's error. An interrupt sends SIGTERM through the whole process tree, then SIGKILL after the grace period if needed. + The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. ### Email diff --git a/src/args.rs b/src/args.rs index 78d045a76..be50ed280 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex or agy or muse", + "harness must be followed by claude or codex or agy or muse or opencode", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex or agy or muse, not gemini", + "harness must be followed by claude or codex or agy or muse or opencode, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/config.rs b/src/config.rs index 3daf7c57b..c81db67b9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude, codex, agy or muse; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex, agy, muse or opencode; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -404,6 +404,10 @@ effort = "" # how hard that Model reasons; default blank, for agy's own [harness.muse] model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own effort = "" # how hard that Model reasons; default blank, for Muse Code's own + +[harness.opencode] +model = "" # the Model OpenCode's sessions run on, provider/model; default blank, for OpenCode's own +effort = "" # the Model's variant, passed as #effort; default blank, for OpenCode's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -539,7 +543,9 @@ mod tests { "harness.agy.model", "harness.agy.effort", "harness.muse.model", - "harness.muse.effort" + "harness.muse.effort", + "harness.opencode.model", + "harness.opencode.effort" ] ); let commented_out: Vec<&str> = DEFAULTS @@ -600,7 +606,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex or agy or muse, not \"gemini\"", + "harness.default must be claude or codex or agy or muse or opencode, not \"gemini\"", ), ( "[harness]\ndefault = true\n", diff --git a/src/harness.rs b/src/harness.rs index 7e7cc573a..6b2449b07 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -15,6 +15,7 @@ pub(crate) mod claude; pub(crate) mod codex; mod instructions; pub(crate) mod muse; +pub(crate) mod opencode; mod process_tree; pub use adapter::{Adapter, Invocation}; @@ -27,6 +28,7 @@ pub enum Harness { Codex, Agy, Muse, + OpenCode, } impl Harness { @@ -100,6 +102,8 @@ pub struct Settings { pub agy: ModelAndEffort, /// `[harness.muse]`. pub muse: ModelAndEffort, + /// `[harness.opencode]`. + pub opencode: ModelAndEffort, } impl Settings { diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 858af7083..8ffbed0a7 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -5,7 +5,7 @@ use std::process::Child; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, agy, claude, codex, muse}; +use super::{Choice, Harness, ModelAndEffort, Settings, agy, claude, codex, muse, opencode}; use crate::progress::Stream; use crate::setup::Outside; @@ -90,9 +90,10 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 4] = [ +pub const REGISTERED: [(Harness, &dyn Adapter); 5] = [ claude::REGISTRATION, codex::REGISTRATION, agy::REGISTRATION, muse::REGISTRATION, + opencode::REGISTRATION, ]; diff --git a/src/harness/opencode.rs b/src/harness/opencode.rs new file mode 100644 index 000000000..f6a73116e --- /dev/null +++ b/src/harness/opencode.rs @@ -0,0 +1,160 @@ +//! OpenCode: private standalone sessions, stdin prompts and an authoritative export. +mod export; +mod stream; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings, said}; +use crate::progress::Stream; +use crate::setup::{Outside, questions::ask_setting}; +use anyhow::{Context, Result, bail}; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +pub struct OpenCode; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::OpenCode, &OpenCode); + +impl Adapter for OpenCode { + fn name(&self) -> &'static str { + "opencode" + } + fn product_name(&self) -> &'static str { + "OpenCode" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Tool + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[("OPENCODE_DISABLE_AUTOUPDATE", "1")] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.opencode + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.opencode + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args = ["run", "--standalone", "--format", "json", "--auto"] + .map(String::from) + .to_vec(); + if let Some(model) = &choice.model { + let model = if let Some(effort) = &choice.effort { + format!("{}#{effort}", model.split('#').next().unwrap_or(model)) + } else { + model.clone() + }; + args.extend(["-m".into(), model]); + } + if let Some(id) = resume { + args.extend(["-s".into(), id.into()]); + } + Invocation { + args, + stdin: Some(self.skill_loading().prompt(prompt)), + } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + check_model_and_effort(&ModelAndEffort { + model: choice.model.clone(), + effort: choice.effort.clone(), + }) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + loop { + let chosen = ModelAndEffort { + model: ask_setting( + outside, + "Model", + Harness::OpenCode, + current.model.as_deref(), + )?, + effort: ask_setting( + outside, + "Effort", + Harness::OpenCode, + current.effort.as_deref(), + )?, + }; + match outside.opencode_check(&chosen) { + Ok(()) => return Ok(Some(chosen)), + Err(error) => outside.say(format!("{error:#}")), + } + } + } + fn stream(&self, _worktree: &Path, prompt: &str) -> Box { + Box::new(stream::OpenCodeProgress::for_prompt(prompt)) + } + fn link_instruction_fallback(&self, worktree: &Path) -> Result<()> { + super::instructions::link_fallback(worktree, "AGENTS.md", &[]) + } + fn read_after_exit(&self, worktree: &Path, stream: Box) -> Box { + export::read_after_exit(worktree, stream) + } +} + +/// Setup and Commands make the same minimal call: there is no free catalog +/// without the shared service, and an invalid route fails before a model turn. +pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result<()> { + if chosen.effort.is_some() && chosen.model.is_none() { + bail!("the OpenCode Effort needs a Model: set model / too"); + } + let choice = Choice { + harness: Harness::OpenCode, + model: chosen.model.clone(), + effort: chosen.effort.clone(), + chosen_by: super::ChosenBy::UserConfig, + }; + crate::progress::step("checking OpenCode's Model and Effort with a standalone test call"); + let invocation = OpenCode.session(&choice, None, "Reply with OK."); + let mut child = Command::new(OpenCode.name()) + .args(invocation.args) + .envs(OpenCode.environment().iter().copied()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("could not run opencode to check the Model and Effort")?; + let written = child + .stdin + .take() + .context("no stdin from opencode")? + .write_all(invocation.stdin.as_deref().unwrap_or("").as_bytes()); + if let Err(error) = written { + let _ = child.kill(); + let _ = child.wait(); + return Err(error).context("could not write the check prompt to opencode"); + } + let output = child + .wait_with_output() + .context("could not wait for opencode's test call")?; + let mut stream = OpenCode.stream(Path::new("."), "Reply with OK."); + for line in String::from_utf8_lossy(&output.stdout).lines() { + stream.condense(line); + } + let stream = OpenCode.read_after_exit(Path::new("."), stream); + if !output.status.success() || stream.failed() { + bail!( + "opencode refused the Model or Effort in its test call: {}", + stream + .error() + .map(String::from) + .unwrap_or_else(|| said(&output)) + ); + } + Ok(()) +} + +/// OpenCode errors appear as strings or typed objects in its event/export formats. +fn error_text(error: &serde_json::Value) -> Option<&str> { + error.as_str().or_else(|| error["message"].as_str()) +} diff --git a/src/harness/opencode/export.rs b/src/harness/opencode/export.rs new file mode 100644 index 000000000..c130b462c --- /dev/null +++ b/src/harness/opencode/export.rs @@ -0,0 +1,182 @@ +//! Read OpenCode 2's standalone export, whose assistant content and usage +//! survive dropped stream events. Session totals also count the title call. +use super::OpenCode; +use crate::harness::Adapter; +use crate::progress::Stream; +use anyhow::{Result, anyhow}; +use serde_json::Value; +use std::path::Path; +use std::process::{Command, Stdio}; + +pub struct SessionExport { + message: Option, + summary: Option, + failure: Option, +} + +impl SessionExport { + pub fn parse(text: &str) -> Result { + let export: Value = serde_json::from_str(text)?; + let outcome = export["info"]["outcome"] + .as_str() + .ok_or_else(|| anyhow!("no OpenCode outcome in session export"))?; + let messages = export["messages"] + .as_array() + .ok_or_else(|| anyhow!("no messages in OpenCode session export"))?; + let mut result = Self { + message: None, + summary: None, + failure: None, + }; + let (mut input, mut cached, mut output) = (0, 0, 0); + let mut usage = false; + for message in messages + .iter() + .filter(|message| message["type"] == "assistant") + { + result.message = message["content"] + .as_array() + .and_then(|parts| parts.iter().rev().find(|part| part["type"] == "text")) + .and_then(|part| part["text"].as_str()) + .map(String::from); + if let Some(tokens) = message["tokens"].as_object() { + usage = true; + input += tokens.get("input").and_then(Value::as_u64).unwrap_or(0); + cached += message["tokens"]["cache"]["read"].as_u64().unwrap_or(0); + output += tokens.get("output").and_then(Value::as_u64).unwrap_or(0); + } + if let Some(error) = super::error_text(&message["error"]) { + result.failure = Some(error.into()); + } + } + if usage { + result.summary = Some(format!( + "{input} input tokens ({cached} cached), {output} output tokens" + )); + } + if outcome == "failed" { + result.failure = Some( + super::error_text(&export["info"]["error"]) + .map(String::from) + .or(result.failure) + .unwrap_or_else(|| "OpenCode's exported outcome is failed".into()), + ); + } else { + result.failure = None; + } + Ok(result) + } +} + +pub fn read_after_exit(worktree: &Path, stream: Box) -> Box { + let export = stream.session_id().and_then(|id| { + let output = Command::new(OpenCode.name()) + .args(["session", "export", "--standalone", id]) + .envs(OpenCode.environment().iter().copied()) + .current_dir(worktree) + .stdin(Stdio::null()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + SessionExport::parse(&String::from_utf8_lossy(&output.stdout)).ok() + }); + Box::new(AfterExit { stream, export }) +} + +struct AfterExit { + stream: Box, + export: Option, +} +impl Stream for AfterExit { + fn condense(&mut self, raw: &str) -> Vec { + self.stream.condense(raw) + } + fn summary(&self) -> Option { + self.export + .as_ref() + .and_then(|export| export.summary.clone()) + } + fn final_message(&self) -> Option<&str> { + match &self.export { + Some(export) => export.message.as_deref(), + None => self.stream.final_message(), + } + } + fn session_id(&self) -> Option<&str> { + self.stream.session_id() + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed() { + Vec::new() + } else { + self.stream.killed_background_work() + } + } + fn failed(&self) -> bool { + self.stream.failed() + || self + .export + .as_ref() + .is_some_and(|export| export.failure.is_some()) + } + fn error(&self) -> Option<&str> { + self.stream.error().or_else(|| { + self.export + .as_ref() + .and_then(|export| export.failure.as_deref()) + }) + } + fn warnings(&self) -> Vec { + self.stream.warnings() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_last_assistant_text_and_summed_usage_exclude_the_title_call_and_user() { + // OpenCode 2.0.24's exported message format, with irrelevant fields removed. + let export = SessionExport::parse(r#"{ + "info":{"outcome":"succeeded","tokens":{"input":999999},"cost":9}, + "messages":[ + {"type":"user","text":"prompt","tokens":{"input":999999}}, + {"type":"assistant","tokens":{"input":31,"output":37,"reasoning":158,"cache":{"read":7840,"write":0}},"content":[{"type":"text","text":"earlier reply"}]}, + {"type":"assistant","tokens":{"input":39,"output":11,"reasoning":42,"cache":{"read":496,"write":0}},"content":[{"type":"text","text":"first part"},{"type":"reasoning","text":"private reasoning"},{"type":"text","text":"Architecture review idea: https://github.com/acme/widgets/issues/8"}]}, + {"type":"idle","outcome":"succeeded"} + ] + }"#).unwrap(); + assert_eq!( + export.message.as_deref(), + Some("Architecture review idea: https://github.com/acme/widgets/issues/8") + ); + assert_eq!( + export.summary.as_deref(), + Some("70 input tokens (8336 cached), 48 output tokens") + ); + assert!(export.failure.is_none()); + } + + #[test] + fn a_failed_export_reports_the_assistants_error_when_the_session_has_none() { + let export = SessionExport::parse(r#"{"info":{"outcome":"failed"},"messages":[{"type":"assistant","content":[],"error":{"type":"provider.no-route","message":"Variant unavailable"}}]}"#).unwrap(); + assert_eq!(export.failure.as_deref(), Some("Variant unavailable")); + } + + #[test] + fn incomplete_or_malformed_exports_are_unreadable() { + for text in ["{", "{}", r#"{"info":{"outcome":"succeeded"}}"#] { + assert!(SessionExport::parse(text).is_err()); + } + } + + #[test] + fn an_assistant_without_text_does_not_reuse_an_earlier_reply() { + let export = SessionExport::parse(r#"{"info":{"outcome":"succeeded"},"messages":[{"type":"assistant","content":[{"type":"text","text":"earlier reply"}]},{"type":"assistant","content":[{"type":"tool","name":"bash"}]}]}"#).unwrap(); + assert!(export.message.is_none()); + assert!(export.summary.is_none()); + } +} diff --git a/src/harness/opencode/stream.rs b/src/harness/opencode/stream.rs new file mode 100644 index 000000000..a8048fffb --- /dev/null +++ b/src/harness/opencode/stream.rs @@ -0,0 +1,183 @@ +//! OpenCode's JSONL progress; final usage is deliberately left to session export. +use crate::progress::{Stream, bash, shorten}; +use serde_json::Value; + +#[derive(Default)] +pub struct OpenCodeProgress { + session_id: Option, + started: bool, + message: Option, + failure: Option, + expected_skill: Option, + skill_loaded: bool, + killed: Vec, +} + +impl OpenCodeProgress { + pub fn for_prompt(prompt: &str) -> Self { + Self { + expected_skill: prompt + .strip_prefix('/') + .and_then(|rest| rest.split_whitespace().next()) + .filter(|name| name.starts_with("thirdshift-")) + .map(String::from), + ..Self::default() + } + } +} + +impl Stream for OpenCodeProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if let Some(id) = event["sessionID"].as_str().filter(|id| !id.is_empty()) { + self.session_id = Some(id.into()); + } + let part = &event["part"]; + match event["type"].as_str() { + Some("step_start") if !self.started => { + self.started = true; + vec!["session started".into()] + } + Some("text") => { + if let Some(text) = part["text"].as_str() { + self.message = Some(text.into()); + } + Vec::new() + } + Some("tool_use") => { + let tool = part["tool"].as_str().unwrap_or(""); + let state = &part["state"]; + if matches!(tool, "bash" | "task") + && state["status"] == "error" + && state["error"]["type"] == "aborted" + { + self.killed.push( + state["input"]["command"] + .as_str() + .unwrap_or(tool) + .to_string(), + ); + } + if tool == "skill" { + let skill = state["input"]["id"].as_str().unwrap_or(""); + if state["status"] == "completed" + && self.expected_skill.as_deref() == Some(skill) + { + self.skill_loaded = true; + } + vec![format!("skill {skill}")] + } else if tool == "bash" { + state["input"]["command"] + .as_str() + .map(|command| vec![bash(command)]) + .unwrap_or_default() + } else if tool.is_empty() { + Vec::new() + } else { + vec![shorten(tool)] + } + } + Some("step_finish") => vec!["step finished".into()], + Some("error") => { + let error = &event["error"]; + let message = super::error_text(error).unwrap_or("OpenCode's turn failed"); + self.failure = Some(if let Some(kind) = error["type"].as_str() { + format!("{kind}: {message}") + } else { + message.into() + }); + vec![format!( + "error: {}", + shorten(self.failure.as_deref().unwrap()) + )] + } + _ => Vec::new(), + } + } + fn summary(&self) -> Option { + None + } + fn final_message(&self) -> Option<&str> { + self.message.as_deref() + } + fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed() { + Vec::new() + } else { + self.killed.iter().map(String::as_str).collect() + } + } + fn failed(&self) -> bool { + self.failure.is_some() + } + fn error(&self) -> Option<&str> { + self.failure.as_deref() + } + fn warnings(&self) -> Vec { + self.expected_skill + .as_ref() + .filter(|_| !self.skill_loaded) + .map(|skill| { + vec![format!( + "warning: the session never loaded {skill} with its skill tool" + )] + }) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_dropped_last_step_keeps_the_last_text_without_trusting_stream_usage() { + let mut stream = OpenCodeProgress::for_prompt("/thirdshift-implement issue-url"); + for line in [ + r#"{"type":"step_start","sessionID":"s1","part":{}}"#, + r#"{"type":"tool_use","sessionID":"s1","part":{"tool":"skill","state":{"status":"completed","input":{"id":"thirdshift-implement"}}}}"#, + r#"{"type":"text","sessionID":"s1","part":{"text":"earlier reply"}}"#, + r#"{"type":"step_finish","sessionID":"s1","part":{"tokens":{"input":1200,"output":300}}}"#, + r#"{"type":"text","sessionID":"s1","part":{"text":"last reply"}}"#, + "not JSON", + ] { + stream.condense(line); + } + assert_eq!(stream.final_message(), Some("last reply")); + assert_eq!(stream.session_id(), Some("s1")); + assert!(stream.warnings().is_empty()); + assert!(stream.summary().is_none()); + } + + #[test] + fn a_wrong_or_failed_skill_load_still_warns() { + let mut stream = OpenCodeProgress::for_prompt("/thirdshift-implement issue-url"); + for line in [ + r#"{"type":"tool_use","part":{"tool":"skill","state":{"status":"completed","input":{"id":"thirdshift-tdd"}}}}"#, + r#"{"type":"tool_use","part":{"tool":"skill","state":{"status":"error","input":{"id":"thirdshift-implement"}}}}"#, + ] { + stream.condense(line); + } + assert_eq!( + stream.warnings(), + ["warning: the session never loaded thirdshift-implement with its skill tool"] + ); + assert!(stream.session_id().is_none()); + } + + #[test] + fn a_recorded_no_route_error_fails_and_names_the_model_or_effort_problem() { + let mut stream = OpenCodeProgress::default(); + stream.condense(r#"{"type":"error","sessionID":"s1","error":{"type":"provider.no-route","message":"Model unavailable: thirdshift-invalid-provider/invalid"}}"#); + assert!(stream.failed()); + assert_eq!( + stream.error(), + Some("provider.no-route: Model unavailable: thirdshift-invalid-provider/invalid") + ); + } +} diff --git a/src/main.rs b/src/main.rs index aa93ecc43..d32984753 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,13 @@ session and Resume uses -p --dangerously-skip-permissions --output-format stream null stdin and AGY_CLI_DISABLE_AUTO_UPDATE=true; a Resume uses --conversation . agy reads AGENTS.md and GEMINI.md, falling back through an excluded GEMINI.md link to root CLAUDE.md when neither exists. It does not read ~/.claude/. +On opencode (OpenCode), the Model id comes from your providers as provider/model; +Effort is its #variant suffix and needs a Model. A minimal standalone call checks both. +Every session, check and Resume uses run --standalone --format json --auto with the prompt +on stdin and OPENCODE_DISABLE_AUTOUPDATE=1; a Resume adds -s . +Skills load through the skill tool. OpenCode reads AGENTS.md, never CLAUDE.md or ~/.claude/; +thirdshift supplies an excluded AGENTS.md link to root CLAUDE.md when needed. Session export +supplies the last assistant text, usage and outcome even when the last stream event is lost. --email sends one Run notification when the Run ends, whatever the outcome: ready for review, merged, failed or interrupted. --email
sends it to
; a word diff --git a/src/setup.rs b/src/setup.rs index 26276c177..35275154d 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -371,6 +371,8 @@ pub(crate) trait Outside { fn agy_catalog(&mut self) -> Result; /// Check Muse's settings against its cache, or with its minimal test call. fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result; + /// Check OpenCode with its minimal standalone test call. + fn opencode_check(&mut self, chosen: &ModelAndEffort) -> Result<()>; /// Read the Credentials, as strictly as a Run does: where a Resend API /// key is found, if anywhere. fn find_key(&mut self) -> Result>; @@ -471,6 +473,10 @@ impl Outside for OnMachine<'_> { crate::harness::muse::check_model_and_effort(chosen) } + fn opencode_check(&mut self, chosen: &ModelAndEffort) -> Result<()> { + crate::harness::opencode::check_model_and_effort(chosen) + } + fn find_key(&mut self) -> Result> { let credentials = Credentials::read()?; let found = credentials.lookup().map(|key| key.source); @@ -827,6 +833,10 @@ mod scripted { .map_err(|error| anyhow!("{error}")) } + fn opencode_check(&mut self, _chosen: &ModelAndEffort) -> Result<()> { + Ok(()) + } + fn find_key(&mut self) -> Result> { self.key.clone().map_err(|error| anyhow!("{error}")) } @@ -1141,7 +1151,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: ", + "Harness for every Run's sessions, claude or codex or agy or muse or opencode [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1248,6 +1258,10 @@ effort = \"\" [harness.muse] model = \"\" effort = \"\" + +[harness.opencode] +model = \"\" +effort = \"\" "; let mut outside = Scripted { user_config: Some(mine.to_string()), @@ -1277,7 +1291,7 @@ effort = \"\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: ", + "Harness for every Run's sessions, claude or codex or agy or muse or opencode [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1779,11 +1793,11 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: " + "Harness for every Run's sessions, claude or codex or agy or muse or opencode [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex", "agy", "muse"] { + for harness in ["claude", "codex", "agy", "muse", "opencode"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some("")); } @@ -1837,7 +1851,7 @@ always = false # quiet, please outside.said(), [ "No Harness is installed here, so the harness settings stay as they are; \ - install claude or codex or agy or muse, then rerun `thirdshift setup`." + install claude or codex or agy or muse or opencode, then rerun `thirdshift setup`." ] ); let config = table(&outside); diff --git a/tests/fakes.rs b/tests/fakes.rs index 4de34f85d..cdc6f4d31 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -25,6 +25,8 @@ mod gh; mod json; #[path = "fakes/muse.rs"] mod muse; +#[path = "fakes/opencode.rs"] +mod opencode; use std::fs::{self, File}; use std::io::Write; @@ -117,6 +119,7 @@ fn main() { "codex" => codex::main(args.collect()), "agy" => agy::main(args.collect()), "muse" => muse::main(args.collect()), + "opencode" => opencode::main(args.collect()), "detached-command" => { let command = args.next().expect("no detached command"); let mut command = Command::new(command); diff --git a/tests/fakes/muse.rs b/tests/fakes/muse.rs index 3c3654d34..ed2d0d806 100644 --- a/tests/fakes/muse.rs +++ b/tests/fakes/muse.rs @@ -28,7 +28,7 @@ pub(crate) fn stdin_is_null() -> bool { /// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts /// them until its script ends. A caught signal, unlike an ignored one, is /// back to its default in the script, which can trap it. -fn outlast_interrupts() { +pub(crate) fn outlast_interrupts() { // From the C library, which the standard library links: SIGINT and // SIGTERM are 2 and 15 on Linux and macOS alike. unsafe extern "C" { @@ -42,7 +42,7 @@ fn outlast_interrupts() { } /// Print one line of the stream. -fn emit(event: Json) { +pub(crate) fn emit(event: Json) { println!("{event}"); std::io::stdout().flush().unwrap(); } diff --git a/tests/fakes/opencode.rs b/tests/fakes/opencode.rs new file mode 100644 index 000000000..c61e95805 --- /dev/null +++ b/tests/fakes/opencode.rs @@ -0,0 +1,246 @@ +//! OpenCode's JSONL stream and standalone session export, with stdin prompts. +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::json::{Array, Json, Null, number, object, string}; +use crate::muse::{emit, outlast_interrupts}; +use std::fs; +use std::io::Read; +use std::process::Command; + +pub fn main(argv: Vec) { + let record_path = crate::env_path("FAKE_OPENCODE_RECORD"); + let export_path = |id: &str| record_path.with_extension(format!("{id}.export.json")); + if argv.starts_with(&["session".into(), "export".into()]) { + crate::append_record( + &record_path.with_extension("exports.json"), + object([ + ("argv", Array(argv.iter().map(string).collect())), + ( + "no_auto_update", + string(std::env::var("OPENCODE_DISABLE_AUTOUPDATE").unwrap_or_default()), + ), + ]), + ); + if std::env::var("FAKE_OPENCODE_NO_EXPORT").is_ok() { + crate::exit(1); + } + if std::env::var("FAKE_OPENCODE_CORRUPT_EXPORT").is_ok() { + println!("truncated {{"); + crate::exit(0); + } + println!( + "{}", + fs::read_to_string(export_path(argv.last().unwrap())).unwrap() + ); + crate::exit(0); + } + assert_eq!(argv[0], "run"); + assert!(argv.iter().any(|arg| arg == "--standalone")); + let mut prompt = String::new(); + std::io::stdin().read_to_string(&mut prompt).unwrap(); + let record = object([ + ("argv", Array(argv.iter().map(string).collect())), + ("prompt", string(&prompt)), + ( + "no_auto_update", + string(std::env::var("OPENCODE_DISABLE_AUTOUPDATE").unwrap_or_default()), + ), + ]); + if prompt == "Reply with OK." { + crate::append_record(&record_path.with_extension("checks.json"), record); + if argv + .iter() + .any(|arg| arg.contains("bad-model") || arg.contains("#bogus")) + { + emit(object([ + ("type", string("error")), + ( + "error", + object([ + ("type", string("provider.no-route")), + ("message", string("Model or variant unavailable")), + ]), + ), + ])); + crate::exit(1); + } + event("check-session", "text", object([("text", string("OK"))])); + fs::write( + export_path("check-session"), + object([ + ( + "info", + object([( + "outcome", + string( + if std::env::var("FAKE_OPENCODE_CHECK_EXPORT_FAILED").is_ok() { + "failed" + } else { + "succeeded" + }, + ), + )]), + ), + ("messages", Array(vec![])), + ]) + .dump(), + ) + .unwrap(); + crate::exit(0); + } + let (status, _) = git_here(&["status", "--porcelain"]); + let cwd = std::env::current_dir().unwrap(); + let mut record = record; + record.set("cwd", string(cwd.to_str().unwrap())); + record.set("git_status", string(status)); + record.set("skill_files", skills_snapshot(".agents/skills")); + record.set("beside_skills", beside_skills_snapshot(".agents/skills")); + record.set( + "agents_link", + fs::read_link("AGENTS.md") + .ok() + .map(|path| string(path.to_str().unwrap())) + .unwrap_or(Null), + ); + record.set( + "agents_text", + fs::read_to_string("AGENTS.md") + .ok() + .map(string) + .unwrap_or(Null), + ); + let lock = crate::lock_beside(&record_path); + let records = crate::append_record(&record_path, record); + let records = records.items(); + let script = script_for(records); + drop(lock); + let session = records.len(); + let id = argv + .windows(2) + .find(|pair| pair[0] == "-s") + .map(|pair| pair[1].clone()) + .unwrap_or_else(|| format!("fake-opencode-{session}")); + outlast_interrupts(); + event(&id, "step_start", object([])); + if let Some(rest) = prompt.strip_prefix("Load ") { + let skill = rest.split_whitespace().next().unwrap(); + if std::env::var("FAKE_OPENCODE_SKIP_SKILL").is_err() { + event( + &id, + "tool_use", + object([ + ("tool", string("skill")), + ( + "state", + object([ + ("status", string("completed")), + ("input", object([("id", string(skill))])), + ]), + ), + ]), + ); + } + } + let final_message = format!("{}.final-message.{session}", record_path.display()); + let error = format!("{}.error.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_OPENCODE_ERROR", &error) + .status() + .unwrap(); + let code = exit_code(status); + let message = fs::read_to_string(&final_message).unwrap_or_else(|_| "done".into()); + event(&id, "text", object([("text", string("earlier reply"))])); + event(&id, "text", object([("text", string(&message))])); + let failed = code != 0 || std::env::var("FAKE_OPENCODE_EXPORT_FAILED").is_ok(); + let error = fs::read_to_string(error).unwrap_or_else(|_| "OpenCode's turn failed".into()); + let assistant = |text: &str| { + object([ + ("type", string("assistant")), + ( + "tokens", + object([ + ("input", number(1200)), + ("output", number(300)), + ("reasoning", number(100)), + ( + "cache", + object([("read", number(200)), ("write", number(0))]), + ), + ]), + ), + ("cost", number(0)), + ( + "content", + Array(vec![object([ + ("type", string("text")), + ("text", string(text)), + ])]), + ), + ]) + }; + fs::write( + export_path(&id), + object([ + ( + "info", + object([ + ( + "outcome", + string(if failed { "failed" } else { "succeeded" }), + ), + ("tokens", object([("input", number(999999))])), + ("error", object([("message", string(&error))])), + ]), + ), + ( + "messages", + Array(vec![ + object([("type", string("user")), ("text", string("prompt"))]), + assistant("earlier reply"), + assistant(&message), + ]), + ), + ]) + .dump(), + ) + .unwrap(); + // Alternate complete and dropped closing events, as real OpenCode does. + if session % 2 == 0 { + event( + &id, + "step_finish", + object([ + ("reason", string("stop")), + ( + "tokens", + object([("input", number(1200)), ("output", number(300))]), + ), + ]), + ); + } + if code != 0 { + emit(object([ + ("type", string("error")), + ("sessionID", string(&id)), + ("error", object([("message", string(error))])), + ])); + } + crate::exit(code); +} + +fn event(id: &str, kind: &str, part: Json) { + emit(object([ + ("type", string(kind)), + ( + "sessionID", + string(if std::env::var("FAKE_OPENCODE_NO_SESSION_ID").is_ok() { + "" + } else { + id + }), + ), + ("part", part), + ])); +} diff --git a/tests/opencode_sessions.rs b/tests/opencode_sessions.rs new file mode 100644 index 000000000..ec6697d1a --- /dev/null +++ b/tests/opencode_sessions.rs @@ -0,0 +1,426 @@ +//! OpenCode Commands through the scenario harness. +mod support; +use serde_json::{Value, json}; +use support::{FACTORY_SKILLS, Scenario}; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m 'Add feature' +gh pr create --base main --head issue-7 --title 'Add feature' --body 'Closes #7' +"#; + +fn records(scenario: &Scenario, extension: &str) -> Vec { + std::fs::read_to_string(scenario.path(&format!("opencode-calls.{extension}"))) + .map(|text| serde_json::from_str(&text).unwrap()) + .unwrap_or_default() +} + +#[test] +fn opencode_runs_standalone_with_stdin_skills_and_usage_despite_a_dropped_last_step() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "opencode", + "model", + "provider/MiMo-V2.6-Pro", + "effort", + "high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = records(&scenario, "json"); + assert_eq!(calls.len(), 1); + assert_eq!( + calls[0]["argv"], + json!([ + "run", + "--standalone", + "--format", + "json", + "--auto", + "-m", + "provider/MiMo-V2.6-Pro#high" + ]) + ); + assert_eq!(calls[0]["no_auto_update"], "1"); + assert_eq!(calls[0]["git_status"], ""); + assert!( + calls[0]["prompt"] + .as_str() + .unwrap() + .starts_with("Load thirdshift-implement with your skill tool. ") + ); + for skill in FACTORY_SKILLS { + assert!( + calls[0]["skill_files"][format!("{skill}/SKILL.md")].is_string(), + "{skill}" + ); + } + assert!( + result + .stderr + .contains("2400 input tokens (400 cached), 600 output tokens"), + "{}", + result.stderr + ); + assert!(!result.stderr.contains("warning:"), "{}", result.stderr); + let checks = records(&scenario, "checks.json"); + assert_eq!(checks[0]["argv"], calls[0]["argv"]); + assert_eq!(checks[0]["prompt"], "Reply with OK."); + assert_eq!(checks[0]["no_auto_update"], "1"); + let exports = records(&scenario, "exports.json"); + let exports = exports + .iter() + .filter(|call| call["argv"][3] != "check-session") + .collect::>(); + assert_eq!( + exports[0]["argv"], + json!(["session", "export", "--standalone", "fake-opencode-1"]) + ); + assert_eq!(exports[0]["no_auto_update"], "1"); + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with opencode · provider/MiMo-V2.6-Pro · high") + ); +} + +#[test] +fn opencode_resumes_only_the_stream_session_id_with_the_same_standalone_flags() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} +echo '{{"type":"tool_use","sessionID":"fake-opencode-1","part":{{"tool":"bash","state":{{"status":"error","input":{{"command":"cargo test"}},"error":{{"type":"aborted","message":"Tool execution interrupted"}}}}}}}}' +"#)); + let result = scenario.run(&[ + "harness", + "opencode", + "model", + "provider/model", + "effort", + "high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = records(&scenario, "json"); + assert_eq!(calls.len(), 2, "{}", result.stderr); + assert_eq!( + calls[1]["argv"], + json!([ + "run", + "--standalone", + "--format", + "json", + "--auto", + "-m", + "provider/model#high", + "-s", + "fake-opencode-1" + ]) + ); + assert_eq!(calls[1]["no_auto_update"], "1"); + assert!(calls[1]["prompt"].as_str().unwrap().contains("cargo test")); +} + +#[test] +fn user_config_selects_opencode_and_command_effort_overrides_its_variant() { + let scenario = Scenario::new(); + scenario.user_config_is("[harness]\ndefault = \"opencode\"\n[harness.opencode]\nmodel = \"provider/model#low\"\neffort = \"low\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&["effort", "high", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert_eq!( + records(&scenario, "json")[0]["argv"][6], + "provider/model#high" + ); +} + +#[test] +fn bad_models_and_efforts_fail_before_the_claim_worktree_and_command_log() { + for (model, effort) in [("provider/bad-model", "high"), ("provider/model", "bogus")] { + let scenario = Scenario::new(); + let result = scenario.run(&[ + "harness", + "opencode", + "model", + model, + "effort", + effort, + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result.stderr.contains("Model or Effort"), + "{}", + result.stderr + ); + assert!( + result.stderr.contains("provider.no-route"), + "{}", + result.stderr + ); + assert!(records(&scenario, "json").is_empty()); + assert_eq!(records(&scenario, "checks.json").len(), 1); + assert!(scenario.issue_labels(7).is_empty()); + assert_eq!(scenario.entries("work"), ["widgets"]); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); + } +} + +#[test] +fn an_effort_without_a_model_explains_how_to_select_the_route() { + let scenario = Scenario::new(); + let result = scenario.run(&[ + "harness", + "opencode", + "effort", + "high", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result.stderr.contains("set model / too"), + "{}", + result.stderr + ); + assert!(records(&scenario, "checks.json").is_empty()); +} + +#[test] +fn claude_only_instructions_are_linked_as_agents_and_kept_out_of_git() { + for existing in [false, true] { + let scenario = Scenario::new(); + std::fs::write(scenario.launch_dir().join("CLAUDE.md"), "claude rules\n").unwrap(); + if existing { + std::fs::write(scenario.launch_dir().join("AGENTS.md"), "own rules\n").unwrap(); + } + scenario.launch_git(&["add", "."]); + scenario.launch_git(&["commit", "-m", "Project instructions"]); + scenario.launch_git(&["push", "origin", "main"]); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&["harness", "opencode", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = records(&scenario, "json"); + assert_eq!( + calls[0]["agents_link"], + if existing { + Value::Null + } else { + json!("CLAUDE.md") + } + ); + assert_eq!( + calls[0]["agents_text"], + if existing { + "own rules\n" + } else { + "claude rules\n" + } + ); + assert_eq!(calls[0]["git_status"], ""); + assert_eq!( + scenario.origin_file("issue-7", "AGENTS.md"), + existing.then(|| "own rules\n".to_string()) + ); + } +} + +#[test] +fn an_exported_failed_outcome_fails_even_when_the_cli_exits_zero() { + let scenario = Scenario::new(); + scenario.agent_does_for( + 7, + "echo 'route became unavailable' > \"$FAKE_OPENCODE_ERROR\"", + ); + let result = scenario.run_with_env( + &["harness", "opencode", &scenario.issue_url(7)], + &[("FAKE_OPENCODE_EXPORT_FAILED", "1")], + ); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result + .stderr + .contains("opencode's turn failed: route became unavailable"), + "{}", + result.stderr + ); + assert_eq!(records(&scenario, "json").len(), 1); +} + +#[test] +fn a_failed_turn_in_the_stream_or_nonzero_exit_fails_with_opencodes_error() { + for script in [ + "echo 'provider unavailable' > \"$FAKE_OPENCODE_ERROR\"\nexit 1", + r#"echo '{"type":"error","sessionID":"fake-opencode-1","error":{"message":"provider unavailable"}}'"#, + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, script); + let result = scenario.run(&["harness", "opencode", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result.stderr.contains("provider unavailable"), + "{}", + result.stderr + ); + assert_eq!(records(&scenario, "json").len(), 1); + } +} + +#[test] +fn architecture_reviews_receive_the_last_reply_with_or_without_the_export() { + for setting in [ + None, + Some("FAKE_OPENCODE_NO_EXPORT"), + Some("FAKE_OPENCODE_CORRUPT_EXPORT"), + ] { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does_in_session( + 1, + r#" +url=$(gh issue create --title 'An idea' --body 'The idea' --label needs-triage) +printf 'Architecture review idea: %s\n' "$url" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let env = setting + .map(|setting| vec![(setting, "1")]) + .unwrap_or_default(); + let result = scenario.run_with_env(&["architect", "harness", "opencode"], &env); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert_eq!( + result.stdout.trim(), + "https://github.com/acme/widgets/issues/8" + ); + assert_eq!( + result.stderr.contains("2400 input tokens"), + setting.is_none() + ); + } +} + +#[test] +fn a_missing_skill_warns_in_the_progress_and_command_log_without_retrying() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run_with_env( + &["harness", "opencode", &scenario.issue_url(7)], + &[("FAKE_OPENCODE_SKIP_SKILL", "1")], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let warning = "warning: the session never loaded thirdshift-implement with its skill tool"; + assert_eq!(result.stderr.matches(warning).count(), 1); + assert_eq!(records(&scenario, "json").len(), 1); + let dir = "home/.thirdshift/logs/acme/widgets/commands/issue"; + let name = scenario.entries(dir).pop().unwrap(); + let log = std::fs::read_to_string(scenario.path(&format!("{dir}/{name}"))).unwrap(); + assert_eq!(log.matches(warning).count(), 1); +} + +#[test] +fn setup_proposes_no_model_and_checks_the_answer_standalone() { + let scenario = Scenario::new(); + let result = scenario.run_on_terminal( + &["setup"], + &[], + &[ + ("Harness for every Run", "opencode"), + ( + "Model for opencode [opencode's own default]", + "provider/model", + ), + ("Effort for opencode", "high"), + ("Every Run a Merge run", ""), + ("Every Run first fast-forwards", ""), + ("Run notifications", ""), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let config: toml::Table = result.user_config.unwrap().parse().unwrap(); + assert_eq!(config["harness"]["default"].as_str(), Some("opencode")); + assert_eq!( + config["harness"]["opencode"]["model"].as_str(), + Some("provider/model") + ); + assert_eq!( + config["harness"]["opencode"]["effort"].as_str(), + Some("high") + ); + let checks = records(&scenario, "checks.json"); + assert_eq!( + checks[0]["argv"], + json!([ + "run", + "--standalone", + "--format", + "json", + "--auto", + "-m", + "provider/model#high" + ]) + ); + assert_eq!(checks[0]["prompt"], "Reply with OK."); + assert_eq!(checks[0]["no_auto_update"], "1"); +} + +#[test] +fn an_exported_failed_check_stops_before_any_work_even_with_exit_zero() { + let scenario = Scenario::new(); + let result = scenario.run_with_env( + &["harness", "opencode", &scenario.issue_url(7)], + &[("FAKE_OPENCODE_CHECK_EXPORT_FAILED", "1")], + ); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result.stderr.contains("exported outcome is failed"), + "{}", + result.stderr + ); + assert!(scenario.issue_labels(7).is_empty()); + assert!(records(&scenario, "json").is_empty()); +} + +#[test] +fn without_a_stream_session_id_no_resume_or_export_id_is_invented() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#"{AGENT_OPENS_PR} +echo '{{"type":"tool_use","part":{{"tool":"bash","state":{{"status":"error","input":{{"command":"cargo test"}},"error":{{"type":"aborted","message":"Tool execution interrupted"}}}}}}}}' +"#)); + let result = scenario.run_with_env( + &["harness", "opencode", &scenario.issue_url(7)], + &[("FAKE_OPENCODE_NO_SESSION_ID", "1")], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert_eq!(records(&scenario, "json").len(), 1); + assert!(records(&scenario, "exports.json").is_empty()); + assert!(!result.stderr.contains("input tokens")); +} + +#[test] +fn setup_retries_a_refused_model_and_effort_with_the_same_preflight_check() { + let scenario = Scenario::new(); + let result = scenario.run_on_terminal( + &["setup"], + &[], + &[ + ("Harness for every Run", "opencode"), + ("Model for opencode", "provider/bad-model"), + ("Effort for opencode", "high"), + ("Model for opencode", "provider/model"), + ("Effort for opencode", "low"), + ("Every Run a Merge run", ""), + ("Every Run first fast-forwards", ""), + ("Run notifications", ""), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let checks = records(&scenario, "checks.json"); + assert_eq!(checks.len(), 2); + assert_eq!(checks[0]["argv"][6], "provider/bad-model#high"); + assert_eq!(checks[1]["argv"][6], "provider/model#low"); +} diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 0c6e379e9..1308738cb 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -50,6 +50,11 @@ fn interrupting_an_agy_run_stops_a_command_in_its_own_session() { assert_detached_command_stopped("agy"); } +#[test] +fn interrupting_an_opencode_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("opencode"); +} + fn assert_detached_command_stopped(harness: &str) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); diff --git a/tests/setup.rs b/tests/setup.rs index ebab09416..5130105d0 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -72,7 +72,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex", "agy", "muse"] { + for harness in ["claude", "codex", "agy", "muse", "opencode"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 18cf8e04f..c25da943e 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -11,19 +11,28 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 7] = [ +const SOURCES: [&str; 8] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), include_str!("../fakes/codex.rs"), include_str!("../fakes/agy.rs"), include_str!("../fakes/muse.rs"), + include_str!("../fakes/opencode.rs"), include_str!("../fakes/json.rs"), ]; /// Put the fake `gh`, `claude`, `codex`, `agy` and `muse` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex", "agy", "muse", "detached-command"] { + for name in [ + "gh", + "claude", + "codex", + "agy", + "muse", + "opencode", + "detached-command", + ] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 70d295e16..15e4aa9d9 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -606,6 +606,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_AGY_RECORD", self.path("agy-calls.json")) .env("FAKE_AGY_CHECK_RECORD", self.path("agy-checks.json")) .env("FAKE_MUSE_RECORD", self.path("muse-calls.json")) + .env("FAKE_OPENCODE_RECORD", self.path("opencode-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, From 219ef35995d8ca4f97b5f0b02924b498a29c607b Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:39:17 -0400 Subject: [PATCH 13/16] Expect the OpenCode section in Setup defaults (#439) --- tests/setup.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/setup.rs b/tests/setup.rs index 5130105d0..a523bf4fd 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -54,7 +54,8 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[harness.claude]", "[harness.codex]", "[harness.agy]", - "[harness.muse]" + "[harness.muse]", + "[harness.opencode]" ], "{text}" ); From 5e852102661c83e347652e7f3c8c0dcf4995fcf6 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:48:47 -0400 Subject: [PATCH 14/16] Address Standards and Spec findings for OpenCode (#439) --- README.md | 2 +- src/harness.rs | 1 + src/harness/muse/stream.rs | 27 ++++----------- src/harness/opencode.rs | 55 +++++++++++++++++++++++++----- src/harness/opencode/export.rs | 62 ++++++++++++++++++++++++++-------- src/harness/opencode/stream.rs | 26 ++++---------- src/harness/skill_load.rs | 38 +++++++++++++++++++++ tests/fakes/opencode.rs | 8 ++++- tests/opencode_sessions.rs | 8 ++--- tests/process_tree.rs | 34 +++++++++++++++---- 10 files changed, 184 insertions(+), 77 deletions(-) create mode 100644 src/harness/skill_load.rs diff --git a/README.md b/README.md index f246324bd..196e41dd9 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ With `harness opencode`, every session, Resume and preflight check runs `opencod OpenCode takes the Model id from your provider config, as `/`, including MiMo-V2.6-Pro when your provider routes it. Model and Effort are passed together as `-m /#`; an explicit Effort replaces a variant already in the Model id, and needs a Model set too. Before any work, thirdshift makes a minimal standalone test call with the same stdin, flags and environment. `provider.no-route` identifies a bad Model or Effort without a model turn; a successful check uses a model call because OpenCode has no free catalog without its service. Setup checks its answers the same way, proposing no default Model. -The session id comes from `sessionID` in the stream, and a Resume uses `-s ` with every flag and environment override again. OpenCode creates a new session for an unknown id, so thirdshift only resumes an id it read from the stream. Progress comes from `step_start`, `tool_use`, `step_finish` and `error`. An aborted tool in a successful session gets the usual Resume. After exit, thirdshift reads `opencode session export --standalone `: an exported `failed` outcome fails the Run even with exit 0, the final message is the last assistant message's last text part, and usage is summed over assistant messages to exclude the automatic title call. If export fails or is unreadable, the last stream `text` supplies the final message and no usage is shown. A stream error or nonzero exit also fails the Run with OpenCode's error. An interrupt sends SIGTERM through the whole process tree, then SIGKILL after the grace period if needed. +The session id comes from `sessionID` in the stream, and a Resume uses `-s ` with every flag and environment override again. OpenCode creates a new session for an unknown id, so thirdshift only resumes an id it read from the stream. Progress comes from `step_start`, `tool_use`, `step_finish` and `error`. An aborted tool in a successful session gets the usual Resume. After exit, thirdshift reads `opencode session export --standalone `: an exported `failed` outcome fails the Run even with exit 0, the final message is the last assistant message's last text part, and usage and cost are summed over assistant messages to exclude the automatic title call. Input totals include cache reads and writes, output totals include reasoning, and each subtotal is shown. If export fails or is unreadable, the last stream `text` supplies the final message and no usage is shown. A stream error or nonzero exit also fails the Run with OpenCode's error. An interrupt sends SIGTERM through the whole process tree, then SIGKILL after the grace period if needed. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. diff --git a/src/harness.rs b/src/harness.rs index 6b2449b07..ffd7d90de 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -17,6 +17,7 @@ mod instructions; pub(crate) mod muse; pub(crate) mod opencode; mod process_tree; +mod skill_load; pub use adapter::{Adapter, Invocation}; pub use codex::Catalog; diff --git a/src/harness/muse/stream.rs b/src/harness/muse/stream.rs index e0a77a166..cd2938004 100644 --- a/src/harness/muse/stream.rs +++ b/src/harness/muse/stream.rs @@ -1,5 +1,6 @@ //! Muse's JSONL event envelopes. Usage is read from its session log after exit. +use crate::harness::skill_load::SkillLoad; use crate::progress::{Stream, bash, shorten}; use serde_json::Value; @@ -9,8 +10,7 @@ pub struct MuseProgress { started: bool, final_message: Option, failure: Option, - expected_skill: Option, - skill_loaded: bool, + skill_load: SkillLoad, tasks: Vec<(String, String)>, killed: Vec, } @@ -18,11 +18,7 @@ pub struct MuseProgress { impl MuseProgress { pub fn for_prompt(prompt: &str) -> Self { Self { - expected_skill: prompt - .strip_prefix('/') - .and_then(|rest| rest.split_whitespace().next()) - .filter(|name| name.starts_with("thirdshift-")) - .map(String::from), + skill_load: SkillLoad::for_prompt(prompt), ..Self::default() } } @@ -38,11 +34,8 @@ impl MuseProgress { .and_then(|rest| rest.split_once('"')) .map(|(name, _)| name) .unwrap_or(""); - if facts["outcome"] == "success" - && header.contains("status=\"ok\"") - && self.expected_skill.as_deref() == Some(skill) - { - self.skill_loaded = true; + if facts["outcome"] == "success" && header.contains("status=\"ok\"") { + self.skill_load.observed(skill); } return vec![format!("skill {skill}")]; } @@ -158,15 +151,7 @@ impl Stream for MuseProgress { self.failure.as_deref() } fn warnings(&self) -> Vec { - self.expected_skill - .as_ref() - .filter(|_| !self.skill_loaded) - .map(|skill| { - vec![format!( - "warning: the session never loaded {skill} with its skill tool" - )] - }) - .unwrap_or_default() + self.skill_load.warnings() } } diff --git a/src/harness/opencode.rs b/src/harness/opencode.rs index f6a73116e..02e312301 100644 --- a/src/harness/opencode.rs +++ b/src/harness/opencode.rs @@ -6,10 +6,13 @@ use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings, said}; use crate::progress::Stream; use crate::setup::{Outside, questions::ask_setting}; -use anyhow::{Context, Result, bail}; -use std::io::Write; +use anyhow::{Context, Result, anyhow, bail}; +use std::io::{Read, Write}; +use std::os::unix::process::CommandExt; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Output, Stdio}; +use std::thread; +use std::time::Duration; pub struct OpenCode; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::OpenCode, &OpenCode); @@ -119,6 +122,7 @@ pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result<()> { let mut child = Command::new(OpenCode.name()) .args(invocation.args) .envs(OpenCode.environment().iter().copied()) + .process_group(0) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -130,13 +134,10 @@ pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result<()> { .context("no stdin from opencode")? .write_all(invocation.stdin.as_deref().unwrap_or("").as_bytes()); if let Err(error) = written { - let _ = child.kill(); - let _ = child.wait(); + OpenCode.stop(&mut child); return Err(error).context("could not write the check prompt to opencode"); } - let output = child - .wait_with_output() - .context("could not wait for opencode's test call")?; + let output = check_output(&mut child)?; let mut stream = OpenCode.stream(Path::new("."), "Reply with OK."); for line in String::from_utf8_lossy(&output.stdout).lines() { stream.condense(line); @@ -158,3 +159,41 @@ pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result<()> { fn error_text(error: &serde_json::Value) -> Option<&str> { error.as_str().or_else(|| error["message"].as_str()) } + +/// Drain both pipes while watching for interruption, so a slow check cannot +/// strand its private server or commands before a Run has even started work. +fn check_output(child: &mut Child) -> Result { + fn read(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle>> { + thread::spawn(move || { + let mut bytes = Vec::new(); + pipe.read_to_end(&mut bytes)?; + Ok(bytes) + }) + } + let stdout = read(child.stdout.take().context("no stdout from opencode")?); + let stderr = read(child.stderr.take().context("no stderr from opencode")?); + let status = loop { + if crate::interrupt::requested() { + OpenCode.stop(child); + let _ = stdout.join(); + let _ = stderr.join(); + bail!("interrupted"); + } + if let Some(status) = child + .try_wait() + .context("could not wait for opencode's test call")? + { + break status; + } + thread::sleep(Duration::from_millis(100)); + }; + Ok(Output { + status, + stdout: stdout + .join() + .map_err(|_| anyhow!("OpenCode's check stdout reader panicked"))??, + stderr: stderr + .join() + .map_err(|_| anyhow!("OpenCode's check stderr reader panicked"))??, + }) +} diff --git a/src/harness/opencode/export.rs b/src/harness/opencode/export.rs index c130b462c..b44d83df5 100644 --- a/src/harness/opencode/export.rs +++ b/src/harness/opencode/export.rs @@ -14,6 +14,44 @@ pub struct SessionExport { failure: Option, } +/// OpenCode records uncached input and visible output separately from cache +/// and reasoning. Combine them into totals while retaining each subtotal. +#[derive(Default)] +struct Usage { + input: u64, + cache_read: u64, + cache_write: u64, + output: u64, + reasoning: u64, + cost: Option, +} + +impl Usage { + fn add(&mut self, message: &Value) { + let tokens = &message["tokens"]; + self.input += tokens["input"].as_u64().unwrap_or(0); + self.cache_read += tokens["cache"]["read"].as_u64().unwrap_or(0); + self.cache_write += tokens["cache"]["write"].as_u64().unwrap_or(0); + self.output += tokens["output"].as_u64().unwrap_or(0); + self.reasoning += tokens["reasoning"].as_u64().unwrap_or(0); + if let Some(cost) = message["cost"].as_f64() { + *self.cost.get_or_insert(0.0) += cost; + } + } + fn summary(&self) -> String { + let input = self.input + self.cache_read + self.cache_write; + let output = self.output + self.reasoning; + let cost = self + .cost + .map(|cost| format!(", ${cost:.4}")) + .unwrap_or_default(); + format!( + "{input} input tokens ({} cache read, {} cache write), {output} output tokens ({} reasoning){cost}", + self.cache_read, self.cache_write, self.reasoning + ) + } +} + impl SessionExport { pub fn parse(text: &str) -> Result { let export: Value = serde_json::from_str(text)?; @@ -28,8 +66,7 @@ impl SessionExport { summary: None, failure: None, }; - let (mut input, mut cached, mut output) = (0, 0, 0); - let mut usage = false; + let mut usage: Option = None; for message in messages .iter() .filter(|message| message["type"] == "assistant") @@ -39,21 +76,14 @@ impl SessionExport { .and_then(|parts| parts.iter().rev().find(|part| part["type"] == "text")) .and_then(|part| part["text"].as_str()) .map(String::from); - if let Some(tokens) = message["tokens"].as_object() { - usage = true; - input += tokens.get("input").and_then(Value::as_u64).unwrap_or(0); - cached += message["tokens"]["cache"]["read"].as_u64().unwrap_or(0); - output += tokens.get("output").and_then(Value::as_u64).unwrap_or(0); + if message["tokens"].is_object() { + usage.get_or_insert_default().add(message); } if let Some(error) = super::error_text(&message["error"]) { result.failure = Some(error.into()); } } - if usage { - result.summary = Some(format!( - "{input} input tokens ({cached} cached), {output} output tokens" - )); - } + result.summary = usage.as_ref().map(Usage::summary); if outcome == "failed" { result.failure = Some( super::error_text(&export["info"]["error"]) @@ -144,8 +174,8 @@ mod tests { "info":{"outcome":"succeeded","tokens":{"input":999999},"cost":9}, "messages":[ {"type":"user","text":"prompt","tokens":{"input":999999}}, - {"type":"assistant","tokens":{"input":31,"output":37,"reasoning":158,"cache":{"read":7840,"write":0}},"content":[{"type":"text","text":"earlier reply"}]}, - {"type":"assistant","tokens":{"input":39,"output":11,"reasoning":42,"cache":{"read":496,"write":0}},"content":[{"type":"text","text":"first part"},{"type":"reasoning","text":"private reasoning"},{"type":"text","text":"Architecture review idea: https://github.com/acme/widgets/issues/8"}]}, + {"type":"assistant","tokens":{"input":31,"output":37,"reasoning":158,"cache":{"read":7840,"write":20}},"cost":0.004,"content":[{"type":"text","text":"earlier reply"}]}, + {"type":"assistant","tokens":{"input":39,"output":11,"reasoning":42,"cache":{"read":496,"write":7}},"cost":0.002,"content":[{"type":"text","text":"first part"},{"type":"reasoning","text":"private reasoning"},{"type":"text","text":"Architecture review idea: https://github.com/acme/widgets/issues/8"}]}, {"type":"idle","outcome":"succeeded"} ] }"#).unwrap(); @@ -155,7 +185,9 @@ mod tests { ); assert_eq!( export.summary.as_deref(), - Some("70 input tokens (8336 cached), 48 output tokens") + Some( + "8433 input tokens (8336 cache read, 27 cache write), 248 output tokens (200 reasoning), $0.0060" + ) ); assert!(export.failure.is_none()); } diff --git a/src/harness/opencode/stream.rs b/src/harness/opencode/stream.rs index a8048fffb..9fe8c8a5c 100644 --- a/src/harness/opencode/stream.rs +++ b/src/harness/opencode/stream.rs @@ -1,4 +1,5 @@ //! OpenCode's JSONL progress; final usage is deliberately left to session export. +use crate::harness::skill_load::SkillLoad; use crate::progress::{Stream, bash, shorten}; use serde_json::Value; @@ -8,19 +9,14 @@ pub struct OpenCodeProgress { started: bool, message: Option, failure: Option, - expected_skill: Option, - skill_loaded: bool, + skill_load: SkillLoad, killed: Vec, } impl OpenCodeProgress { pub fn for_prompt(prompt: &str) -> Self { Self { - expected_skill: prompt - .strip_prefix('/') - .and_then(|rest| rest.split_whitespace().next()) - .filter(|name| name.starts_with("thirdshift-")) - .map(String::from), + skill_load: SkillLoad::for_prompt(prompt), ..Self::default() } } @@ -62,10 +58,8 @@ impl Stream for OpenCodeProgress { } if tool == "skill" { let skill = state["input"]["id"].as_str().unwrap_or(""); - if state["status"] == "completed" - && self.expected_skill.as_deref() == Some(skill) - { - self.skill_loaded = true; + if state["status"] == "completed" { + self.skill_load.observed(skill); } vec![format!("skill {skill}")] } else if tool == "bash" { @@ -119,15 +113,7 @@ impl Stream for OpenCodeProgress { self.failure.as_deref() } fn warnings(&self) -> Vec { - self.expected_skill - .as_ref() - .filter(|_| !self.skill_loaded) - .map(|skill| { - vec![format!( - "warning: the session never loaded {skill} with its skill tool" - )] - }) - .unwrap_or_default() + self.skill_load.warnings() } } diff --git a/src/harness/skill_load.rs b/src/harness/skill_load.rs new file mode 100644 index 000000000..6fe4ddf62 --- /dev/null +++ b/src/harness/skill_load.rs @@ -0,0 +1,38 @@ +//! Warning policy for Harnesses whose models load Factory skills with a tool. +#[derive(Default)] +pub struct SkillLoad { + expected: Option, + loaded: bool, +} + +impl SkillLoad { + pub fn for_prompt(prompt: &str) -> Self { + Self { + expected: prompt + .strip_prefix('/') + .and_then(|rest| rest.split_whitespace().next()) + .filter(|name| name.starts_with("thirdshift-")) + .map(String::from), + loaded: false, + } + } + + /// Adapters decode their own event format and report successful loads only. + pub fn observed(&mut self, name: &str) { + if self.expected.as_deref() == Some(name) { + self.loaded = true; + } + } + + pub fn warnings(&self) -> Vec { + self.expected + .as_ref() + .filter(|_| !self.loaded) + .map(|skill| { + vec![format!( + "warning: the session never loaded {skill} with its skill tool" + )] + }) + .unwrap_or_default() + } +} diff --git a/tests/fakes/opencode.rs b/tests/fakes/opencode.rs index c61e95805..610042f84 100644 --- a/tests/fakes/opencode.rs +++ b/tests/fakes/opencode.rs @@ -63,6 +63,12 @@ pub fn main(argv: Vec) { ])); crate::exit(1); } + if let Ok(script) = std::env::var("FAKE_OPENCODE_CHECK_SCRIPT") { + let status = Command::new("bash").arg("-e").arg(script).status().unwrap(); + if !status.success() { + crate::exit(exit_code(status)); + } + } event("check-session", "text", object([("text", string("OK"))])); fs::write( export_path("check-session"), @@ -207,7 +213,7 @@ pub fn main(argv: Vec) { ) .unwrap(); // Alternate complete and dropped closing events, as real OpenCode does. - if session % 2 == 0 { + if session.is_multiple_of(2) { event( &id, "step_finish", diff --git a/tests/opencode_sessions.rs b/tests/opencode_sessions.rs index ec6697d1a..08cce3024 100644 --- a/tests/opencode_sessions.rs +++ b/tests/opencode_sessions.rs @@ -59,9 +59,9 @@ fn opencode_runs_standalone_with_stdin_skills_and_usage_despite_a_dropped_last_s ); } assert!( - result - .stderr - .contains("2400 input tokens (400 cached), 600 output tokens"), + result.stderr.contains( + "2800 input tokens (400 cache read, 0 cache write), 800 output tokens (200 reasoning)" + ), "{}", result.stderr ); @@ -298,7 +298,7 @@ printf 'Architecture review idea: %s\n' "$url" > "$FAKE_CLAUDE_FINAL_MESSAGE" "https://github.com/acme/widgets/issues/8" ); assert_eq!( - result.stderr.contains("2400 input tokens"), + result.stderr.contains("2800 input tokens"), setting.is_none() ); } diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 1308738cb..da993303f 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -37,40 +37,56 @@ impl Drop for DetachedCommand { #[test] fn interrupting_a_claude_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("claude"); + assert_detached_command_stopped("claude", false); } #[test] fn interrupting_a_codex_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("codex"); + assert_detached_command_stopped("codex", false); } #[test] fn interrupting_an_agy_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("agy"); + assert_detached_command_stopped("agy", false); } #[test] fn interrupting_an_opencode_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("opencode"); + assert_detached_command_stopped("opencode", false); } -fn assert_detached_command_stopped(harness: &str) { +#[test] +fn interrupting_an_opencode_check_stops_its_whole_process_tree_before_work() { + assert_detached_command_stopped("opencode", true); +} + +fn assert_detached_command_stopped(harness: &str, checking: bool) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); + let ending = if checking { + "sleep 9" + } else { + "while :; do sleep 0.1; done" + }; scenario.agent_does(&format!( r#"detached-command bash -c 'trap "" INT TERM; echo $$ > "{pid}"; exec sleep 120' >/dev/null 2>&1 & while ! test -s "{pid}"; do sleep 0.01; done touch "{started}" -while :; do sleep 0.1; done +{ending} "#, pid = pid.display(), started = scenario.path("agent-started").display(), )); + let script = scenario.path("claude-script.sh"); + let env = if checking { + vec![("FAKE_OPENCODE_CHECK_SCRIPT", script.to_str().unwrap())] + } else { + Vec::new() + }; let mut held = scenario.run_until( &["harness", harness, &scenario.issue_url(7)], - &[], + &env, "agent-started", ); let command = DetachedCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); @@ -80,6 +96,10 @@ while :; do sleep 0.1; done let result = held.finish(); assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + if checking { + assert!(scenario.issue_labels(7).is_empty()); + assert_eq!(scenario.entries("work"), ["widgets"]); + } assert!( interrupted_at.elapsed() < Duration::from_secs(8), "a session that exited on its first signal incurred another grace period" From a6c6a3b4d74c2869645f832f686024a2669975b3 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:57:38 -0400 Subject: [PATCH 15/16] Use documented OpenCode stream events for Resume coverage (#439) --- README.md | 2 +- src/harness/opencode.rs | 42 ++++++++++++++++++++++++++++++++++ src/harness/opencode/stream.rs | 20 +++------------- tests/fakes.rs | 2 +- tests/opencode_sessions.rs | 40 +------------------------------- tests/support/fakes.rs | 4 ++-- 6 files changed, 50 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 9279fd2ba..b410771b6 100644 --- a/README.md +++ b/README.md @@ -346,7 +346,7 @@ With `harness opencode`, every session, Resume and preflight check runs `opencod OpenCode takes the Model id from your provider config, as `/`, including MiMo-V2.6-Pro when your provider routes it. Model and Effort are passed together as `-m /#`; an explicit Effort replaces a variant already in the Model id, and needs a Model set too. Before any work, thirdshift makes a minimal standalone test call with the same stdin, flags and environment. `provider.no-route` identifies a bad Model or Effort without a model turn; a successful check uses a model call because OpenCode has no free catalog without its service. Setup checks its answers the same way, proposing no default Model. -The session id comes from `sessionID` in the stream, and a Resume uses `-s ` with every flag and environment override again. OpenCode creates a new session for an unknown id, so thirdshift only resumes an id it read from the stream. Progress comes from `step_start`, `tool_use`, `step_finish` and `error`. An aborted tool in a successful session gets the usual Resume. After exit, thirdshift reads `opencode session export --standalone `: an exported `failed` outcome fails the Run even with exit 0, the final message is the last assistant message's last text part, and usage and cost are summed over assistant messages to exclude the automatic title call. Input totals include cache reads and writes, output totals include reasoning, and each subtotal is shown. If export fails or is unreadable, the last stream `text` supplies the final message and no usage is shown. A stream error or nonzero exit also fails the Run with OpenCode's error. An interrupt sends SIGTERM through the whole process tree, then SIGKILL after the grace period if needed. +The session id comes from `sessionID` in the stream, and a Resume uses `-s ` with every flag and environment override again. OpenCode creates a new session for an unknown id, so thirdshift only resumes an id it read from the stream. Progress comes from `step_start`, `tool_use`, `step_finish` and `error`. After exit, thirdshift reads `opencode session export --standalone `: an exported `failed` outcome fails the Run even with exit 0, the final message is the last assistant message's last text part, and usage and cost are summed over assistant messages to exclude the automatic title call. Input totals include cache reads and writes, output totals include reasoning, and each subtotal is shown. If export fails or is unreadable, the last stream `text` supplies the final message and no usage is shown. A stream error or nonzero exit also fails the Run with OpenCode's error. An interrupt sends SIGTERM through the whole process tree, then SIGKILL after the grace period if needed. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. diff --git a/src/harness/opencode.rs b/src/harness/opencode.rs index 02e312301..568917380 100644 --- a/src/harness/opencode.rs +++ b/src/harness/opencode.rs @@ -197,3 +197,45 @@ fn check_output(child: &mut Child) -> Result { .map_err(|_| anyhow!("OpenCode's check stderr reader panicked"))??, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_resume_uses_the_streamed_session_id_with_every_flag_and_the_prompt_on_stdin() { + let choice = Choice { + harness: Harness::OpenCode, + model: Some("provider/model".into()), + effort: Some("high".into()), + chosen_by: super::super::ChosenBy::Command, + }; + let prompt = "/thirdshift-implement issue-url"; + let mut stream = OpenCode.stream(Path::new("."), prompt); + stream.condense(r#"{"type":"step_start","sessionID":"ses_eedb9657fffec0RAa52IqAvyM1","part":{"type":"step-start"}}"#); + let invocation = OpenCode.session(&choice, stream.session_id(), prompt); + assert_eq!( + invocation.args, + [ + "run", + "--standalone", + "--format", + "json", + "--auto", + "-m", + "provider/model#high", + "-s", + "ses_eedb9657fffec0RAa52IqAvyM1" + ] + ); + assert_eq!( + invocation.stdin.as_deref(), + Some("Load thirdshift-implement with your skill tool. issue-url") + ); + assert_eq!( + OpenCode.environment(), + [("OPENCODE_DISABLE_AUTOUPDATE", "1")] + ); + assert!(stream.killed_background_work().is_empty()); + } +} diff --git a/src/harness/opencode/stream.rs b/src/harness/opencode/stream.rs index 9fe8c8a5c..58d767315 100644 --- a/src/harness/opencode/stream.rs +++ b/src/harness/opencode/stream.rs @@ -10,7 +10,6 @@ pub struct OpenCodeProgress { message: Option, failure: Option, skill_load: SkillLoad, - killed: Vec, } impl OpenCodeProgress { @@ -45,17 +44,6 @@ impl Stream for OpenCodeProgress { Some("tool_use") => { let tool = part["tool"].as_str().unwrap_or(""); let state = &part["state"]; - if matches!(tool, "bash" | "task") - && state["status"] == "error" - && state["error"]["type"] == "aborted" - { - self.killed.push( - state["input"]["command"] - .as_str() - .unwrap_or(tool) - .to_string(), - ); - } if tool == "skill" { let skill = state["input"]["id"].as_str().unwrap_or(""); if state["status"] == "completed" { @@ -100,11 +88,9 @@ impl Stream for OpenCodeProgress { self.session_id.as_deref() } fn killed_background_work(&self) -> Vec<&str> { - if self.failed() { - Vec::new() - } else { - self.killed.iter().map(String::as_str).collect() - } + // OpenCode's stream has no documented event proving background work + // was killed at session end. Tool cancellation alone is not evidence. + Vec::new() } fn failed(&self) -> bool { self.failure.is_some() diff --git a/tests/fakes.rs b/tests/fakes.rs index 8fee4e4ca..f2ea69807 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse` the end-to-end tests put first on +//! The fake `gh`, `claude`, `codex`, `agy`, `grok`, `muse` and `opencode` the end-to-end tests put first on //! PATH: one executable, which acts as whichever its name says. See //! `fakes/gh.rs`, `fakes/claude.rs` and `fakes/codex.rs` for what each does, //! and `support/fakes.rs` for how the harness builds it. diff --git a/tests/opencode_sessions.rs b/tests/opencode_sessions.rs index 08cce3024..0e4f5dddc 100644 --- a/tests/opencode_sessions.rs +++ b/tests/opencode_sessions.rs @@ -88,42 +88,6 @@ fn opencode_runs_standalone_with_stdin_skills_and_usage_despite_a_dropped_last_s ); } -#[test] -fn opencode_resumes_only_the_stream_session_id_with_the_same_standalone_flags() { - let scenario = Scenario::new(); - scenario.agent_does_for_in_session(7, 1, &format!(r#"{AGENT_OPENS_PR} -echo '{{"type":"tool_use","sessionID":"fake-opencode-1","part":{{"tool":"bash","state":{{"status":"error","input":{{"command":"cargo test"}},"error":{{"type":"aborted","message":"Tool execution interrupted"}}}}}}}}' -"#)); - let result = scenario.run(&[ - "harness", - "opencode", - "model", - "provider/model", - "effort", - "high", - &scenario.issue_url(7), - ]); - assert_eq!(result.code, Some(0), "{}", result.stderr); - let calls = records(&scenario, "json"); - assert_eq!(calls.len(), 2, "{}", result.stderr); - assert_eq!( - calls[1]["argv"], - json!([ - "run", - "--standalone", - "--format", - "json", - "--auto", - "-m", - "provider/model#high", - "-s", - "fake-opencode-1" - ]) - ); - assert_eq!(calls[1]["no_auto_update"], "1"); - assert!(calls[1]["prompt"].as_str().unwrap().contains("cargo test")); -} - #[test] fn user_config_selects_opencode_and_command_effort_overrides_its_variant() { let scenario = Scenario::new(); @@ -388,9 +352,7 @@ fn an_exported_failed_check_stops_before_any_work_even_with_exit_zero() { #[test] fn without_a_stream_session_id_no_resume_or_export_id_is_invented() { let scenario = Scenario::new(); - scenario.agent_does_for(7, &format!(r#"{AGENT_OPENS_PR} -echo '{{"type":"tool_use","part":{{"tool":"bash","state":{{"status":"error","input":{{"command":"cargo test"}},"error":{{"type":"aborted","message":"Tool execution interrupted"}}}}}}}}' -"#)); + scenario.agent_does_for(7, AGENT_OPENS_PR); let result = scenario.run_with_env( &["harness", "opencode", &scenario.issue_url(7)], &[("FAKE_OPENCODE_NO_SESSION_ID", "1")], diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 3a2738fab..6cecdb2ca 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse`, built from +//! The fake `gh`, `claude`, `codex`, `agy`, `grok`, `muse` and `opencode`, built from //! `tests/fakes.rs` with plain `rustc` the first time a test needs them and //! kept in Cargo's scratch directory for tests. They aren't a binary target, //! so neither the crate nor a release ships them. @@ -25,7 +25,7 @@ const SOURCES: [&str; 11] = [ include_str!("../fixtures/grok-models.txt"), ]; -/// Put the fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse` in `bin`. +/// Put the fake `gh`, `claude`, `codex`, `agy`, `grok`, `muse` and `opencode` in `bin`. pub fn install(bin: &Path) { for name in [ "gh", From a739d3c047762505a2ea06725b49ec744232a146 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 14:28:08 -0400 Subject: [PATCH 16/16] Fix Harness interruption and progress gaps (#432) --- README.md | 4 +- src/harness.rs | 1 + src/harness/adapter.rs | 10 ++- src/harness/grok/stream.rs | 30 +++++++- src/harness/muse.rs | 22 +++--- src/harness/opencode.rs | 75 +++--------------- src/harness/opencode/export.rs | 35 +++++---- src/harness/process_output.rs | 84 +++++++++++++++++++++ src/session.rs | 2 +- tests/fakes.rs | 31 ++++++++ tests/fakes/agy.rs | 2 +- tests/fakes/codex.rs | 32 +------- tests/fakes/grok.rs | 2 +- tests/fakes/muse.rs | 48 +++++------- tests/fakes/opencode.rs | 12 ++- tests/process_tree.rs | 134 ++++++++++++++++++++++++--------- 16 files changed, 325 insertions(+), 199 deletions(-) create mode 100644 src/harness/process_output.rs diff --git a/README.md b/README.md index b410771b6..f96e88f78 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,7 @@ One statically linked Linux binary covers the first three, so it doesn't depend The user account that runs thirdshift needs: -- **`claude`** (Claude Code), logged in. -- **`codex`** (Codex CLI), logged in, only for `harness codex`. -- **`agy`** (Google Antigravity CLI), logged in, only for `harness agy`. +- The selected **Harness** installed and authenticated: **`claude`** (Claude Code, the default), **`codex`** (Codex CLI), **`agy`** (Google Antigravity CLI), **`grok`** (Grok Build), **`muse`** (Muse Code), or **`opencode`** (OpenCode, with a configured Model provider). See [Harness, Model and Effort](#harness-model-and-effort) for each CLI's flags and quirks. - **`gh`** (GitHub CLI), logged in. - **`git`** with a global `user.name` and `user.email`, and credentials that can push to the repository (`gh auth setup-git` makes git use `gh`'s login). The agents commit as this identity; without it, an agent may borrow the author of the last commit. diff --git a/src/harness.rs b/src/harness.rs index 207ae8bb4..b0ecdf1d0 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -17,6 +17,7 @@ pub(crate) mod grok; mod instructions; pub(crate) mod muse; pub(crate) mod opencode; +mod process_output; mod process_tree; mod skill_load; diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index f962bfcae..09aab6690 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -50,9 +50,13 @@ pub trait Adapter: Sync { /// Optionally reconcile final text, usage and outcome with the CLI's /// own session log or export after exit, before reporting the ending. - /// Returning the stream unchanged is the fallback when no read is needed. - fn read_after_exit(&self, _worktree: &Path, stream: Box) -> Box { - stream + /// Missing or unreadable data falls back to the stream; interruption fails. + fn read_after_exit( + &self, + _worktree: &Path, + stream: Box, + ) -> Result> { + Ok(stream) } } diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs index e984b6f30..ca3ba0c28 100644 --- a/src/harness/grok/stream.rs +++ b/src/harness/grok/stream.rs @@ -3,7 +3,7 @@ use serde_json::Value; use crate::harness::claude::stream::ClaudeProgress; -use crate::progress::Stream; +use crate::progress::{Stream, shorten}; #[derive(Default)] pub struct GrokProgress { @@ -85,7 +85,14 @@ impl Stream for GrokProgress { } } } - self.claude.condense(&event.to_string()) + let mut lines = self.claude.condense(&event.to_string()); + if self.failed && matches!(event["type"].as_str(), Some("error" | "result")) { + lines.push(format!( + "error: {}", + shorten(self.error.as_deref().unwrap_or("Grok's turn failed")) + )); + } + lines } fn summary(&self) -> Option { if self.cost_known { @@ -141,6 +148,25 @@ mod tests { ); } + #[test] + fn error_events_are_condensed_to_progress_with_groks_failure_text() { + for (raw, message) in [ + ( + r#"{"type":"error","message":"backend unavailable"}"#, + "backend unavailable", + ), + ( + r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#, + "quota exceeded", + ), + ] { + let mut stream = GrokProgress::default(); + assert_eq!(stream.condense(raw), [format!("error: {message}")]); + assert!(stream.failed()); + assert_eq!(stream.error(), Some(message)); + } + } + #[test] fn malformed_and_unrelated_lines_are_ignored_and_error_frames_report_the_failure() { let mut stream = GrokProgress::default(); diff --git a/src/harness/muse.rs b/src/harness/muse.rs index e1c136057..44c88b561 100644 --- a/src/harness/muse.rs +++ b/src/harness/muse.rs @@ -5,7 +5,7 @@ mod log; mod stream; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::Command; use anyhow::{Context, Result, bail}; @@ -96,8 +96,12 @@ impl Adapter for Muse { fn stream(&self, _worktree: &Path, prompt: &str) -> Box { Box::new(stream::MuseProgress::for_prompt(prompt)) } - fn read_after_exit(&self, _worktree: &Path, stream: Box) -> Box { - log::read_after_exit(stream) + fn read_after_exit( + &self, + _worktree: &Path, + stream: Box, + ) -> Result> { + Ok(log::read_after_exit(stream)) } } @@ -131,12 +135,12 @@ pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result "checking the Model {model} with a test call to muse" )); let invocation = Muse.session(&choice, None, "Reply with OK."); - let output = Command::new(Muse.name()) - .args(invocation.args) - .envs(Muse.environment().iter().copied()) - .stdin(Stdio::null()) - .output() - .context("could not run muse to check the Model")?; + let output = super::process_output::output( + &Muse, + Command::new(Muse.name()).args(invocation.args), + None, + ) + .context("could not run muse to check the Model")?; let mut stream = Muse.stream(Path::new("."), "Reply with OK."); for line in String::from_utf8_lossy(&output.stdout).lines() { stream.condense(line); diff --git a/src/harness/opencode.rs b/src/harness/opencode.rs index 568917380..ff4f254b5 100644 --- a/src/harness/opencode.rs +++ b/src/harness/opencode.rs @@ -6,13 +6,9 @@ use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings, said}; use crate::progress::Stream; use crate::setup::{Outside, questions::ask_setting}; -use anyhow::{Context, Result, anyhow, bail}; -use std::io::{Read, Write}; -use std::os::unix::process::CommandExt; +use anyhow::{Context, Result, bail}; use std::path::Path; -use std::process::{Child, Command, Output, Stdio}; -use std::thread; -use std::time::Duration; +use std::process::Command; pub struct OpenCode; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::OpenCode, &OpenCode); @@ -100,7 +96,7 @@ impl Adapter for OpenCode { fn link_instruction_fallback(&self, worktree: &Path) -> Result<()> { super::instructions::link_fallback(worktree, "AGENTS.md", &[]) } - fn read_after_exit(&self, worktree: &Path, stream: Box) -> Box { + fn read_after_exit(&self, worktree: &Path, stream: Box) -> Result> { export::read_after_exit(worktree, stream) } } @@ -119,30 +115,17 @@ pub fn check_model_and_effort(chosen: &ModelAndEffort) -> Result<()> { }; crate::progress::step("checking OpenCode's Model and Effort with a standalone test call"); let invocation = OpenCode.session(&choice, None, "Reply with OK."); - let mut child = Command::new(OpenCode.name()) - .args(invocation.args) - .envs(OpenCode.environment().iter().copied()) - .process_group(0) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("could not run opencode to check the Model and Effort")?; - let written = child - .stdin - .take() - .context("no stdin from opencode")? - .write_all(invocation.stdin.as_deref().unwrap_or("").as_bytes()); - if let Err(error) = written { - OpenCode.stop(&mut child); - return Err(error).context("could not write the check prompt to opencode"); - } - let output = check_output(&mut child)?; + let output = super::process_output::output( + &OpenCode, + Command::new(OpenCode.name()).args(invocation.args), + invocation.stdin.as_deref(), + ) + .context("could not run opencode to check the Model and Effort")?; let mut stream = OpenCode.stream(Path::new("."), "Reply with OK."); for line in String::from_utf8_lossy(&output.stdout).lines() { stream.condense(line); } - let stream = OpenCode.read_after_exit(Path::new("."), stream); + let stream = OpenCode.read_after_exit(Path::new("."), stream)?; if !output.status.success() || stream.failed() { bail!( "opencode refused the Model or Effort in its test call: {}", @@ -160,44 +143,6 @@ fn error_text(error: &serde_json::Value) -> Option<&str> { error.as_str().or_else(|| error["message"].as_str()) } -/// Drain both pipes while watching for interruption, so a slow check cannot -/// strand its private server or commands before a Run has even started work. -fn check_output(child: &mut Child) -> Result { - fn read(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle>> { - thread::spawn(move || { - let mut bytes = Vec::new(); - pipe.read_to_end(&mut bytes)?; - Ok(bytes) - }) - } - let stdout = read(child.stdout.take().context("no stdout from opencode")?); - let stderr = read(child.stderr.take().context("no stderr from opencode")?); - let status = loop { - if crate::interrupt::requested() { - OpenCode.stop(child); - let _ = stdout.join(); - let _ = stderr.join(); - bail!("interrupted"); - } - if let Some(status) = child - .try_wait() - .context("could not wait for opencode's test call")? - { - break status; - } - thread::sleep(Duration::from_millis(100)); - }; - Ok(Output { - status, - stdout: stdout - .join() - .map_err(|_| anyhow!("OpenCode's check stdout reader panicked"))??, - stderr: stderr - .join() - .map_err(|_| anyhow!("OpenCode's check stderr reader panicked"))??, - }) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/harness/opencode/export.rs b/src/harness/opencode/export.rs index b44d83df5..e82a4b004 100644 --- a/src/harness/opencode/export.rs +++ b/src/harness/opencode/export.rs @@ -6,7 +6,7 @@ use crate::progress::Stream; use anyhow::{Result, anyhow}; use serde_json::Value; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::Command; pub struct SessionExport { message: Option, @@ -98,21 +98,26 @@ impl SessionExport { } } -pub fn read_after_exit(worktree: &Path, stream: Box) -> Box { - let export = stream.session_id().and_then(|id| { - let output = Command::new(OpenCode.name()) - .args(["session", "export", "--standalone", id]) - .envs(OpenCode.environment().iter().copied()) - .current_dir(worktree) - .stdin(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; +pub fn read_after_exit(worktree: &Path, stream: Box) -> Result> { + let export = if let Some(id) = stream.session_id() { + let output = super::super::process_output::output( + &OpenCode, + Command::new(OpenCode.name()) + .args(["session", "export", "--standalone", id]) + .current_dir(worktree), + None, + ); + match output { + Ok(output) if output.status.success() => { + SessionExport::parse(&String::from_utf8_lossy(&output.stdout)).ok() + } + Err(error) if crate::interrupt::requested() => return Err(error), + _ => None, } - SessionExport::parse(&String::from_utf8_lossy(&output.stdout)).ok() - }); - Box::new(AfterExit { stream, export }) + } else { + None + }; + Ok(Box::new(AfterExit { stream, export })) } struct AfterExit { diff --git a/src/harness/process_output.rs b/src/harness/process_output.rs new file mode 100644 index 000000000..2a9c301a1 --- /dev/null +++ b/src/harness/process_output.rs @@ -0,0 +1,84 @@ +//! Capture a Harness check or export while retaining session interruption rules. + +use std::io::{Read, Write}; +use std::os::unix::process::CommandExt; +use std::process::{Command, Output, Stdio}; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; + +use super::Adapter; + +/// Isolate the CLI's process group and drain both pipes while watching for +/// interruption. Checks and exports use the same fixed environment and +/// whole-process-tree stop as the adapter's live sessions. +pub(super) fn output( + adapter: &dyn Adapter, + command: &mut Command, + input: Option<&str>, +) -> Result { + if crate::interrupt::requested() { + bail!("interrupted"); + } + let cli = adapter.name(); + let mut child = command + .envs(adapter.environment().iter().copied()) + .process_group(0) + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("could not run {cli}"))?; + + fn read(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle>> { + thread::spawn(move || { + let mut bytes = Vec::new(); + pipe.read_to_end(&mut bytes)?; + Ok(bytes) + }) + } + let stdout = read(child.stdout.take().context("no captured stdout")?); + let stderr = read(child.stderr.take().context("no captured stderr")?); + if let Some(input) = input + && let Some(mut stdin) = child.stdin.take() + && let Err(error) = stdin.write_all(input.as_bytes()) + { + adapter.stop(&mut child); + let _ = stdout.join(); + let _ = stderr.join(); + return Err(error).with_context(|| format!("could not write the prompt to {cli}")); + } + let status = loop { + if crate::interrupt::requested() { + adapter.stop(&mut child); + let _ = stdout.join(); + let _ = stderr.join(); + bail!("interrupted"); + } + if let Some(status) = child + .try_wait() + .with_context(|| format!("could not wait for {cli}"))? + // A child command can retain either pipe after the CLI exits. + // Keep watching interruptions until both readers can be joined. + && stdout.is_finished() + && stderr.is_finished() + { + break status; + } + thread::sleep(Duration::from_millis(100)); + }; + Ok(Output { + status, + stdout: stdout + .join() + .map_err(|_| anyhow!("{cli}'s stdout reader panicked"))??, + stderr: stderr + .join() + .map_err(|_| anyhow!("{cli}'s stderr reader panicked"))??, + }) +} diff --git a/src/session.rs b/src/session.rs index b3240a183..107d94d07 100644 --- a/src/session.rs +++ b/src/session.rs @@ -353,7 +353,7 @@ fn run( .join() .map_err(|_| anyhow!("the session stream reader panicked"))?; - let stream = adapter.read_after_exit(worktree, stream); + let stream = adapter.read_after_exit(worktree, stream)?; for warning in stream.warnings() { progress::step(format_args!("{kind}: {warning}")); } diff --git a/tests/fakes.rs b/tests/fakes.rs index f2ea69807..22f7914b5 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -32,12 +32,43 @@ mod opencode; use std::fs::{self, File}; use std::io::Write; +use std::os::fd::AsFd; +use std::os::unix::fs::MetadataExt; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use json::{Array, Json}; +/// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. +fn stdin_is_null() -> bool { + let stdin = std::io::stdin() + .as_fd() + .try_clone_to_owned() + .map(fs::File::from) + .and_then(|file| file.metadata()); + let (Ok(stdin), Ok(null)) = (stdin, fs::metadata("/dev/null")) else { + return false; + }; + stdin.rdev() == null.rdev() && stdin.ino() == null.ino() +} + +/// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts +/// them until its script ends. A caught signal, unlike an ignored one, is +/// back to its default in the script, which can trap it. +fn outlast_interrupts() { + // From the C library, which the standard library links: SIGINT and + // SIGTERM are 2 and 15 on Linux and macOS alike. + unsafe extern "C" { + fn signal(signum: i32, handler: extern "C" fn(i32)) -> usize; + } + extern "C" fn caught(_: i32) {} + for signum in [2, 15] { + // SAFETY: the handler does nothing, so it is async-signal-safe. + unsafe { signal(signum, caught) }; + } +} + /// Print `message` to stderr and exit with `code`, as the fakes fail. fn die(message: &str, code: i32) -> ! { eprintln!("{message}"); diff --git a/tests/fakes/agy.rs b/tests/fakes/agy.rs index 84d38d095..703e62fb2 100644 --- a/tests/fakes/agy.rs +++ b/tests/fakes/agy.rs @@ -5,8 +5,8 @@ use std::io::Write; use std::process::Command; use crate::claude::{exit_code, git_here, script_for, skills_snapshot}; -use crate::codex::stdin_is_null; use crate::json::{Array, Bool, Json, Null, number, object, string}; +use crate::stdin_is_null; const CATALOG: &str = "gemini-3.8-flash-high\tGemini 3.8 Flash (High)\ngemini-3.8-flash-medium\tGemini 3.8 Flash (Medium)\ngemini-3.8-flash-low\tGemini 3.8 Flash (Low)\ngemini-3.1-pro-high\tGemini 3.1 Pro (High)\ngemini-3.1-pro-low\tGemini 3.1 Pro (Low)\n"; diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 3fb7c7a9e..cd09ebe40 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -27,12 +27,11 @@ use std::fs; use std::io::Write; -use std::os::fd::AsFd; -use std::os::unix::fs::MetadataExt; use std::process::Command; use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; use crate::json::{Array, Bool, Json, Null, object, string}; +use crate::{outlast_interrupts, stdin_is_null}; /// Where Codex finds a worktree's project skills. const SKILLS: &str = ".agents/skills"; @@ -48,35 +47,6 @@ const CATALOG: &str = r#"{"models": [ "supported_reasoning_levels": [{"effort": "low"}, {"effort": "medium"}, {"effort": "high"}, {"effort": "xhigh"}]} ]}"#; -/// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. -pub fn stdin_is_null() -> bool { - let stdin = std::io::stdin() - .as_fd() - .try_clone_to_owned() - .map(fs::File::from) - .and_then(|file| file.metadata()); - let (Ok(stdin), Ok(null)) = (stdin, fs::metadata("/dev/null")) else { - return false; - }; - stdin.rdev() == null.rdev() && stdin.ino() == null.ino() -} - -/// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts -/// them until its script ends. A caught signal, unlike an ignored one, is -/// back to its default in the script, which can trap it. -pub fn outlast_interrupts() { - // From the C library, which the standard library links: SIGINT and - // SIGTERM are 2 and 15 on Linux and macOS alike. - unsafe extern "C" { - fn signal(signum: i32, handler: extern "C" fn(i32)) -> usize; - } - extern "C" fn caught(_: i32) {} - for signum in [2, 15] { - // SAFETY: the handler does nothing, so it is async-signal-safe. - unsafe { signal(signum, caught) }; - } -} - /// Print one line of the stream. fn emit(event: Json) { println!("{event}"); diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs index 76b7b6d3a..f24ccf1f3 100644 --- a/tests/fakes/grok.rs +++ b/tests/fakes/grok.rs @@ -5,8 +5,8 @@ use std::io::Write; use std::process::Command; use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; -use crate::codex::{outlast_interrupts, stdin_is_null}; use crate::json::{Array, Bool, Null, object, string}; +use crate::{outlast_interrupts, stdin_is_null}; pub const MODELS: &str = include_str!("../fixtures/grok-models.txt"); const CACHE: &str = include_str!("../fixtures/grok-models.json"); diff --git a/tests/fakes/muse.rs b/tests/fakes/muse.rs index ed2d0d806..0af3f8b94 100644 --- a/tests/fakes/muse.rs +++ b/tests/fakes/muse.rs @@ -2,45 +2,15 @@ use std::fs; use std::io::Write; -use std::os::fd::AsFd; -use std::os::unix::fs::MetadataExt; use std::process::Command; use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; use crate::json::{Array, Bool, Json, Null, object, string}; +use crate::{outlast_interrupts, stdin_is_null}; /// Where Muse finds a worktree's project skills. const SKILLS: &str = ".agents/skills"; -/// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. -pub(crate) fn stdin_is_null() -> bool { - let stdin = std::io::stdin() - .as_fd() - .try_clone_to_owned() - .map(fs::File::from) - .and_then(|file| file.metadata()); - let (Ok(stdin), Ok(null)) = (stdin, fs::metadata("/dev/null")) else { - return false; - }; - stdin.rdev() == null.rdev() && stdin.ino() == null.ino() -} - -/// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts -/// them until its script ends. A caught signal, unlike an ignored one, is -/// back to its default in the script, which can trap it. -pub(crate) fn outlast_interrupts() { - // From the C library, which the standard library links: SIGINT and - // SIGTERM are 2 and 15 on Linux and macOS alike. - unsafe extern "C" { - fn signal(signum: i32, handler: extern "C" fn(i32)) -> usize; - } - extern "C" fn caught(_: i32) {} - for signum in [2, 15] { - // SAFETY: the handler does nothing, so it is async-signal-safe. - unsafe { signal(signum, caught) }; - } -} - /// Print one line of the stream. pub(crate) fn emit(event: Json) { println!("{event}"); @@ -82,6 +52,22 @@ pub fn main(argv: Vec) { ); crate::exit(1); } + if let Ok(script) = std::env::var("FAKE_MUSE_CHECK_SCRIPT") { + outlast_interrupts(); + let mut child = Command::new("bash") + .arg("-e") + .arg(script) + .env("FAKE_MUSE_CHECK_PID", std::process::id().to_string()) + .spawn() + .unwrap(); + if std::env::var("FAKE_MUSE_CHECK_EXIT_EARLY").is_ok() { + crate::exit(0); + } + let status = child.wait().unwrap(); + if !status.success() { + crate::exit(exit_code(status)); + } + } emit_event( "fake-check", "run.terminal.completed", diff --git a/tests/fakes/opencode.rs b/tests/fakes/opencode.rs index 610042f84..5640c84cc 100644 --- a/tests/fakes/opencode.rs +++ b/tests/fakes/opencode.rs @@ -1,7 +1,8 @@ //! OpenCode's JSONL stream and standalone session export, with stdin prompts. use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; use crate::json::{Array, Json, Null, number, object, string}; -use crate::muse::{emit, outlast_interrupts}; +use crate::muse::emit; +use crate::outlast_interrupts; use std::fs; use std::io::Read; use std::process::Command; @@ -27,6 +28,15 @@ pub fn main(argv: Vec) { println!("truncated {{"); crate::exit(0); } + if argv.last().is_some_and(|id| id != "check-session") + && let Ok(script) = std::env::var("FAKE_OPENCODE_EXPORT_SCRIPT") + { + outlast_interrupts(); + let status = Command::new("bash").arg("-e").arg(script).status().unwrap(); + if !status.success() { + crate::exit(exit_code(status)); + } + } println!( "{}", fs::read_to_string(export_path(argv.last().unwrap())).unwrap() diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 2c14c2e83..6af9d5a38 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -9,9 +9,16 @@ use std::time::{Duration, Instant}; use support::Scenario; /// Clean up by the recorded pid even when the regression assertion fails. -struct DetachedCommand(libc::pid_t); +struct SessionCommand(libc::pid_t); -impl DetachedCommand { +enum During { + Session, + Check, + CheckAfterExit, + Export, +} + +impl SessionCommand { fn exists(&self) -> bool { // SAFETY: kill has no memory-safety preconditions; signal 0 only probes. unsafe { libc::kill(self.0, 0) == 0 } @@ -26,95 +33,150 @@ impl DetachedCommand { } } -impl Drop for DetachedCommand { +impl Drop for SessionCommand { fn drop(&mut self) { if self.exists() { - // SAFETY: the recorded pid is this command's process group leader. - unsafe { libc::kill(-self.0, libc::SIGKILL) }; + // SAFETY: the recorded pid belongs to the command. Stop its group + // when it made one, and the command itself when it inherited one. + unsafe { + libc::kill(-self.0, libc::SIGKILL); + libc::kill(self.0, libc::SIGKILL); + } } } } #[test] fn interrupting_a_claude_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("claude", false); + assert_session_command_stopped("claude", During::Session); } #[test] fn interrupting_a_codex_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("codex", false); + assert_session_command_stopped("codex", During::Session); } #[test] fn interrupting_an_agy_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("agy", false); + assert_session_command_stopped("agy", During::Session); } #[test] fn interrupting_an_opencode_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("opencode", false); + assert_session_command_stopped("opencode", During::Session); } #[test] fn interrupting_an_opencode_check_stops_its_whole_process_tree_before_work() { - assert_detached_command_stopped("opencode", true); + assert_session_command_stopped("opencode", During::Check); +} + +#[test] +fn interrupting_a_muse_check_stops_its_whole_process_tree_before_work() { + assert_session_command_stopped("muse", During::Check); +} + +#[test] +fn interrupting_a_muse_check_after_cli_exit_stops_the_command_holding_its_stream() { + assert_session_command_stopped("muse", During::CheckAfterExit); +} + +#[test] +fn interrupting_an_opencode_export_stops_its_whole_process_tree() { + assert_session_command_stopped("opencode", During::Export); } #[test] fn interrupting_a_grok_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("grok", false); + assert_session_command_stopped("grok", During::Session); } #[test] fn interrupting_a_muse_run_stops_a_command_in_its_own_session() { - assert_detached_command_stopped("muse", false); + assert_session_command_stopped("muse", During::Session); } -fn assert_detached_command_stopped(harness: &str, checking: bool) { +fn assert_session_command_stopped(harness: &str, during: During) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); - let ending = if checking { - "sleep 9" - } else { + let checking = matches!(during, During::Check | During::CheckAfterExit); + let ending = if matches!(during, During::Session) { "while :; do sleep 0.1; done" + } else { + "sleep 9" }; - scenario.agent_does(&format!( - r#"detached-command bash -c 'trap "" INT TERM; echo $$ > "{pid}"; exec sleep 120' >/dev/null 2>&1 & + let interrupt_script = if matches!(during, During::CheckAfterExit) { + // Wait for the recorded CLI to exit so interruption arrives while + // only this command is keeping the captured output pipes open. + format!( + r#"while kill -0 "$FAKE_MUSE_CHECK_PID" 2>/dev/null; do sleep 0.01; done +echo $$ > "{pid}" +touch "{started}" +exec sleep 9 +"#, + pid = pid.display(), + started = scenario.path("agent-started").display(), + ) + } else { + format!( + r#"detached-command bash -c 'trap "" INT TERM; echo $$ > "{pid}"; exec sleep 120' >/dev/null 2>&1 & while ! test -s "{pid}"; do sleep 0.01; done touch "{started}" {ending} "#, - pid = pid.display(), - started = scenario.path("agent-started").display(), - )); + pid = pid.display(), + started = scenario.path("agent-started").display(), + ) + }; - let script = scenario.path("claude-script.sh"); - let env = if checking { - vec![("FAKE_OPENCODE_CHECK_SCRIPT", script.to_str().unwrap())] - } else { - Vec::new() + let script = scenario.path("interrupt-script.sh"); + fs::write(&script, &interrupt_script).unwrap(); + let env = match during { + During::Session => { + scenario.agent_does(&interrupt_script); + Vec::new() + } + During::Check | During::CheckAfterExit => { + let name = if harness == "muse" { + "FAKE_MUSE_CHECK_SCRIPT" + } else { + "FAKE_OPENCODE_CHECK_SCRIPT" + }; + let mut env = vec![(name, script.to_str().unwrap())]; + if matches!(during, During::CheckAfterExit) { + env.push(("FAKE_MUSE_CHECK_EXIT_EARLY", "1")); + } + env + } + During::Export => { + scenario.agent_does("true"); + vec![("FAKE_OPENCODE_EXPORT_SCRIPT", script.to_str().unwrap())] + } }; - let mut held = scenario.run_until( - &["harness", harness, &scenario.issue_url(7)], - &env, - "agent-started", - ); - let command = DetachedCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); - assert!(command.exists(), "the detached command never started"); + let url = scenario.issue_url(7); + let mut args = vec!["harness", harness]; + if checking && harness == "muse" { + args.extend(["model", "muse-spark-1.3"]); + } + args.push(&url); + let mut held = scenario.run_until(&args, &env, "agent-started"); + let command = SessionCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); + assert!(command.exists(), "the command never started"); let interrupted_at = Instant::now(); held.signal("INT"); let result = held.finish(); assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!(result.stderr.contains("interrupted"), "{}", result.stderr); if checking { assert!(scenario.issue_labels(7).is_empty()); assert_eq!(scenario.entries("work"), ["widgets"]); } assert!( interrupted_at.elapsed() < Duration::from_secs(8), - "a session that exited on its first signal incurred another grace period" + "interruption waited for the command instead of stopping it" ); - command.assert_stopped("the detached command survived interruption"); + command.assert_stopped("the session's command survived interruption"); } #[test] @@ -142,7 +204,7 @@ while :; do sleep 0.1 || :; done "agent-started", "INT", ); - let command = DetachedCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); + let command = SessionCommand(fs::read_to_string(pid).unwrap().trim().parse().unwrap()); assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); assert_eq!(fs::read_to_string(signals).unwrap(), "INT\nTERM\n");