diff --git a/README.md b/README.md index 2c308ccbe..80cdc0e5d 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude, codex, agy or muse; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex, agy, grok or muse; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -264,6 +264,10 @@ effort = "" # how hard that Model reasons; default blank, for Codex's own model = "" # the Model Antigravity CLI's sessions run on; default blank, for agy's own effort = "" # how hard that Model reasons; default blank, for agy's own +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own + [harness.muse] model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own effort = "" # how hard that Model reasons; default blank, for Muse Code's own @@ -271,7 +275,7 @@ effort = "" # how hard that Model reasons; default blank, for Muse Code's own Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing only those installed on `PATH`. An unavailable Harness or unknown name is refused and asked again. The default is the configured Harness when installed, else the first installed, starting with `claude`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For agy, Setup lists `agy models` and its supported Efforts, checks each answer without a model turn, and writes the canonical names regardless of case. Muse proposes `muse-spark-1.3` and checks the answers against its cache, or with a minimal call; a refusal asks for the Model and Effort again. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing only those installed on `PATH`. An unavailable Harness or unknown name is refused and asked again. The default is the configured Harness when installed, else the first installed in `claude`, `codex`, `agy`, `grok`, `muse` order. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For agy, Setup lists `agy models` and its supported Efforts, checks each answer without a model turn, and writes the canonical names regardless of case. For Grok Build, Setup lists `grok models` and the chosen Model's Efforts from its refreshed cache, and checks each answer against them. Muse proposes `muse-spark-1.3` and checks the answers against its cache, or with a minimal call; a refusal asks for the Model and Effort again. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -302,7 +306,7 @@ A Run reads the file before any work. One that isn't valid TOML, or that has a k ### Harness, Model and Effort -The Harnesses are Claude Code (`claude`), Codex (`codex`), Antigravity CLI (`agy`) and Muse Code (`muse`). +The available Harnesses are Claude Code (`claude`), Codex (`codex`), Antigravity CLI (`agy`), Grok Build (`grok`) and Muse Code (`muse`). Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: @@ -310,7 +314,7 @@ Every session of a command, its implement session, each Repair and Resume, a Spe thirdshift https://github.com///issues/ model claude-opus-5-5 effort high ``` -For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. +For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Grok Build takes `-m ` and `--reasoning-effort `. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. @@ -320,9 +324,13 @@ agy's `init` supplies the conversation id and `step_update` supplies tool progre Before any work on agy, a named Model or Effort is checked against the free `agy models` catalog, which lists IDs, labels and effort-suffixed variants. IDs, display labels and bare aliases are matched regardless of case, as are Efforts. `model Gemini-3.8-Flash effort Medium` becomes `--model gemini-3.8-flash --effort medium`; `model gemini-3.8-flash-medium` already carries its Effort. A bare alias requires an Effort; a Model or Effort the catalog lacks fails naming valid choices. This check uses null stdin and disables self-update too, without a model turn. +With `harness grok`, every session runs `grok -p --always-approve --sandbox off --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag and disabled sandbox are always explicit, overriding the CLI’s own config and `GROK_SANDBOX`. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok documents no stream notification proving that background work was killed at exit, so thirdshift does not infer an automatic Resume from its tool activity. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. + +Grok reads both `AGENTS.md` and `CLAUDE.md` when both exist, along with `AGENT.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md` and project rules under `.grok`, `.claude` and `.cursor`. It also reads `~/.grok/rules`, `~/.claude/CLAUDE.md` and `~/.claude/rules`; thirdshift writes none of these user-level files. Grok Build uses `~/.grok` for sign-in and sessions (or `GROK_HOME` when set). + Interrupting a Command stops every session's process tree on every Harness, including commands that started their own process group or session. thirdshift snapshots the descendants with `ps` on Linux and macOS, sends each process group the Harness's stop signals with a 10-second grace period each, then takes another snapshot before SIGKILL to catch commands started in the meantime. If the CLI exits on an earlier signal, thirdshift kills the remaining commands immediately without another grace period. -Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. +Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. On Grok Build, `grok models` lists the available Models without a model call and refreshes `models_cache.json` under `GROK_HOME` or `~/.grok`; thirdshift reads that cache for each Model's supported Efforts. Both are matched regardless of case and passed as Grok names them. With no Model named, a named Effort is checked against Grok's current default Model. Unknown Models and unsupported Efforts fail with the valid choices, and an unreadable catalog fails before work. The check has the same null stdin and fixed environment as sessions. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. With `harness muse`, every session and Resume runs `muse exec --json --yolo`, with stdin null and `MUSE_NO_AUTO_UPDATE=1`. `--yolo` disables approvals and the sandbox and trusts the worktree ([ADR-0013](docs/adr/0013-every-harness-runs-unattended-and-fully-trusted.md)). Muse reads the project's `AGENTS.md`, falling back to `CLAUDE.md`, and also reads `~/.claude/CLAUDE.md`. The Factory skills are linked into `.agents/skills/` and kept out of git. The prompt's first line tells the model to load its `thirdshift-` with its skill tool. If the stream shows no `read_skill` load of that skill, thirdshift writes a warning to the progress lines and Command log; it neither fails nor retries the session. @@ -886,7 +894,7 @@ Editing a skill in `skills/` has no effect until you rebuild and reinstall ([ADR The [Prompts and skills page](https://thirdshift.app/prompts/) is generated from the prompts, the `claude` arguments and the skills, and each Session prompt is also generated as a Markdown file in [`prompts/`](prompts/), with a [`README.md`](prompts/README.md) that lists them with the `claude` command lines. `cargo test` fails until they are regenerated after a change to any of them. Regenerate the page and `prompts/` with `UPDATE_PROMPTS=1 cargo test prompts_page`, which also deletes any file in `prompts/` that it doesn't generate. -The integration tests swap in a fake `gh` and `claude`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. +The integration tests swap in a fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. ### The Rust version diff --git a/src/args.rs b/src/args.rs index 78d045a76..aa7acf88b 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex or agy or muse", + "harness must be followed by claude or codex or agy or grok or muse", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex or agy or muse, not gemini", + "harness must be followed by claude or codex or agy or grok or muse, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/asks.rs b/src/asks.rs index 6eeafe446..c03951ddf 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -262,7 +262,7 @@ mod tests { default: Some(Harness::Codex), claude: model_and_effort("opus", "high"), codex: model_and_effort("gpt-6.1-sol", "max"), - ..Default::default() + ..harness::Settings::default() }, ..no_settings() } diff --git a/src/config.rs b/src/config.rs index 3daf7c57b..24b16d178 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude, codex, agy or muse; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex, agy, grok or muse; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -401,6 +401,10 @@ effort = "" # how hard that Model reasons; default blank, for Codex's own model = "" # the Model Antigravity CLI's sessions run on; default blank, for agy's own effort = "" # how hard that Model reasons; default blank, for agy's own +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own + [harness.muse] model = "" # the Model Muse Code's sessions run on; default blank, for Muse Code's own effort = "" # how hard that Model reasons; default blank, for Muse Code's own @@ -538,6 +542,8 @@ mod tests { "harness.codex.effort", "harness.agy.model", "harness.agy.effort", + "harness.grok.model", + "harness.grok.effort", "harness.muse.model", "harness.muse.effort" ] @@ -589,7 +595,7 @@ mod tests { model: Some("gpt-6.1-sol".to_string()), effort: Some("max".to_string()), }, - ..Default::default() + ..harness::Settings::default() } ); assert_eq!(parse("").unwrap().harness, harness::Settings::default()); @@ -600,7 +606,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex or agy or muse, not \"gemini\"", + "harness.default must be claude or codex or agy or grok or muse, not \"gemini\"", ), ( "[harness]\ndefault = true\n", @@ -881,7 +887,7 @@ mod tests { let config: toml::Table = completed.parse().unwrap(); let harness = config["harness"].as_table().unwrap(); assert!(harness.contains_key("default"), "{text:?}:\n{completed}"); - for name in ["claude", "codex"] { + for name in ["claude", "codex", "agy", "grok"] { let settings = harness[name].as_table().unwrap(); for key in ["model", "effort"] { assert!(settings.contains_key(key), "{text:?}:\n{completed}"); diff --git a/src/harness.rs b/src/harness.rs index 7e7cc573a..c70379066 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -13,6 +13,7 @@ mod adapter; pub(crate) mod agy; pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod grok; mod instructions; pub(crate) mod muse; mod process_tree; @@ -26,6 +27,7 @@ pub enum Harness { Claude, Codex, Agy, + Grok, Muse, } @@ -98,6 +100,8 @@ pub struct Settings { pub codex: ModelAndEffort, /// `[harness.agy]`. pub agy: ModelAndEffort, + /// `[harness.grok]`. + pub grok: ModelAndEffort, /// `[harness.muse]`. pub muse: ModelAndEffort, } @@ -286,7 +290,7 @@ mod tests { default: Some(Harness::Codex), claude: set("opus", "high"), codex: set("gpt-6.1-sol", "max"), - ..Default::default() + ..Settings::default() } } diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 858af7083..18c6e7a81 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -5,7 +5,7 @@ use std::process::Child; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, agy, claude, codex, muse}; +use super::{Choice, Harness, ModelAndEffort, Settings, agy, claude, codex, grok, muse}; use crate::progress::Stream; use crate::setup::Outside; @@ -90,9 +90,10 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 4] = [ +pub const REGISTERED: [(Harness, &dyn Adapter); 5] = [ claude::REGISTRATION, codex::REGISTRATION, agy::REGISTRATION, + grok::REGISTRATION, muse::REGISTRATION, ]; diff --git a/src/harness/codex.rs b/src/harness/codex.rs index e46782693..1ccb63400 100644 --- a/src/harness/codex.rs +++ b/src/harness/codex.rs @@ -11,7 +11,7 @@ use anyhow::Result; use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings}; use crate::progress::{self, Stream}; -use crate::setup::{Outside, questions::ask_setting}; +use crate::setup::{Outside, questions::ask_checked_setting}; pub struct Codex; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Codex, &Codex); @@ -135,13 +135,13 @@ fn ask_codex( Codex.product_name(), catalog.listed().join(", ") )); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Codex, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; + let model = ask_checked_setting( + outside, + "Model", + Harness::Codex, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; let efforts = catalog.efforts(model.as_deref()).join(", "); outside.say(match &model { Some(model) => format!("Efforts {model} supports: {efforts}"), @@ -150,13 +150,18 @@ fn ask_codex( Codex.product_name() ), }); - loop { - let effort = ask_setting(outside, "Effort", Harness::Codex, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(settled), - Err(error) => outside.say(format!("{error:#}")), - } - } + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Codex, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(ModelAndEffort { model, effort }) } /// `prompt` as Codex takes it: a first line that loads a Factory skill, diff --git a/src/harness/grok.rs b/src/harness/grok.rs new file mode 100644 index 000000000..56e069822 --- /dev/null +++ b/src/harness/grok.rs @@ -0,0 +1,168 @@ +//! Grok Build: unattended sessions and its Claude-shaped Messages stream. + +mod catalog; +mod stream; +pub use catalog::Catalog; + +use std::path::Path; + +use anyhow::Result; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings}; +use crate::progress::Stream; +use crate::setup::{Outside, questions::ask_checked_setting}; + +pub struct Grok; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Grok, &Grok); + +impl Adapter for Grok { + fn name(&self) -> &'static str { + "grok" + } + fn product_name(&self) -> &'static str { + "Grok Build" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Slash + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[ + ("GROK_DISABLE_AUTOUPDATER", "1"), + ("GROK_FOLDER_TRUST", "0"), + ] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.grok + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.grok + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args: Vec = [ + "--always-approve", + "--sandbox", + "off", + "--output-format", + "streaming-messages-json", + ] + .map(String::from) + .to_vec(); + if let Some(model) = &choice.model { + args.extend(["-m".to_string(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend(["--reasoning-effort".to_string(), effort.clone()]); + } + if let Some(id) = resume { + args.extend(["-r".to_string(), id.to_string()]); + } + args.extend(["-p".to_string(), self.skill_loading().prompt(prompt)]); + Invocation { args, stdin: None } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + crate::progress::step("checking the Model and Effort against grok models"); + let settled = Catalog::read()?.settle(choice.model.as_deref(), choice.effort.as_deref())?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + let catalog = match outside.grok_catalog() { + Ok(catalog) => catalog, + Err(error) => { + outside.say(format!("{error:#}\nThe harness settings stay as they are; rerun `thirdshift setup` once grok models works.")); + return Ok(None); + } + }; + outside.say(format!( + "{}'s Models: {}", + self.product_name(), + catalog.listed().join(", ") + )); + let model = ask_checked_setting( + outside, + "Model", + Harness::Grok, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; + outside.say(format!( + "Efforts {} supports: {}", + model.as_deref().unwrap_or("Grok Build's default Model"), + catalog.efforts(model.as_deref()).join(", ") + )); + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Grok, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(Some(ModelAndEffort { model, effort })) + } + fn stream(&self, _worktree: &Path, _prompt: &str) -> Box { + Box::new(stream::GrokProgress::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::harness::ChosenBy; + + #[test] + fn a_resume_uses_a_streamed_id_with_the_same_flags_null_stdin_and_fixed_environment() { + let mut progress = stream::GrokProgress::default(); + progress + .condense(r#"{"type":"system","subtype":"init","session_id":"abc123","cwd":"/repo"}"#); + let choice = Choice { + harness: Harness::Grok, + model: Some("grok-4.7".to_string()), + effort: Some("low".to_string()), + chosen_by: ChosenBy::Command, + }; + let adapter = choice.harness.adapter(); + let resume = adapter.session(&choice, progress.session_id(), "Continue."); + assert_eq!( + resume.args, + [ + "--always-approve", + "--sandbox", + "off", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "low", + "-r", + "abc123", + "-p", + "Continue." + ] + ); + assert_eq!(resume.stdin, None); + assert_eq!( + adapter.environment(), + [ + ("GROK_DISABLE_AUTOUPDATER", "1"), + ("GROK_FOLDER_TRUST", "0") + ] + ); + } +} diff --git a/src/harness/grok/catalog.rs b/src/harness/grok/catalog.rs new file mode 100644 index 000000000..7712df051 --- /dev/null +++ b/src/harness/grok/catalog.rs @@ -0,0 +1,198 @@ +//! Grok's free model list and per-model Efforts from the cache it refreshes. + +use std::fs; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use serde_json::Value; + +use super::Grok; +use crate::harness::{Adapter, ModelAndEffort, said}; + +#[derive(Debug)] +struct Model { + name: String, + efforts: Vec, +} + +#[derive(Debug)] +pub struct Catalog { + models: Vec, + default: String, +} + +impl Catalog { + pub fn read() -> Result { + let output = Command::new(Grok.name()) + .arg("models") + .envs(Grok.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run grok models to read Grok Build's Models")?; + if !output.status.success() { + bail!( + "grok models failed, so Grok Build's Models can't be read: {}", + said(&output) + ); + } + // `grok models` prints names, but refreshes the effort metadata here. + let home = match std::env::var_os("GROK_HOME").filter(|home| !home.is_empty()) { + Some(home) => std::path::PathBuf::from(home), + None => crate::config::home()?.join(".grok"), + }; + let path = home.join("models_cache.json"); + let cache = fs::read_to_string(&path).with_context(|| { + format!( + "could not read Grok Build's Model and Effort catalog {} after grok models", + path.display() + ) + })?; + Self::parse(&String::from_utf8_lossy(&output.stdout), &cache) + } + + /// The text list and JSON cache recorded from Grok Build 1.0.46. + pub fn parse(list: &str, cache: &str) -> Result { + let cache: Value = + serde_json::from_str(cache).context("Grok Build's Model cache is not JSON")?; + let metadata = cache["models"] + .as_object() + .context("Grok Build's cache has no Models")?; + let default = list + .lines() + .find_map(|line| line.trim().strip_prefix("Default model: ")) + .context("grok models printed no default Model")? + .trim() + .to_string(); + let available = list + .split_once("Available models:") + .context("grok models printed no list of Models")? + .1; + let mut models = Vec::new(); + for name in available.lines().filter_map(|line| { + let line = line.trim(); + line.strip_prefix("* ") + .or_else(|| line.strip_prefix("- "))? + .split_whitespace() + .next() + }) { + let info = metadata.get(name).context(format!( + "Grok Build's cache has no metadata for Model {name}" + ))?; + let efforts = info["info"]["reasoning_efforts"] + .as_array() + .into_iter() + .flatten() + .filter_map(|effort| effort["value"].as_str()) + .map(String::from) + .collect(); + models.push(Model { + name: name.to_string(), + efforts, + }); + } + if models.is_empty() || !models.iter().any(|model| model.name == default) { + bail!("grok models printed no available default Model"); + } + Ok(Self { models, default }) + } + + pub fn listed(&self) -> Vec<&str> { + self.models + .iter() + .map(|model| model.name.as_str()) + .collect() + } + + fn model(&self, name: &str) -> Result<&Model> { + self.models + .iter() + .find(|model| model.name.eq_ignore_ascii_case(name)) + .with_context(|| { + format!( + "the Model {name} is not in Grok Build's catalog: choose one of {}", + self.listed().join(", ") + ) + }) + } + + pub fn efforts(&self, model: Option<&str>) -> Vec<&str> { + self.model(model.unwrap_or(&self.default)) + .map(|model| model.efforts.iter().map(String::as_str).collect()) + .unwrap_or_default() + } + + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { + let found = self.model(model.unwrap_or(&self.default))?; + let effort = match effort { + None => None, + Some(effort) => { + let Some(valid) = found + .efforts + .iter() + .find(|valid| valid.eq_ignore_ascii_case(effort)) + else { + bail!( + "the Effort {effort} is not one the Grok Build Model {} supports: choose one of {}", + found.name, + found.efforts.join(", ") + ); + }; + Some(valid.clone()) + } + }; + Ok(ModelAndEffort { + model: model.map(|_| found.name.clone()), + effort, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIST: &str = include_str!("../../../tests/fixtures/grok-models.txt"); + const CACHE: &str = include_str!("../../../tests/fixtures/grok-models.json"); + + #[test] + fn recorded_models_have_their_own_efforts_and_default_model_is_checked_when_no_model_is_set() { + let catalog = Catalog::parse(LIST, CACHE).unwrap(); + assert_eq!( + catalog.listed(), + ["grok-4.7", "grok-4.7-build-fast", "grok-4.6", "grok-4.5"] + ); + assert_eq!(catalog.efforts(Some("grok-4.5")), ["high", "medium", "low"]); + assert_eq!( + catalog.settle(None, Some("XHIGH")).unwrap(), + ModelAndEffort { + model: None, + effort: Some("xhigh".to_string()) + } + ); + let older_default = LIST.replace("Default model: grok-4.7", "Default model: grok-4.5"); + let older = Catalog::parse(&older_default, CACHE).unwrap(); + assert!( + older + .settle(None, Some("xhigh")) + .unwrap_err() + .to_string() + .contains("choose one of high, medium, low") + ); + } + + #[test] + fn an_unreadable_catalog_fails_instead_of_accepting_unchecked_settings() { + for (list, cache) in [ + ("", CACHE), + (LIST, "not json"), + (LIST, "{}"), + (LIST, "{\"models\": {}}"), + ( + "Default model: missing\nAvailable models:\n - missing", + CACHE, + ), + ] { + assert!(Catalog::parse(list, cache).is_err(), "{list:?} {cache:?}"); + } + } +} diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs new file mode 100644 index 000000000..e984b6f30 --- /dev/null +++ b/src/harness/grok/stream.rs @@ -0,0 +1,163 @@ +//! Grok's Messages stream shares Claude's framing, with different tool names. + +use serde_json::Value; + +use crate::harness::claude::stream::ClaudeProgress; +use crate::progress::Stream; + +#[derive(Default)] +pub struct GrokProgress { + claude: ClaudeProgress, + failed: bool, + error: Option, + session_id: Option, + usage: Option, + cost_known: bool, +} + +impl Stream for GrokProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(mut event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if let Some(id) = event["session_id"].as_str() { + self.session_id.get_or_insert_with(|| id.to_string()); + } + match event["type"].as_str() { + Some("error") => { + self.failed = true; + self.error = event["message"].as_str().map(String::from); + } + Some("result") => { + // Messages streams use zero as a placeholder for unknown cost. + self.cost_known = event["total_cost_usd"] + .as_f64() + .is_some_and(|cost| cost > 0.0); + self.usage = None; + self.failed = event["is_error"] == true || event["subtype"] != "success"; + let errors: Vec<_> = event["errors"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + if !errors.is_empty() { + self.error = Some(errors.join("\n")); + } else if self.failed + && let Some(text) = event["result"].as_str().filter(|text| !text.is_empty()) + { + self.error = Some(text.to_string()); + } + if let (Some(input), Some(output)) = ( + event["usage"]["input_tokens"].as_u64(), + event["usage"]["output_tokens"].as_u64(), + ) { + let cached = event["usage"]["cache_read_input_tokens"] + .as_u64() + .unwrap_or(0); + let created = event["usage"]["cache_creation_input_tokens"] + .as_u64() + .unwrap_or(0); + self.usage = Some(format!( + "{input} input tokens, {cached} cache read tokens, {created} cache creation tokens, {output} output tokens" + )); + } + } + _ => {} + } + if let Some(content) = event + .get_mut("message") + .and_then(|message| message.get_mut("content")) + .and_then(Value::as_array_mut) + { + for block in content + .iter_mut() + .filter(|block| block["type"] == "tool_use") + { + if matches!( + block["name"].as_str(), + Some("bash" | "run_terminal_command") + ) { + block["name"] = Value::String("Bash".to_string()); + } + if let Some(path) = block["input"]["path"].as_str().map(String::from) { + block["input"]["file_path"] = Value::String(path); + } + } + } + self.claude.condense(&event.to_string()) + } + fn summary(&self) -> Option { + if self.cost_known { + self.claude.summary().or_else(|| self.usage.clone()) + } else { + self.usage.clone() + } + } + fn final_message(&self) -> Option<&str> { + self.claude.final_message() + } + fn session_id(&self) -> Option<&str> { + self.claude.session_id().or(self.session_id.as_deref()) + } + fn killed_background_work(&self) -> Vec<&str> { + // Grok documents no Messages frame proving work was killed at exit. + Vec::new() + } + fn failed(&self) -> bool { + self.failed + } + fn error(&self) -> Option<&str> { + self.error.as_deref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn messages_frames_reuse_claude_progress_and_keep_groks_id_final_text_and_spend() { + let mut stream = GrokProgress::default(); + assert_eq!(stream.condense(r#"{"type":"system","subtype":"init","session_id":"abc123","apiKeySource":"oauth","cwd":"/repo"}"#), ["session started"]); + assert_eq!(stream.condense(r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"read_file","input":{"path":"/repo/src/main.rs"}},{"type":"tool_use","name":"bash","input":{"command":"cargo test"}}]}}"#), ["read_file src/main.rs", "$ cargo test"]); + stream.condense(r#"{"type":"result","subtype":"success","is_error":false,"num_turns":7,"result":"Here's a summary...","total_cost_usd":0.0127,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":0,"cache_creation_input_tokens":0},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!(stream.final_message(), Some("Here's a summary...")); + assert_eq!(stream.summary().as_deref(), Some("7 turns, $0.01")); + assert!(!stream.failed()); + } + + #[test] + fn usage_without_a_complete_cost_and_a_result_without_init_are_still_read() { + let mut stream = GrokProgress::default(); + stream.condense(r#"{"type":"result","subtype":"success","result":"done","num_turns":1,"total_cost_usd":0,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!( + stream.summary().as_deref(), + Some( + "812 input tokens, 45 cache read tokens, 12 cache creation tokens, 210 output tokens" + ) + ); + } + + #[test] + fn malformed_and_unrelated_lines_are_ignored_and_error_frames_report_the_failure() { + let mut stream = GrokProgress::default(); + for line in [ + "not json", + "null", + "[]", + "{}", + r#"{"type":"assistant","message":"bad"}"#, + ] { + assert!(stream.condense(line).is_empty()); + } + stream.condense(r#"{"type":"error","message":"backend unavailable"}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("backend unavailable")); + stream.condense(r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("quota exceeded")); + } +} diff --git a/src/main.rs b/src/main.rs index aa93ecc43..1eb3afded 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,7 +73,7 @@ merge, --no-merge, --email, --no-email, base-fix, --no-base-fix, parallel (o --parallel ), harness , model and effort (or --harness, --model and --effort) go before or after the Issue URL, in any order. -harness, model and effort choose the Harness every session of the command runs on, and its +harness (claude, codex, agy, grok or muse), model and effort choose the Harness every session runs on, and its Model and Effort, for each Ticket's Run and a Base fix too. For each, the command wins, then the User config, then the default: claude, with its own Model and Effort. A Model and Effort in the User config come from the chosen Harness's own section, so harness claude over a codex @@ -89,7 +89,10 @@ default takes [harness.claude]: Before any work, the Harness's CLI must be on PATH. On claude, a named Model gets a minimal test call with its Effort, which must succeed. On codex, a named Model and Effort must be in codex debug models, matched regardless of case, a Model by slug or display name, and are -passed on as Codex names them. A failure stops the command naming what to fix, before the +passed on as Codex names them. Grok Build checks grok models and its refreshed effort cache, +then runs grok -p --always-approve --sandbox off --output-format streaming-messages-json with null stdin, +GROK_DISABLE_AUTOUPDATER=1 and GROK_FOLDER_TRUST=0. Resume uses -r . +A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json --dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, as in Built with claude · opus · diff --git a/src/session.rs b/src/session.rs index ed07d3091..b3240a183 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,4 +1,4 @@ -//! Headless agent sessions, on Claude Code or Codex, and their logs. +//! Headless agent sessions on the chosen Harness, and their logs. use std::cell::RefCell; use std::fs::{self, File}; diff --git a/src/setup.rs b/src/setup.rs index 26276c177..cec600489 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -369,6 +369,8 @@ pub(crate) trait Outside { fn codex_catalog(&mut self) -> Result; /// Read Antigravity CLI's free Model catalog. fn agy_catalog(&mut self) -> Result; + /// Read Grok Build's catalog, refreshed by grok models. + fn grok_catalog(&mut self) -> Result; /// Check Muse's settings against its cache, or with its minimal test call. fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result; /// Read the Credentials, as strictly as a Run does: where a Resend API @@ -467,6 +469,10 @@ impl Outside for OnMachine<'_> { crate::harness::agy::Catalog::read() } + fn grok_catalog(&mut self) -> Result { + crate::harness::grok::Catalog::read() + } + fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result { crate::harness::muse::check_model_and_effort(chosen) } @@ -663,6 +669,8 @@ mod scripted { /// The catalog's JSON, or what reading it fails with. pub catalog: Result<&'static str, &'static str>, pub agy_catalog: Result<&'static str, &'static str>, + /// Grok's text model list and refreshed effort cache, or a read failure. + pub grok_catalog: Result<(&'static str, &'static str), &'static str>, pub muse_checks: VecDeque>, /// Where a key is found, or what reading the Credentials fails with. pub key: Result, &'static str>, @@ -689,6 +697,10 @@ mod scripted { agy_catalog: Ok( "gemini-3.8-flash-high\tGemini 3.8 Flash (High)\ngemini-3.8-flash-medium\tGemini 3.8 Flash (Medium)\ngemini-3.8-flash-low\tGemini 3.8 Flash (Low)\n", ), + grok_catalog: Ok(( + include_str!("../tests/fixtures/grok-models.txt"), + include_str!("../tests/fixtures/grok-models.json"), + )), muse_checks: VecDeque::new(), key: Ok(None), github_email: Ok(None), @@ -819,6 +831,13 @@ mod scripted { } } + fn grok_catalog(&mut self) -> Result { + match self.grok_catalog { + Ok((list, cache)) => crate::harness::grok::Catalog::parse(list, cache), + Err(error) => bail!("{error}"), + } + } + fn muse_check(&mut self, chosen: &ModelAndEffort) -> Result { self.calls.push(Call::MuseChecked(chosen.clone())); self.muse_checks @@ -1141,7 +1160,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: ", + "Harness for every Run's sessions, claude or codex or agy or grok or muse [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1245,6 +1264,10 @@ effort = \"max\" model = \"\" effort = \"\" +[harness.grok] +model = \"\" +effort = \"\" + [harness.muse] model = \"\" effort = \"\" @@ -1277,7 +1300,7 @@ effort = \"\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: ", + "Harness for every Run's sessions, claude or codex or agy or grok or muse [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1779,11 +1802,11 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex or agy or muse [claude]: " + "Harness for every Run's sessions, claude or codex or agy or grok or muse [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex", "agy", "muse"] { + for harness in ["claude", "codex", "agy", "grok", "muse"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some("")); } @@ -1837,7 +1860,7 @@ always = false # quiet, please outside.said(), [ "No Harness is installed here, so the harness settings stay as they are; \ - install claude or codex or agy or muse, then rerun `thirdshift setup`." + install claude or codex or agy or grok or muse, then rerun `thirdshift setup`." ] ); let config = table(&outside); @@ -2023,6 +2046,55 @@ always = false # quiet, please ); } + #[test] + fn setup_offers_grok_when_it_is_the_only_installed_harness_and_checks_its_model_and_effort() { + let mut outside = Scripted { + installed: vec![Harness::Grok], + ..Scripted::answering(&then_nothing_else(&[ + (HARNESS, ""), + ("Model for grok", "bogus"), + ("Model for grok", "GROK-4.5"), + ("Effort for grok", "xhigh"), + ("Effort for grok", "High"), + ])) + }; + + setup(&mut outside).unwrap(); + + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("grok")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("high")); + let said = outside.said().join("\n"); + assert!( + said.contains("choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5"), + "{said}" + ); + assert!(said.contains("choose one of high, medium, low"), "{said}"); + assert!(outside.test_calls().is_empty()); + } + + #[test] + fn when_groks_catalog_cannot_be_read_setup_preserves_the_harness_settings() { + let mut outside = Scripted { + grok_catalog: Err("catalog unavailable"), + user_config: Some("[harness]\ndefault = \"codex\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n".to_string()), + ..Scripted::answering(&then_nothing_else(&[(HARNESS, "grok")])) + }; + setup(&mut outside).unwrap(); + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("medium")); + assert!(outside.said().join("\n").contains("once grok models works")); + } + #[test] fn codex_is_offered_with_its_models_and_the_chosen_models_efforts() { let mut outside = Scripted::answering(&then_nothing_else(&[ diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 481f0c19e..92fa1f664 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -229,6 +229,24 @@ pub(crate) fn ask_setting( Ok(Some(answer).filter(|answer| !answer.is_empty() && answer != "-")) } +/// Ask one catalog-backed setting, retrying refusals with the catalog's +/// valid choices and returning the name the Harness takes. +pub(crate) fn ask_checked_setting( + outside: &mut (impl Outside + ?Sized), + setting: &str, + harness: Harness, + current: Option<&str>, + settle: impl Fn(Option<&str>) -> Result>, +) -> Result> { + loop { + let answer = ask_setting(outside, setting, harness, current)?; + match settle(answer.as_deref()) { + Ok(settled) => return Ok(settled), + Err(error) => outside.say(format!("{error:#}")), + } + } +} + /// Ask `question` until the answer is yes, no or nothing, which is `default`. fn yes_or_no(outside: &mut impl Outside, question: &str, default: bool) -> Result { let choices = if default { "[Y/n]" } else { "[y/N]" }; diff --git a/tests/fakes.rs b/tests/fakes.rs index 4de34f85d..d1a5d79c9 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh`, `claude`, `codex`, `agy` and `muse` the end-to-end tests put first on +//! The fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse` the end-to-end tests put first on //! PATH: one executable, which acts as whichever its name says. See //! `fakes/gh.rs`, `fakes/claude.rs` and `fakes/codex.rs` for what each does, //! and `support/fakes.rs` for how the harness builds it. @@ -21,6 +21,8 @@ mod claude; mod codex; #[path = "fakes/gh.rs"] mod gh; +#[path = "fakes/grok.rs"] +mod grok; #[path = "fakes/json.rs"] mod json; #[path = "fakes/muse.rs"] @@ -116,6 +118,7 @@ fn main() { "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), "agy" => agy::main(args.collect()), + "grok" => grok::main(args.collect()), "muse" => muse::main(args.collect()), "detached-command" => { let command = args.next().expect("no detached command"); diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 2396018a3..3fb7c7a9e 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -64,7 +64,7 @@ pub fn stdin_is_null() -> bool { /// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts /// them until its script ends. A caught signal, unlike an ignored one, is /// back to its default in the script, which can trap it. -fn outlast_interrupts() { +pub fn outlast_interrupts() { // From the C library, which the standard library links: SIGINT and // SIGTERM are 2 and 15 on Linux and macOS alike. unsafe extern "C" { diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs new file mode 100644 index 000000000..76b7b6d3a --- /dev/null +++ b/tests/fakes/grok.rs @@ -0,0 +1,140 @@ +//! Fake Grok Build: its text catalog, cache and Claude-shaped Messages stream. + +use std::fs; +use std::io::Write; +use std::process::Command; + +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::codex::{outlast_interrupts, stdin_is_null}; +use crate::json::{Array, Bool, Null, object, string}; + +pub const MODELS: &str = include_str!("../fixtures/grok-models.txt"); +const CACHE: &str = include_str!("../fixtures/grok-models.json"); + +pub fn main(argv: Vec) { + let catalog = argv.first().is_some_and(|arg| arg == "models"); + if !catalog && !argv.iter().any(|arg| arg == "-p") { + crate::die("fake grok: expected models or -p", 2); + } + if !catalog { + let prompt = argv + .iter() + .position(|arg| arg == "-p") + .and_then(|at| argv.get(at + 1)); + if prompt.is_none_or(|prompt| prompt.starts_with('-')) { + crate::die("a value is required for --single ", 2); + } + } + let record_path = crate::env_path("FAKE_GROK_RECORD"); + let lock = crate::lock_beside(&record_path); + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let cwd = std::env::current_dir().unwrap(); + let records = crate::append_record( + &record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ( + "prompt", + if catalog { + Null + } else { + argv.last().map(string).unwrap_or(Null) + }, + ), + ("cwd", string(cwd.to_str().unwrap())), + ("branch", string(branch)), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ("skill_files", skills_snapshot(".agents/skills")), + ("beside_skills", beside_skills_snapshot(".agents/skills")), + ( + "GROK_DISABLE_AUTOUPDATER", + string(std::env::var("GROK_DISABLE_AUTOUPDATER").unwrap_or_default()), + ), + ( + "GROK_FOLDER_TRUST", + string(std::env::var("GROK_FOLDER_TRUST").unwrap_or_default()), + ), + ]), + ); + let sessions: Vec<_> = records + .items() + .iter() + .filter(|call| call.at("prompt") != &Null) + .cloned() + .collect(); + drop(lock); + if catalog { + let home = crate::env_path("HOME"); + let cache = home.join(".grok/models_cache.json"); + fs::create_dir_all(cache.parent().unwrap()).unwrap(); + fs::write(cache, CACHE).unwrap(); + println!("{MODELS}"); + crate::exit(0); + } + outlast_interrupts(); + let session = sessions.len(); + let script = script_for(&sessions); + let id = argv + .iter() + .position(|arg| arg == "-r") + .and_then(|at| argv.get(at + 1)) + .cloned() + .unwrap_or_else(|| format!("fake-grok-{session}")); + println!( + "{}", + object([ + ("type", string("system")), + ("subtype", string("init")), + ("session_id", string(&id)), + ("cwd", string(cwd.to_str().unwrap())), + ]) + ); + std::io::stdout().flush().unwrap(); + let final_message = format!("{}.final-message.{session}", record_path.display()); + let result = format!("{}.result.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_GROK_RESULT", &result) + .status() + .unwrap(); + let code = exit_code(status); + if let Ok(result) = fs::read_to_string(result) { + print!("{result}"); + } else { + println!( + "{}", + object([ + ("type", string("result")), + ( + "subtype", + string(if code == 0 { + "success" + } else { + "error_during_execution" + }) + ), + ("is_error", Bool(code != 0)), + ("session_id", string(id)), + ( + "result", + string(fs::read_to_string(&final_message).unwrap_or_default()) + ), + ("num_turns", crate::json::number(2)), + ("total_cost_usd", crate::json::number(0.0127)), + ( + "usage", + object([ + ("input_tokens", crate::json::number(812)), + ("output_tokens", crate::json::number(210)), + ("cache_read_input_tokens", crate::json::number(0)), + ]) + ), + ]) + ); + } + crate::exit(code); +} diff --git a/tests/fixtures/grok-models.json b/tests/fixtures/grok-models.json new file mode 100644 index 000000000..7787a2d15 --- /dev/null +++ b/tests/fixtures/grok-models.json @@ -0,0 +1,6 @@ +{"models": { + "grok-4.7": {"info": {"id": "grok-4.7", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.7-build-fast": {"info": {"id": "grok-4.7-build-fast", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.6": {"info": {"id": "grok-4.6", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.5": {"info": {"id": "grok-4.5", "reasoning_efforts": [{"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}} +}} diff --git a/tests/fixtures/grok-models.txt b/tests/fixtures/grok-models.txt new file mode 100644 index 000000000..2a84b9e2b --- /dev/null +++ b/tests/fixtures/grok-models.txt @@ -0,0 +1,9 @@ +You are logged in with grok.com. + +Default model: grok-4.7 + +Available models: + * grok-4.7 (default) + - grok-4.7-build-fast + - grok-4.6 + - grok-4.5 diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs new file mode 100644 index 000000000..71a149a93 --- /dev/null +++ b/tests/grok_sessions.rs @@ -0,0 +1,230 @@ +//! Commands on Grok Build through the scenario harness. + +mod support; + +use support::Scenario; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +#[test] +fn a_run_can_choose_grok_build() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert!(result.stderr.contains("sessions run on grok")); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_invalid_model_or_effort_fails_before_any_work_and_names_the_valid_choices() { + for (model, effort, error) in [ + ( + "grok-9.9-bogus", + "high", + "the Model grok-9.9-bogus is not in Grok Build's catalog: choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5", + ), + ( + "grok-4.5", + "xhigh", + "the Effort xhigh is not one the Grok Build Model grok-4.5 supports: choose one of high, medium, low", + ), + ( + "grok-4.7", + "max", + "the Effort max is not one the Grok Build Model grok-4.7 supports: choose one of xhigh, high, medium, low", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "grok", + "model", + model, + "effort", + effort, + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!(result.stderr.contains(error), "{}", result.stderr); + assert_eq!(scenario.entries("work"), ["widgets"]); + assert!(scenario.issue_labels(7).is_empty()); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0]["argv"], serde_json::json!(["models"])); + assert_eq!(calls[0]["stdin_null"], true); + assert_eq!(calls[0]["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(calls[0]["GROK_FOLDER_TRUST"], "0"); + } +} + +#[test] +fn grok_sessions_are_unattended_load_the_factory_skill_and_report_tools_and_usage() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#" +echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"run_terminal_command","input":{{"command":"cargo test"}}}}]}}}}' +{AGENT_OPENS_PR} +"#)); + let url = scenario.issue_url(7); + let result = scenario.run_with_env( + &[ + "harness", "grok", "model", "GROK-4.7", "effort", "High", &url, + ], + &[ + ("GROK_DISABLE_AUTOUPDATER", "0"), + ("GROK_FOLDER_TRUST", "1"), + ("GROK_SANDBOX", "workspace"), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + for line in [ + "implement: $ cargo test", + "2 turns, $0.01", + "sessions run on grok · grok-4.7 · high", + ] { + assert!( + result.stderr.contains(line), + "missing {line:?}: {}", + result.stderr + ); + } + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 2); + let session = &calls[1]; + let prompt = session["prompt"].as_str().unwrap(); + assert_eq!( + prompt.lines().next().unwrap(), + format!("/thirdshift-implement {url}") + ); + assert_eq!( + session["argv"], + serde_json::json!([ + "--always-approve", + "--sandbox", + "off", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "high", + "-p", + prompt + ]) + ); + assert_eq!(session["git_status"], ""); + for call in calls { + assert_eq!(call["stdin_null"], true); + assert_eq!(call["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(call["GROK_FOLDER_TRUST"], "0"); + } + scenario.assert_every_grok_session_found_the_factory_skills(); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + assert!(log.contains("fake-grok-1")); + assert!(log.contains("\"type\": \"result\"")); + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with grok · grok-4.7 · high") + ); +} + +#[test] +fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { + for (event, exit, cause) in [ + ( + r#"{"type":"error","message":"backend unavailable"}"#, + "exit 1", + "grok exited 1: backend unavailable", + ), + ( + r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#, + "", + "grok's turn failed: quota exceeded", + ), + ] { + let scenario = Scenario::new(); + let script = if exit.is_empty() { + format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_GROK_RESULT\"\n") + } else { + format!("echo '{event}'\n{exit}") + }; + scenario.agent_does_for(7, &script); + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(cause), "{}", result.stderr); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + let results = log + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["type"] == "result") + .count(); + assert_eq!(results, 1, "{log}"); + } +} + +#[test] +fn the_user_config_can_choose_grok_and_its_own_model_and_effort() { + let scenario = Scenario::new(); + scenario.user_config_is("[harness]\ndefault = \"grok\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[&scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("sessions run on grok · grok-4.5 · medium") + ); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_architecture_reviews_final_message_reaches_the_command() { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does( + r#" +idea=$(gh issue create --title "Deepen sessions" --body "An idea" --label needs-triage) +printf 'Architecture review idea: %s\n' "$idea" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let result = scenario.run(&["architect", "harness", "grok"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + scenario + .issue_labels(8) + .contains(&"architect-idea".to_string()) + ); + let calls = scenario.grok_calls(); + assert_eq!( + calls[1]["prompt"].as_str().unwrap().lines().next().unwrap(), + "/thirdshift-improve-codebase-architecture" + ); + scenario.assert_every_grok_session_found_the_factory_skills(); +} diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 0c6e379e9..a18325259 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -50,6 +50,16 @@ fn interrupting_an_agy_run_stops_a_command_in_its_own_session() { assert_detached_command_stopped("agy"); } +#[test] +fn interrupting_a_grok_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("grok"); +} + +#[test] +fn interrupting_a_muse_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("muse"); +} + fn assert_detached_command_stopped(harness: &str) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); diff --git a/tests/setup.rs b/tests/setup.rs index ebab09416..7d17bb962 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -54,6 +54,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[harness.claude]", "[harness.codex]", "[harness.agy]", + "[harness.grok]", "[harness.muse]" ], "{text}" @@ -72,7 +73,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex", "agy", "muse"] { + for harness in ["claude", "codex", "agy", "grok", "muse"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 18cf8e04f..f19d4d8af 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -1,7 +1,7 @@ -//! The fake `gh`, `claude`, `codex`, `agy` and `muse`, built from `tests/fakes.rs` with plain -//! `rustc` the first time a test needs them and kept in Cargo's scratch -//! directory for tests. They aren't a binary target, so neither the crate nor -//! a release ships them. +//! The fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse`, built from +//! `tests/fakes.rs` with plain `rustc` the first time a test needs them and +//! kept in Cargo's scratch directory for tests. They aren't a binary target, +//! so neither the crate nor a release ships them. use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; @@ -11,7 +11,7 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 7] = [ +const SOURCES: [&str; 10] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), @@ -19,11 +19,22 @@ const SOURCES: [&str; 7] = [ include_str!("../fakes/agy.rs"), include_str!("../fakes/muse.rs"), include_str!("../fakes/json.rs"), + include_str!("../fakes/grok.rs"), + include_str!("../fixtures/grok-models.json"), + include_str!("../fixtures/grok-models.txt"), ]; -/// Put the fake `gh`, `claude`, `codex`, `agy` and `muse` in `bin`. +/// Put the fake `gh`, `claude`, `codex`, `agy`, `grok` and `muse` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex", "agy", "muse", "detached-command"] { + for name in [ + "gh", + "claude", + "codex", + "agy", + "grok", + "muse", + "detached-command", + ] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 70d295e16..1375d01f6 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -605,6 +605,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) .env("FAKE_AGY_RECORD", self.path("agy-calls.json")) .env("FAKE_AGY_CHECK_RECORD", self.path("agy-checks.json")) + .env("FAKE_GROK_RECORD", self.path("grok-calls.json")) .env("FAKE_MUSE_RECORD", self.path("muse-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts @@ -840,6 +841,23 @@ test -f {root}/{COPY_REPLACED} .unwrap_or_default() } + /// Every call the fake received on Grok, including its catalog check. + pub fn grok_calls(&self) -> Vec { + match fs::read_to_string(self.path("grok-calls.json")) { + Ok(text) => serde_json::from_str(&text).unwrap(), + Err(_) => Vec::new(), + } + } + + pub fn assert_every_grok_session_found_the_factory_skills(&self) { + assert_found_the_factory_skills( + self.grok_calls() + .into_iter() + .filter(|call| call["prompt"].is_string()) + .collect(), + ); + } + pub fn assert_every_muse_session_found_the_factory_skills(&self) { assert_found_the_factory_skills(self.muse_calls()); }