From 347dfe772f1b490d092f26ec1bfdce044bf54cf3 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:02:45 -0400 Subject: [PATCH 1/3] Add Grok Build Harness (#437) --- CONTEXT.md | 2 +- README.md | 20 ++- src/args.rs | 4 +- src/asks.rs | 1 + src/config.rs | 15 +- src/harness.rs | 8 +- src/harness/adapter.rs | 8 +- src/harness/grok.rs | 115 ++++++++++++++ src/harness/grok/catalog.rs | 198 ++++++++++++++++++++++++ src/harness/grok/stream.rs | 156 +++++++++++++++++++ src/main.rs | 7 +- src/progress.rs | 2 +- src/session.rs | 2 +- src/setup.rs | 92 +++++++++-- src/setup/questions.rs | 4 +- tests/fakes.rs | 3 + tests/fakes/codex.rs | 4 +- tests/fakes/grok.rs | 131 ++++++++++++++++ tests/fixtures/grok-models.json | 6 + tests/fixtures/grok-models.txt | 9 ++ tests/grok_sessions.rs | 261 ++++++++++++++++++++++++++++++++ tests/process_tree.rs | 5 + tests/setup.rs | 5 +- tests/support/fakes.rs | 7 +- tests/support/mod.rs | 18 +++ 25 files changed, 1046 insertions(+), 37 deletions(-) create mode 100644 src/harness/grok.rs create mode 100644 src/harness/grok/catalog.rs create mode 100644 src/harness/grok/stream.rs create mode 100644 tests/fakes/grok.rs create mode 100644 tests/fixtures/grok-models.json create mode 100644 tests/fixtures/grok-models.txt create mode 100644 tests/grok_sessions.rs diff --git a/CONTEXT.md b/CONTEXT.md index 3db2d76f9..1c8881191 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -31,7 +31,7 @@ The message thirdshift starts or resumes an agent session with. It carries the * _Avoid_: instructions, system prompt **Harness**: -The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`) or Codex (`codex`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. +The headless agent CLI that runs every agent session of a **Command**: Claude Code (`claude`), Codex (`codex`) or Grok Build (`grok`). One per Command, its child Runs and every **Resume** and **Repair** included. Chosen by the command, else the **User config**, else Claude Code. _Avoid_: agent, provider, backend **Model**: diff --git a/README.md b/README.md index fdc837fc9..74181775f 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or grok; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -254,11 +254,15 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude` when both are, and `codex` when only it is. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. The other Harness's section is left as it is. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing Claude Code, Codex and Grok Build, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude`, then `codex`, then `grok`. Then the Model (`harness..model`) and the Effort (`harness..effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. For Grok Build, Setup lists `grok models` and the chosen Model's Efforts from its refreshed cache, and checks each answer against them. The other Harnesses' sections are left as they are. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) @@ -289,19 +293,25 @@ A Run reads the file before any work. One that isn't valid TOML, or that has a k ### Harness, Model and Effort +The available Harnesses are Claude Code (`claude`), Codex (`codex`) and Grok Build (`grok`). + Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: ```sh thirdshift https://github.com///issues/ model claude-opus-5-5 effort high ``` -For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. +For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Grok Build takes `-m ` and `--reasoning-effort `. Codex takes them as it names them, which the check below settles: `-m ` and `-c model_reasoning_effort=""` when set. With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. +With `harness grok`, every session runs `grok -p --always-approve --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag is always explicit. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. + +Grok reads both `AGENTS.md` and `CLAUDE.md` when both exist, along with `AGENT.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md` and project rules under `.grok`, `.claude` and `.cursor`. It also reads `~/.grok/rules`, `~/.claude/CLAUDE.md` and `~/.claude/rules`; thirdshift writes none of these user-level files. Grok Build uses `~/.grok` for sign-in and sessions (or `GROK_HOME` when set). + Interrupting a Command stops every session's process tree on every Harness, including commands that started their own process group or session. thirdshift snapshots the descendants with `ps` on Linux and macOS, sends each process group the Harness's stop signals with a 10-second grace period each, then takes another snapshot before SIGKILL to catch commands started in the meantime. If the CLI exits on an earlier signal, thirdshift kills the remaining commands immediately without another grace period. -Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. +Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. On Grok Build, `grok models` lists the available Models without a model call and refreshes `models_cache.json` under `GROK_HOME` or `~/.grok`; thirdshift reads that cache for each Model's supported Efforts. Both are matched regardless of case and passed as Grok names them. With no Model named, a named Effort is checked against Grok's current default Model. Unknown Models and unsupported Efforts fail with the valid choices, and an unreadable catalog fails before work. The check has the same null stdin and fixed environment as sessions. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. @@ -859,7 +869,7 @@ Editing a skill in `skills/` has no effect until you rebuild and reinstall ([ADR The [Prompts and skills page](https://thirdshift.app/prompts/) is generated from the prompts, the `claude` arguments and the skills, and each Session prompt is also generated as a Markdown file in [`prompts/`](prompts/), with a [`README.md`](prompts/README.md) that lists them with the `claude` command lines. `cargo test` fails until they are regenerated after a change to any of them. Regenerate the page and `prompts/` with `UPDATE_PROMPTS=1 cargo test prompts_page`, which also deletes any file in `prompts/` that it doesn't generate. -The integration tests swap in a fake `gh` and `claude`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. +The integration tests swap in a fake `gh`, `claude`, `codex` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. ### The Rust version diff --git a/src/args.rs b/src/args.rs index b445828cf..ec34ac380 100644 --- a/src/args.rs +++ b/src/args.rs @@ -995,11 +995,11 @@ mod tests { for (args, error) in [ ( vec![URL, "harness"], - "harness must be followed by claude or codex", + "harness must be followed by claude or codex or grok", ), ( vec![URL, "harness", "gemini"], - "harness must be followed by claude or codex, not gemini", + "harness must be followed by claude or codex or grok, not gemini", ), ( vec!["--harness", "claude", URL, "harness", "codex"], diff --git a/src/asks.rs b/src/asks.rs index 74f5f68af..c03951ddf 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -262,6 +262,7 @@ mod tests { default: Some(Harness::Codex), claude: model_and_effort("opus", "high"), codex: model_and_effort("gpt-6.1-sol", "max"), + ..harness::Settings::default() }, ..no_settings() } diff --git a/src/config.rs b/src/config.rs index 5bbd324d6..2d13e7a3f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -387,7 +387,7 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 [harness] -default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude +default = "claude" # the Harness every Run's sessions run on, claude, codex or grok; default claude [harness.claude] model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own @@ -396,6 +396,10 @@ effort = "" # how hard that Model reasons; default blank, for Claude Code's ow [harness.codex] model = "" # the Model Codex's sessions run on; default blank, for Codex's own effort = "" # how hard that Model reasons; default blank, for Codex's own + +[harness.grok] +model = "" # the Model Grok Build's sessions run on; default blank, for Grok Build's own +effort = "" # how hard that Model reasons; default blank, for Grok Build's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -527,7 +531,9 @@ mod tests { "harness.claude.model", "harness.claude.effort", "harness.codex.model", - "harness.codex.effort" + "harness.codex.effort", + "harness.grok.model", + "harness.grok.effort" ] ); let commented_out: Vec<&str> = DEFAULTS @@ -577,6 +583,7 @@ mod tests { model: Some("gpt-6.1-sol".to_string()), effort: Some("max".to_string()), }, + ..harness::Settings::default() } ); assert_eq!(parse("").unwrap().harness, harness::Settings::default()); @@ -587,7 +594,7 @@ mod tests { for (text, error) in [ ( "[harness]\ndefault = \"gemini\"\n", - "harness.default must be claude or codex, not \"gemini\"", + "harness.default must be claude or codex or grok, not \"gemini\"", ), ( "[harness]\ndefault = true\n", @@ -868,7 +875,7 @@ mod tests { let config: toml::Table = completed.parse().unwrap(); let harness = config["harness"].as_table().unwrap(); assert!(harness.contains_key("default"), "{text:?}:\n{completed}"); - for name in ["claude", "codex"] { + for name in ["claude", "codex", "grok"] { let settings = harness[name].as_table().unwrap(); for key in ["model", "effort"] { assert!(settings.contains_key(key), "{text:?}:\n{completed}"); diff --git a/src/harness.rs b/src/harness.rs index d20d5074c..f835fcb81 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -12,6 +12,7 @@ use anyhow::{Result, bail}; mod adapter; pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod grok; mod process_tree; pub use adapter::{Adapter, Invocation}; @@ -22,6 +23,7 @@ pub use codex::Catalog; pub enum Harness { Claude, Codex, + Grok, } impl Harness { @@ -91,6 +93,8 @@ pub struct Settings { pub claude: ModelAndEffort, /// `[harness.codex]`. pub codex: ModelAndEffort, + /// `[harness.grok]`. + pub grok: ModelAndEffort, } impl Settings { @@ -194,7 +198,8 @@ impl Choice { /// minimal test call, with the Effort, if any. A Model or Effort named /// for Codex must be in its catalog, which costs no tokens to read: each /// is matched regardless of case, a Model by its slug or display name, - /// and becomes the name Codex takes. Each failure says what to change. + /// and becomes the name Codex takes. Grok Build checks its free model + /// list and refreshed effort cache. Each failure says what to change. pub fn check(&mut self) -> Result<()> { let adapter = self.harness.adapter(); let cli = adapter.name(); @@ -281,6 +286,7 @@ mod tests { default: Some(Harness::Codex), claude: set("opus", "high"), codex: set("gpt-6.1-sol", "max"), + ..Settings::default() } } diff --git a/src/harness/adapter.rs b/src/harness/adapter.rs index 10a03c07c..0a1f60b90 100644 --- a/src/harness/adapter.rs +++ b/src/harness/adapter.rs @@ -5,7 +5,7 @@ use std::process::Child; use anyhow::Result; -use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex}; +use super::{Choice, Harness, ModelAndEffort, Settings, claude, codex, grok}; use crate::progress::Stream; use crate::setup::Outside; @@ -91,4 +91,8 @@ impl SkillLoading { } /// Registration comes from the adapters, including Setup's order. -pub const REGISTERED: [(Harness, &dyn Adapter); 2] = [claude::REGISTRATION, codex::REGISTRATION]; +pub const REGISTERED: [(Harness, &dyn Adapter); 3] = [ + claude::REGISTRATION, + codex::REGISTRATION, + grok::REGISTRATION, +]; diff --git a/src/harness/grok.rs b/src/harness/grok.rs new file mode 100644 index 000000000..442652d42 --- /dev/null +++ b/src/harness/grok.rs @@ -0,0 +1,115 @@ +//! Grok Build: unattended sessions and its Claude-shaped Messages stream. + +mod catalog; +mod stream; +pub use catalog::Catalog; + +use std::path::Path; + +use anyhow::Result; + +use super::adapter::{Adapter, Invocation, SkillLoading}; +use super::{Choice, Harness, ModelAndEffort, Settings}; +use crate::progress::Stream; +use crate::setup::{Outside, questions::ask_setting}; + +pub struct Grok; +pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Grok, &Grok); + +impl Adapter for Grok { + fn name(&self) -> &'static str { + "grok" + } + fn product_name(&self) -> &'static str { + "Grok Build" + } + fn project_skills(&self) -> &'static str { + ".agents/skills" + } + fn skill_loading(&self) -> SkillLoading { + SkillLoading::Slash + } + fn stop_signals(&self) -> &'static [libc::c_int] { + &[libc::SIGTERM] + } + fn environment(&self) -> &'static [(&'static str, &'static str)] { + &[ + ("GROK_DISABLE_AUTOUPDATER", "1"), + ("GROK_FOLDER_TRUST", "0"), + ] + } + fn settings<'a>(&self, settings: &'a Settings) -> &'a ModelAndEffort { + &settings.grok + } + fn settings_mut<'a>(&self, settings: &'a mut Settings) -> &'a mut ModelAndEffort { + &mut settings.grok + } + fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { + let mut args: Vec = [ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + ] + .map(String::from) + .to_vec(); + if let Some(model) = &choice.model { + args.extend(["-m".to_string(), model.clone()]); + } + if let Some(effort) = &choice.effort { + args.extend(["--reasoning-effort".to_string(), effort.clone()]); + } + if let Some(id) = resume { + args.extend(["-r".to_string(), id.to_string()]); + } + args.push(self.skill_loading().prompt(prompt)); + Invocation { args, stdin: None } + } + fn check(&self, choice: &mut Choice) -> Result<()> { + crate::progress::step("checking the Model and Effort against grok models"); + let settled = Catalog::read()?.settle(choice.model.as_deref(), choice.effort.as_deref())?; + choice.model = settled.model; + choice.effort = settled.effort; + Ok(()) + } + fn ask_settings( + &self, + outside: &mut dyn Outside, + current: &ModelAndEffort, + ) -> Result> { + let catalog = match outside.grok_catalog() { + Ok(catalog) => catalog, + Err(error) => { + outside.say(format!("{error:#}\nThe harness settings stay as they are; rerun `thirdshift setup` once grok models works.")); + return Ok(None); + } + }; + outside.say(format!( + "{}'s Models: {}", + self.product_name(), + catalog.listed().join(", ") + )); + let model = loop { + let model = ask_setting(outside, "Model", Harness::Grok, current.model.as_deref())?; + match catalog.settle(model.as_deref(), None) { + Ok(settled) => break settled.model, + Err(error) => outside.say(format!("{error:#}")), + } + }; + outside.say(format!( + "Efforts {} supports: {}", + model.as_deref().unwrap_or("Grok Build's default Model"), + catalog.efforts(model.as_deref()).join(", ") + )); + loop { + let effort = ask_setting(outside, "Effort", Harness::Grok, current.effort.as_deref())?; + match catalog.settle(model.as_deref(), effort.as_deref()) { + Ok(settled) => return Ok(Some(settled)), + Err(error) => outside.say(format!("{error:#}")), + } + } + } + fn stream(&self, _worktree: &Path) -> Box { + Box::new(stream::GrokProgress::default()) + } +} diff --git a/src/harness/grok/catalog.rs b/src/harness/grok/catalog.rs new file mode 100644 index 000000000..7712df051 --- /dev/null +++ b/src/harness/grok/catalog.rs @@ -0,0 +1,198 @@ +//! Grok's free model list and per-model Efforts from the cache it refreshes. + +use std::fs; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use serde_json::Value; + +use super::Grok; +use crate::harness::{Adapter, ModelAndEffort, said}; + +#[derive(Debug)] +struct Model { + name: String, + efforts: Vec, +} + +#[derive(Debug)] +pub struct Catalog { + models: Vec, + default: String, +} + +impl Catalog { + pub fn read() -> Result { + let output = Command::new(Grok.name()) + .arg("models") + .envs(Grok.environment().iter().copied()) + .stdin(Stdio::null()) + .output() + .context("could not run grok models to read Grok Build's Models")?; + if !output.status.success() { + bail!( + "grok models failed, so Grok Build's Models can't be read: {}", + said(&output) + ); + } + // `grok models` prints names, but refreshes the effort metadata here. + let home = match std::env::var_os("GROK_HOME").filter(|home| !home.is_empty()) { + Some(home) => std::path::PathBuf::from(home), + None => crate::config::home()?.join(".grok"), + }; + let path = home.join("models_cache.json"); + let cache = fs::read_to_string(&path).with_context(|| { + format!( + "could not read Grok Build's Model and Effort catalog {} after grok models", + path.display() + ) + })?; + Self::parse(&String::from_utf8_lossy(&output.stdout), &cache) + } + + /// The text list and JSON cache recorded from Grok Build 1.0.46. + pub fn parse(list: &str, cache: &str) -> Result { + let cache: Value = + serde_json::from_str(cache).context("Grok Build's Model cache is not JSON")?; + let metadata = cache["models"] + .as_object() + .context("Grok Build's cache has no Models")?; + let default = list + .lines() + .find_map(|line| line.trim().strip_prefix("Default model: ")) + .context("grok models printed no default Model")? + .trim() + .to_string(); + let available = list + .split_once("Available models:") + .context("grok models printed no list of Models")? + .1; + let mut models = Vec::new(); + for name in available.lines().filter_map(|line| { + let line = line.trim(); + line.strip_prefix("* ") + .or_else(|| line.strip_prefix("- "))? + .split_whitespace() + .next() + }) { + let info = metadata.get(name).context(format!( + "Grok Build's cache has no metadata for Model {name}" + ))?; + let efforts = info["info"]["reasoning_efforts"] + .as_array() + .into_iter() + .flatten() + .filter_map(|effort| effort["value"].as_str()) + .map(String::from) + .collect(); + models.push(Model { + name: name.to_string(), + efforts, + }); + } + if models.is_empty() || !models.iter().any(|model| model.name == default) { + bail!("grok models printed no available default Model"); + } + Ok(Self { models, default }) + } + + pub fn listed(&self) -> Vec<&str> { + self.models + .iter() + .map(|model| model.name.as_str()) + .collect() + } + + fn model(&self, name: &str) -> Result<&Model> { + self.models + .iter() + .find(|model| model.name.eq_ignore_ascii_case(name)) + .with_context(|| { + format!( + "the Model {name} is not in Grok Build's catalog: choose one of {}", + self.listed().join(", ") + ) + }) + } + + pub fn efforts(&self, model: Option<&str>) -> Vec<&str> { + self.model(model.unwrap_or(&self.default)) + .map(|model| model.efforts.iter().map(String::as_str).collect()) + .unwrap_or_default() + } + + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result { + let found = self.model(model.unwrap_or(&self.default))?; + let effort = match effort { + None => None, + Some(effort) => { + let Some(valid) = found + .efforts + .iter() + .find(|valid| valid.eq_ignore_ascii_case(effort)) + else { + bail!( + "the Effort {effort} is not one the Grok Build Model {} supports: choose one of {}", + found.name, + found.efforts.join(", ") + ); + }; + Some(valid.clone()) + } + }; + Ok(ModelAndEffort { + model: model.map(|_| found.name.clone()), + effort, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIST: &str = include_str!("../../../tests/fixtures/grok-models.txt"); + const CACHE: &str = include_str!("../../../tests/fixtures/grok-models.json"); + + #[test] + fn recorded_models_have_their_own_efforts_and_default_model_is_checked_when_no_model_is_set() { + let catalog = Catalog::parse(LIST, CACHE).unwrap(); + assert_eq!( + catalog.listed(), + ["grok-4.7", "grok-4.7-build-fast", "grok-4.6", "grok-4.5"] + ); + assert_eq!(catalog.efforts(Some("grok-4.5")), ["high", "medium", "low"]); + assert_eq!( + catalog.settle(None, Some("XHIGH")).unwrap(), + ModelAndEffort { + model: None, + effort: Some("xhigh".to_string()) + } + ); + let older_default = LIST.replace("Default model: grok-4.7", "Default model: grok-4.5"); + let older = Catalog::parse(&older_default, CACHE).unwrap(); + assert!( + older + .settle(None, Some("xhigh")) + .unwrap_err() + .to_string() + .contains("choose one of high, medium, low") + ); + } + + #[test] + fn an_unreadable_catalog_fails_instead_of_accepting_unchecked_settings() { + for (list, cache) in [ + ("", CACHE), + (LIST, "not json"), + (LIST, "{}"), + (LIST, "{\"models\": {}}"), + ( + "Default model: missing\nAvailable models:\n - missing", + CACHE, + ), + ] { + assert!(Catalog::parse(list, cache).is_err(), "{list:?} {cache:?}"); + } + } +} diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs new file mode 100644 index 000000000..59a000cc7 --- /dev/null +++ b/src/harness/grok/stream.rs @@ -0,0 +1,156 @@ +//! Grok's Messages stream shares Claude's framing, with different tool names. + +use serde_json::Value; + +use crate::harness::claude::stream::ClaudeProgress; +use crate::progress::Stream; + +#[derive(Default)] +pub struct GrokProgress { + claude: ClaudeProgress, + failed: bool, + error: Option, + session_id: Option, + usage: Option, +} + +impl Stream for GrokProgress { + fn condense(&mut self, raw: &str) -> Vec { + let Ok(mut event) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if let Some(id) = event["session_id"].as_str() { + self.session_id.get_or_insert_with(|| id.to_string()); + } + match event["type"].as_str() { + Some("error") => { + self.failed = true; + self.error = event["message"].as_str().map(String::from); + } + Some("result") => { + self.failed = event["is_error"] == true || event["subtype"] != "success"; + let errors: Vec<_> = event["errors"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + if !errors.is_empty() { + self.error = Some(errors.join("\n")); + } else if self.failed + && let Some(text) = event["result"].as_str().filter(|text| !text.is_empty()) + { + self.error = Some(text.to_string()); + } + if let (Some(input), Some(output)) = ( + event["usage"]["input_tokens"].as_u64(), + event["usage"]["output_tokens"].as_u64(), + ) { + let cached = event["usage"]["cache_read_input_tokens"] + .as_u64() + .unwrap_or(0); + let created = event["usage"]["cache_creation_input_tokens"] + .as_u64() + .unwrap_or(0); + self.usage = Some(format!( + "{input} input tokens, {cached} cache read tokens, {created} cache creation tokens, {output} output tokens" + )); + } + } + _ => {} + } + if let Some(content) = event + .get_mut("message") + .and_then(|message| message.get_mut("content")) + .and_then(Value::as_array_mut) + { + for block in content + .iter_mut() + .filter(|block| block["type"] == "tool_use") + { + if matches!( + block["name"].as_str(), + Some("bash" | "run_terminal_command") + ) { + block["name"] = Value::String("Bash".to_string()); + } + if let Some(path) = block["input"]["path"].as_str().map(String::from) { + block["input"]["file_path"] = Value::String(path); + } + } + } + self.claude.condense(&event.to_string()) + } + fn summary(&self) -> Option { + self.claude.summary().or_else(|| self.usage.clone()) + } + fn final_message(&self) -> Option<&str> { + self.claude.final_message() + } + fn session_id(&self) -> Option<&str> { + self.claude.session_id().or(self.session_id.as_deref()) + } + fn killed_background_work(&self) -> Vec<&str> { + if self.failed { + Vec::new() + } else { + self.claude.killed_background_work() + } + } + fn failed(&self) -> bool { + self.failed + } + fn error(&self) -> Option<&str> { + self.error.as_deref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn messages_frames_reuse_claude_progress_and_keep_groks_id_final_text_and_spend() { + let mut stream = GrokProgress::default(); + assert_eq!(stream.condense(r#"{"type":"system","subtype":"init","session_id":"abc123","apiKeySource":"oauth","cwd":"/repo"}"#), ["session started"]); + assert_eq!(stream.condense(r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"read_file","input":{"path":"/repo/src/main.rs"}},{"type":"tool_use","name":"bash","input":{"command":"cargo test"}}]}}"#), ["read_file src/main.rs", "$ cargo test"]); + stream.condense(r#"{"type":"result","subtype":"success","is_error":false,"num_turns":7,"result":"Here's a summary...","total_cost_usd":0.0127,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":0,"cache_creation_input_tokens":0},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!(stream.final_message(), Some("Here's a summary...")); + assert_eq!(stream.summary().as_deref(), Some("7 turns, $0.01")); + assert!(!stream.failed()); + } + + #[test] + fn usage_without_a_complete_cost_and_a_result_without_init_are_still_read() { + let mut stream = GrokProgress::default(); + stream.condense(r#"{"type":"result","subtype":"success","result":"done","usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); + assert_eq!(stream.session_id(), Some("abc123")); + assert_eq!( + stream.summary().as_deref(), + Some( + "812 input tokens, 45 cache read tokens, 12 cache creation tokens, 210 output tokens" + ) + ); + } + + #[test] + fn malformed_and_unrelated_lines_are_ignored_and_error_frames_report_the_failure() { + let mut stream = GrokProgress::default(); + for line in [ + "not json", + "null", + "[]", + "{}", + r#"{"type":"assistant","message":"bad"}"#, + ] { + assert!(stream.condense(line).is_empty()); + } + stream.condense(r#"{"type":"error","message":"backend unavailable"}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("backend unavailable")); + stream.condense(r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#); + assert!(stream.failed()); + assert_eq!(stream.error(), Some("quota exceeded")); + } +} diff --git a/src/main.rs b/src/main.rs index bc55f95bf..2f790f7cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,7 +73,7 @@ merge, --no-merge, --email, --no-email, base-fix, --no-base-fix, parallel (o --parallel ), harness , model and effort (or --harness, --model and --effort) go before or after the Issue URL, in any order. -harness, model and effort choose the Harness every session of the command runs on, and its +harness (claude, codex or grok), model and effort choose the Harness every session runs on, and its Model and Effort, for each Ticket's Run and a Base fix too. For each, the command wins, then the User config, then the default: claude, with its own Model and Effort. A Model and Effort in the User config come from the chosen Harness's own section, so harness claude over a codex @@ -89,7 +89,10 @@ default takes [harness.claude]: Before any work, the Harness's CLI must be on PATH. On claude, a named Model gets a minimal test call with its Effort, which must succeed. On codex, a named Model and Effort must be in codex debug models, matched regardless of case, a Model by slug or display name, and are -passed on as Codex names them. A failure stops the command naming what to fix, before the +passed on as Codex names them. Grok Build checks grok models and its refreshed effort cache, +then runs grok -p --always-approve --output-format streaming-messages-json with null stdin, +GROK_DISABLE_AUTOUPDATER=1 and GROK_FOLDER_TRUST=0. Resume uses -r . +A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json --dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, as in Built with claude · opus · diff --git a/src/progress.rs b/src/progress.rs index e0e9e74d6..3b583fe10 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1,5 +1,5 @@ //! Progress lines on stderr: thirdshift's own steps, and a session's -//! stream, Claude's stream-json or Codex's JSONL, condensed to one short +//! stream, parsed by its Harness adapter, condensed to one short //! line per notable event. Each line is stamped with the local time it was //! printed, so a stalled Run can be told from a busy one. Each is kept in the //! Command log too, if the command keeps one. diff --git a/src/session.rs b/src/session.rs index 174af6d2f..9f3f5f0a7 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,4 +1,4 @@ -//! Headless agent sessions, on Claude Code or Codex, and their logs. +//! Headless agent sessions on the chosen Harness, and their logs. use std::cell::RefCell; use std::fs::{self, File}; diff --git a/src/setup.rs b/src/setup.rs index d6804fcf3..dc6f4a6ef 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -367,6 +367,8 @@ pub(crate) trait Outside { fn test_call(&mut self, chosen: &ModelAndEffort) -> Result<()>; /// Read Codex's catalog of Models. fn codex_catalog(&mut self) -> Result; + /// Read Grok Build's catalog, refreshed by grok models. + fn grok_catalog(&mut self) -> Result; /// Read the Credentials, as strictly as a Run does: where a Resend API /// key is found, if anywhere. fn find_key(&mut self) -> Result>; @@ -459,6 +461,10 @@ impl Outside for OnMachine<'_> { Catalog::read() } + fn grok_catalog(&mut self) -> Result { + crate::harness::grok::Catalog::read() + } + fn find_key(&mut self) -> Result> { let credentials = Credentials::read()?; let found = credentials.lookup().map(|key| key.source); @@ -649,6 +655,8 @@ mod scripted { pub test_calls: VecDeque>, /// The catalog's JSON, or what reading it fails with. pub catalog: Result<&'static str, &'static str>, + /// Grok's text model list and refreshed effort cache, or a read failure. + pub grok_catalog: Result<(&'static str, &'static str), &'static str>, /// Where a key is found, or what reading the Credentials fails with. pub key: Result, &'static str>, pub github_email: Result, &'static str>, @@ -662,7 +670,7 @@ mod scripted { impl Scripted { /// A terminal answering `answers` in order, then closing stdin, - /// with both Harnesses installed, no key, no email to suggest and no + /// with every Harness installed, no key, no email to suggest and no /// User config. pub fn answering(answers: &[(&'static str, &'static str)]) -> Self { Scripted { @@ -671,6 +679,10 @@ mod scripted { installed: Harness::ALL.to_vec(), test_calls: VecDeque::new(), catalog: Ok(CATALOG), + grok_catalog: Ok(( + include_str!("../tests/fixtures/grok-models.txt"), + include_str!("../tests/fixtures/grok-models.json"), + )), key: Ok(None), github_email: Ok(None), git_email: Err("git config --global user.email: exit status 1"), @@ -793,6 +805,13 @@ mod scripted { } } + fn grok_catalog(&mut self) -> Result { + match self.grok_catalog { + Ok((list, cache)) => crate::harness::grok::Catalog::parse(list, cache), + Err(error) => bail!("{error}"), + } + } + fn find_key(&mut self) -> Result> { self.key.clone().map_err(|error| anyhow!("{error}")) } @@ -1107,7 +1126,7 @@ to = \"me@example.com\" # my inbox assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or grok [claude]: ", "Model for claude [claude's own default]: ", "Effort for claude [claude's own default]: ", "Every Run a Merge run? [y/N] ", @@ -1206,6 +1225,10 @@ effort = \"high\" [harness.codex] model = \"gpt-6.1-sol\" effort = \"max\" + +[harness.grok] +model = \"\" +effort = \"\" "; let mut outside = Scripted { user_config: Some(mine.to_string()), @@ -1235,7 +1258,7 @@ effort = \"max\" assert_eq!( outside.prompts(), [ - "Harness for every Run's sessions, claude or codex [claude]: ", + "Harness for every Run's sessions, claude or codex or grok [claude]: ", "Model for claude, - for claude's own default [opus]: ", "Effort for claude, - for claude's own default [high]: ", "Every Run a Merge run? [Y/n] ", @@ -1695,14 +1718,14 @@ always = false # quiet, please // The Harness, Model and Effort. #[test] - fn the_harness_question_lists_both_harnesses_and_defaults_to_claude() { + fn the_harness_question_lists_every_harness_and_defaults_to_claude() { let mut outside = Scripted::answering(&ENTER_THROUGHOUT); setup(&mut outside).unwrap(); assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude or codex [claude]: " + "Harness for every Run's sessions, claude or codex or grok [claude]: " ); let config = table(&outside); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); @@ -1731,13 +1754,13 @@ always = false # quiet, please assert_eq!( outside.prompts()[0], - "Harness for every Run's sessions, claude (not installed) or codex [codex]: " + "Harness for every Run's sessions, claude (not installed) or codex or grok (not installed) [codex]: " ); assert_eq!( outside.said()[..2], [ "claude is not installed: it isn't on PATH.", - "Choose claude or codex." + "Choose claude or codex or grok." ] ); assert_eq!( @@ -1747,7 +1770,7 @@ always = false # quiet, please } #[test] - fn with_neither_harness_installed_the_harness_settings_are_left_as_they_are() { + fn with_no_harness_installed_the_harness_settings_are_left_as_they_are() { let mut outside = Scripted { installed: Vec::new(), user_config: Some("[harness.claude]\nmodel = \"opus\"\n".to_string()), @@ -1760,8 +1783,8 @@ always = false # quiet, please outside.said(), [ "Harness for every Run's sessions: claude (not installed) or codex (not \ - installed). Sessions can run on neither here, so the harness settings stay as \ - they are; install claude or codex, then rerun `thirdshift setup`." + installed) or grok (not installed). Sessions can run on none here, so the harness \ + settings stay as they are; install claude or codex or grok, then rerun `thirdshift setup`." ] ); let config = table(&outside); @@ -1888,6 +1911,55 @@ always = false # quiet, please ); } + #[test] + fn setup_offers_grok_when_it_is_the_only_installed_harness_and_checks_its_model_and_effort() { + let mut outside = Scripted { + installed: vec![Harness::Grok], + ..Scripted::answering(&then_nothing_else(&[ + (HARNESS, ""), + ("Model for grok", "bogus"), + ("Model for grok", "GROK-4.5"), + ("Effort for grok", "xhigh"), + ("Effort for grok", "High"), + ])) + }; + + setup(&mut outside).unwrap(); + + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("grok")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("high")); + let said = outside.said().join("\n"); + assert!( + said.contains("choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5"), + "{said}" + ); + assert!(said.contains("choose one of high, medium, low"), "{said}"); + assert!(outside.test_calls().is_empty()); + } + + #[test] + fn when_groks_catalog_cannot_be_read_setup_preserves_the_harness_settings() { + let mut outside = Scripted { + grok_catalog: Err("catalog unavailable"), + user_config: Some("[harness]\ndefault = \"codex\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n".to_string()), + ..Scripted::answering(&then_nothing_else(&[(HARNESS, "grok")])) + }; + setup(&mut outside).unwrap(); + let config = table(&outside); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!( + config["harness"]["grok"]["model"].as_str(), + Some("grok-4.5") + ); + assert_eq!(config["harness"]["grok"]["effort"].as_str(), Some("medium")); + assert!(outside.said().join("\n").contains("once grok models works")); + } + #[test] fn codex_is_offered_with_its_models_and_the_chosen_models_efforts() { let mut outside = Scripted::answering(&then_nothing_else(&[ diff --git a/src/setup/questions.rs b/src/setup/questions.rs index 4b0d489c2..af44b79e1 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -165,7 +165,7 @@ pub fn ask( /// with those `current` sets for it as the defaults, checked as a Run /// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one -/// installed, so `claude` when both are. With no +/// installed, so `claude` when it is. With no /// Harness to choose, or Codex chosen and its catalog unreadable, the answer /// is `None`. fn ask_harness( @@ -194,7 +194,7 @@ fn ask_harness( let Some(default) = default else { let names = harness::names(); outside.say(format!( - "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ + "Harness for every Run's sessions: {harnesses}. Sessions can run on none here, so \ the harness settings stay as they are; install {names}, then rerun \ `thirdshift setup`." )); diff --git a/tests/fakes.rs b/tests/fakes.rs index 496dfef4d..33d132a8b 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -19,6 +19,8 @@ mod claude; mod codex; #[path = "fakes/gh.rs"] mod gh; +#[path = "fakes/grok.rs"] +mod grok; #[path = "fakes/json.rs"] mod json; @@ -111,6 +113,7 @@ fn main() { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), "codex" => codex::main(args.collect()), + "grok" => grok::main(args.collect()), "detached-command" => { let command = args.next().expect("no detached command"); let mut command = Command::new(command); diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 4afee8063..3fb7c7a9e 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -49,7 +49,7 @@ const CATALOG: &str = r#"{"models": [ ]}"#; /// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. -fn stdin_is_null() -> bool { +pub fn stdin_is_null() -> bool { let stdin = std::io::stdin() .as_fd() .try_clone_to_owned() @@ -64,7 +64,7 @@ fn stdin_is_null() -> bool { /// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts /// them until its script ends. A caught signal, unlike an ignored one, is /// back to its default in the script, which can trap it. -fn outlast_interrupts() { +pub fn outlast_interrupts() { // From the C library, which the standard library links: SIGINT and // SIGTERM are 2 and 15 on Linux and macOS alike. unsafe extern "C" { diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs new file mode 100644 index 000000000..94dd5b03d --- /dev/null +++ b/tests/fakes/grok.rs @@ -0,0 +1,131 @@ +//! Fake Grok Build: its text catalog, cache and Claude-shaped Messages stream. + +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::Command; + +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::codex::{outlast_interrupts, stdin_is_null}; +use crate::json::{Array, Bool, Null, object, string}; + +pub const MODELS: &str = include_str!("../fixtures/grok-models.txt"); +const CACHE: &str = include_str!("../fixtures/grok-models.json"); + +pub fn main(argv: Vec) { + let catalog = argv.first().is_some_and(|arg| arg == "models"); + if !catalog && !argv.iter().any(|arg| arg == "-p") { + crate::die("fake grok: expected models or -p", 2); + } + let record_path = crate::env_path("FAKE_GROK_RECORD"); + let lock = crate::lock_beside(&record_path); + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let cwd = std::env::current_dir().unwrap(); + let records = crate::append_record( + &record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ( + "prompt", + if catalog { + Null + } else { + argv.last().map(string).unwrap_or(Null) + }, + ), + ("cwd", string(cwd.to_str().unwrap())), + ("branch", string(branch)), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ("skill_files", skills_snapshot(".agents/skills")), + ("beside_skills", beside_skills_snapshot(".agents/skills")), + ( + "GROK_DISABLE_AUTOUPDATER", + string(std::env::var("GROK_DISABLE_AUTOUPDATER").unwrap_or_default()), + ), + ( + "GROK_FOLDER_TRUST", + string(std::env::var("GROK_FOLDER_TRUST").unwrap_or_default()), + ), + ]), + ); + let sessions: Vec<_> = records + .items() + .iter() + .filter(|call| call.at("prompt") != &Null) + .cloned() + .collect(); + drop(lock); + if catalog { + let home = crate::env_path("HOME"); + let cache = home.join(".grok/models_cache.json"); + fs::create_dir_all(cache.parent().unwrap()).unwrap(); + fs::write(cache, CACHE).unwrap(); + println!("{MODELS}"); + crate::exit(0); + } + outlast_interrupts(); + let session = sessions.len(); + let script = script_for(&sessions); + let id = argv + .iter() + .position(|arg| arg == "-r") + .and_then(|at| argv.get(at + 1)) + .cloned() + .unwrap_or_else(|| format!("fake-grok-{session}")); + println!( + "{}", + object([ + ("type", string("system")), + ("subtype", string("init")), + ("session_id", string(&id)), + ("cwd", string(cwd.to_str().unwrap())), + ]) + ); + std::io::stdout().flush().unwrap(); + let final_message = format!("{}.final-message.{session}", record_path.display()); + let after_result = format!("{}.after-result.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_CLAUDE_AFTER_RESULT", &after_result) + .status() + .unwrap(); + let code = exit_code(status); + println!( + "{}", + object([ + ("type", string("result")), + ( + "subtype", + string(if code == 0 { + "success" + } else { + "error_during_execution" + }) + ), + ("is_error", Bool(code != 0)), + ("session_id", string(id)), + ( + "result", + string(fs::read_to_string(&final_message).unwrap_or_default()) + ), + ("num_turns", crate::json::number(2)), + ("total_cost_usd", crate::json::number(0.0127)), + ( + "usage", + object([ + ("input_tokens", crate::json::number(812)), + ("output_tokens", crate::json::number(210)), + ("cache_read_input_tokens", crate::json::number(0)), + ]) + ), + ]) + ); + if let Ok(lines) = fs::read_to_string(Path::new(&after_result)) { + print!("{lines}"); + } + crate::exit(code); +} diff --git a/tests/fixtures/grok-models.json b/tests/fixtures/grok-models.json new file mode 100644 index 000000000..7787a2d15 --- /dev/null +++ b/tests/fixtures/grok-models.json @@ -0,0 +1,6 @@ +{"models": { + "grok-4.7": {"info": {"id": "grok-4.7", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.7-build-fast": {"info": {"id": "grok-4.7-build-fast", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.6": {"info": {"id": "grok-4.6", "reasoning_efforts": [{"id": "xhigh", "value": "xhigh"}, {"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}}, + "grok-4.5": {"info": {"id": "grok-4.5", "reasoning_efforts": [{"id": "high", "value": "high"}, {"id": "medium", "value": "medium"}, {"id": "low", "value": "low"}]}} +}} diff --git a/tests/fixtures/grok-models.txt b/tests/fixtures/grok-models.txt new file mode 100644 index 000000000..2a84b9e2b --- /dev/null +++ b/tests/fixtures/grok-models.txt @@ -0,0 +1,9 @@ +You are logged in with grok.com. + +Default model: grok-4.7 + +Available models: + * grok-4.7 (default) + - grok-4.7-build-fast + - grok-4.6 + - grok-4.5 diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs new file mode 100644 index 000000000..05f38d606 --- /dev/null +++ b/tests/grok_sessions.rs @@ -0,0 +1,261 @@ +//! Commands on Grok Build through the scenario harness. + +mod support; + +use support::Scenario; + +const AGENT_OPENS_PR: &str = r#" +echo feature > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +#[test] +fn a_run_can_choose_grok_build() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert!(result.stderr.contains("sessions run on grok")); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_invalid_model_or_effort_fails_before_any_work_and_names_the_valid_choices() { + for (model, effort, error) in [ + ( + "grok-9.9-bogus", + "high", + "the Model grok-9.9-bogus is not in Grok Build's catalog: choose one of grok-4.7, grok-4.7-build-fast, grok-4.6, grok-4.5", + ), + ( + "grok-4.5", + "xhigh", + "the Effort xhigh is not one the Grok Build Model grok-4.5 supports: choose one of high, medium, low", + ), + ( + "grok-4.7", + "max", + "the Effort max is not one the Grok Build Model grok-4.7 supports: choose one of xhigh, high, medium, low", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[ + "harness", + "grok", + "model", + model, + "effort", + effort, + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!(result.stderr.contains(error), "{}", result.stderr); + assert_eq!(scenario.entries("work"), ["widgets"]); + assert!(scenario.issue_labels(7).is_empty()); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists() + ); + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0]["argv"], serde_json::json!(["models"])); + assert_eq!(calls[0]["stdin_null"], true); + assert_eq!(calls[0]["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(calls[0]["GROK_FOLDER_TRUST"], "0"); + } +} + +#[test] +fn grok_sessions_are_unattended_load_the_factory_skill_and_report_tools_and_usage() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, &format!(r#" +echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"run_terminal_command","input":{{"command":"cargo test"}}}}]}}}}' +{AGENT_OPENS_PR} +"#)); + let url = scenario.issue_url(7); + let result = scenario.run_with_env( + &[ + "harness", "grok", "model", "GROK-4.7", "effort", "High", &url, + ], + &[ + ("GROK_DISABLE_AUTOUPDATER", "0"), + ("GROK_FOLDER_TRUST", "1"), + ], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + for line in [ + "implement: $ cargo test", + "2 turns, $0.01", + "sessions run on grok · grok-4.7 · high", + ] { + assert!( + result.stderr.contains(line), + "missing {line:?}: {}", + result.stderr + ); + } + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 2); + let session = &calls[1]; + let prompt = session["prompt"].as_str().unwrap(); + assert_eq!( + prompt.lines().next().unwrap(), + format!("/thirdshift-implement {url}") + ); + assert_eq!( + session["argv"], + serde_json::json!([ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "high", + prompt + ]) + ); + assert_eq!(session["git_status"], ""); + for call in calls { + assert_eq!(call["stdin_null"], true); + assert_eq!(call["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(call["GROK_FOLDER_TRUST"], "0"); + } + scenario.assert_every_grok_session_found_the_factory_skills(); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + assert!(log.contains("fake-grok-1")); + assert!(log.contains("\"type\": \"result\"")); + assert!( + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .contains("Built with grok · grok-4.7 · high") + ); +} + +#[test] +fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { + for (event, exit, cause) in [ + ( + r#"{"type":"error","message":"backend unavailable"}"#, + "exit 1", + "grok exited 1: backend unavailable", + ), + ( + r#"{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["quota exceeded"]}"#, + "", + "grok's turn failed: quota exceeded", + ), + ] { + let scenario = Scenario::new(); + let script = if exit.is_empty() { + format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_CLAUDE_AFTER_RESULT\"\n") + } else { + format!("echo '{event}'\n{exit}") + }; + scenario.agent_does_for(7, &script); + let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(cause), "{}", result.stderr); + } +} + +#[test] +fn a_resume_uses_the_streams_id_and_repeats_the_unattended_flags_and_environment() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!(r#" +{AGENT_OPENS_PR} +echo '{{"type":"system","subtype":"task_started","task_id":"task-1","description":"cargo test"}}' +echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status":"stopped"}}' > "$FAKE_CLAUDE_AFTER_RESULT" +"#)); + let result = scenario.run(&[ + "harness", + "grok", + "model", + "grok-4.7", + "effort", + "low", + &scenario.issue_url(7), + ]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let calls = scenario.grok_calls(); + assert_eq!(calls.len(), 3); + let prompt = calls[2]["prompt"].as_str().unwrap(); + assert!(prompt.starts_with("Your background work (cargo test) was killed")); + assert_eq!( + calls[2]["argv"], + serde_json::json!([ + "-p", + "--always-approve", + "--output-format", + "streaming-messages-json", + "-m", + "grok-4.7", + "--reasoning-effort", + "low", + "-r", + "fake-grok-1", + prompt + ]) + ); + for call in &calls { + assert_eq!(call["stdin_null"], true); + assert_eq!(call["GROK_DISABLE_AUTOUPDATER"], "1"); + assert_eq!(call["GROK_FOLDER_TRUST"], "0"); + } + scenario.assert_every_grok_session_found_the_factory_skills(); +} + +#[test] +fn the_user_config_can_choose_grok_and_its_own_model_and_effort() { + let scenario = Scenario::new(); + scenario.user_config_is("[harness]\ndefault = \"grok\"\n[harness.grok]\nmodel = \"grok-4.5\"\neffort = \"medium\"\n"); + scenario.agent_does_for(7, AGENT_OPENS_PR); + let result = scenario.run(&[&scenario.issue_url(7)]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("sessions run on grok · grok-4.5 · medium") + ); + assert!(scenario.claude_calls().is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn an_architecture_reviews_final_message_reaches_the_command() { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does( + r#" +idea=$(gh issue create --title "Deepen sessions" --body "An idea" --label needs-triage) +printf 'Architecture review idea: %s\n' "$idea" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let result = scenario.run(&["architect", "harness", "grok"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + scenario + .issue_labels(8) + .contains(&"architect-idea".to_string()) + ); + let calls = scenario.grok_calls(); + assert_eq!( + calls[1]["prompt"].as_str().unwrap().lines().next().unwrap(), + "/thirdshift-improve-codebase-architecture" + ); + scenario.assert_every_grok_session_found_the_factory_skills(); +} diff --git a/tests/process_tree.rs b/tests/process_tree.rs index 2c898e7dc..7c4cb606b 100644 --- a/tests/process_tree.rs +++ b/tests/process_tree.rs @@ -45,6 +45,11 @@ fn interrupting_a_codex_run_stops_a_command_in_its_own_session() { assert_detached_command_stopped("codex"); } +#[test] +fn interrupting_a_grok_run_stops_a_command_in_its_own_session() { + assert_detached_command_stopped("grok"); +} + fn assert_detached_command_stopped(harness: &str) { let scenario = Scenario::new(); let pid = scenario.path("detached-pid"); diff --git a/tests/setup.rs b/tests/setup.rs index f830ad39c..d89797123 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -52,7 +52,8 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[pickup]", "[harness]", "[harness.claude]", - "[harness.codex]" + "[harness.codex]", + "[harness.grok]" ], "{text}" ); @@ -70,7 +71,7 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); assert_eq!(config["harness"]["default"].as_str(), Some("claude")); - for harness in ["claude", "codex"] { + for harness in ["claude", "codex", "grok"] { for key in ["model", "effort"] { assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); } diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 2de6a51e2..db7f25d32 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -11,17 +11,20 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 5] = [ +const SOURCES: [&str; 8] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), include_str!("../fakes/codex.rs"), include_str!("../fakes/json.rs"), + include_str!("../fakes/grok.rs"), + include_str!("../fixtures/grok-models.json"), + include_str!("../fixtures/grok-models.txt"), ]; /// Put the fake `gh`, `claude` and `codex` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude", "codex", "detached-command"] { + for name in ["gh", "claude", "codex", "grok", "detached-command"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index f593337b0..df6a8fdd9 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -603,6 +603,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_CLAUDE_SCRIPT", self.path("claude-script.sh")) .env("FAKE_CLAUDE_RECORD", self.path("claude-calls.json")) .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) + .env("FAKE_GROK_RECORD", self.path("grok-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, @@ -815,6 +816,23 @@ test -f {root}/{COPY_REPLACED} } } + /// Every call the fake received on Grok, including its catalog check. + pub fn grok_calls(&self) -> Vec { + match fs::read_to_string(self.path("grok-calls.json")) { + Ok(text) => serde_json::from_str(&text).unwrap(), + Err(_) => Vec::new(), + } + } + + pub fn assert_every_grok_session_found_the_factory_skills(&self) { + assert_found_the_factory_skills( + self.grok_calls() + .into_iter() + .filter(|call| call["prompt"].is_string()) + .collect(), + ); + } + /// Assert every `claude` call found every Factory skill, by its /// `thirdshift-` name, in its worktree's `.claude/skills/`. pub fn assert_every_session_found_the_factory_skills(&self) { From ab32200b11a6c9714c1d06cd8f25819490370490 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:07:58 -0400 Subject: [PATCH 2/3] Address Grok Harness Standards and Spec findings --- README.md | 2 +- src/harness/codex.rs | 35 +++++++++++++++++++-------------- src/harness/grok.rs | 40 ++++++++++++++++++++++---------------- src/harness/grok/stream.rs | 14 +++++++++++-- src/main.rs | 2 +- src/setup/questions.rs | 20 ++++++++++++++++++- tests/fakes/grok.rs | 9 +++++++++ tests/grok_sessions.rs | 9 +++++++-- 8 files changed, 92 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 74181775f..91ebc0980 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,7 @@ For each of the three, the command wins, then the [User config](#user-config), t With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume `, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. -With `harness grok`, every session runs `grok -p --always-approve --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag is always explicit. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. +With `harness grok`, every session runs `grok -p --always-approve --sandbox off --output-format streaming-messages-json` with stdin set to null and with `GROK_DISABLE_AUTOUPDATER=1` and `GROK_FOLDER_TRUST=0`. The approval flag and disabled sandbox are always explicit, overriding the CLI’s own config and `GROK_SANDBOX`. No repository needs a one-time `grok --trust`: the trust environment override allows its project instructions and skills to load in the Run's worktree. The Factory skills are linked into `.agents/skills/`, excluded from git, and the prompt loads its skill as `/thirdshift-` on its first line. A Resume repeats the flags and environment with `-r `. Grok's Messages stream shares Claude's event shapes, with Grok's tool names and errors; it supplies progress, the session id, final message, turns and cost, or token usage when cost is unavailable. Grok starts shell commands in their own process groups, which thirdshift's tree-stop also stops; the CLI gets SIGTERM before SIGKILL. Grok reads both `AGENTS.md` and `CLAUDE.md` when both exist, along with `AGENT.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md` and project rules under `.grok`, `.claude` and `.cursor`. It also reads `~/.grok/rules`, `~/.claude/CLAUDE.md` and `~/.claude/rules`; thirdshift writes none of these user-level files. Grok Build uses `~/.grok` for sign-in and sessions (or `GROK_HOME` when set). diff --git a/src/harness/codex.rs b/src/harness/codex.rs index 734b15a3d..c65dc837c 100644 --- a/src/harness/codex.rs +++ b/src/harness/codex.rs @@ -11,7 +11,7 @@ use anyhow::Result; use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings}; use crate::progress::{self, Stream}; -use crate::setup::{Outside, questions::ask_setting}; +use crate::setup::{Outside, questions::ask_checked_setting}; pub struct Codex; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Codex, &Codex); @@ -135,13 +135,13 @@ fn ask_codex( Codex.product_name(), catalog.listed().join(", ") )); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Codex, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; + let model = ask_checked_setting( + outside, + "Model", + Harness::Codex, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; let efforts = catalog.efforts(model.as_deref()).join(", "); outside.say(match &model { Some(model) => format!("Efforts {model} supports: {efforts}"), @@ -150,13 +150,18 @@ fn ask_codex( Codex.product_name() ), }); - loop { - let effort = ask_setting(outside, "Effort", Harness::Codex, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(settled), - Err(error) => outside.say(format!("{error:#}")), - } - } + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Codex, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(ModelAndEffort { model, effort }) } /// `prompt` as Codex takes it: a first line that loads a Factory skill, diff --git a/src/harness/grok.rs b/src/harness/grok.rs index 442652d42..1c81b8ccd 100644 --- a/src/harness/grok.rs +++ b/src/harness/grok.rs @@ -11,7 +11,7 @@ use anyhow::Result; use super::adapter::{Adapter, Invocation, SkillLoading}; use super::{Choice, Harness, ModelAndEffort, Settings}; use crate::progress::Stream; -use crate::setup::{Outside, questions::ask_setting}; +use crate::setup::{Outside, questions::ask_checked_setting}; pub struct Grok; pub const REGISTRATION: (Harness, &dyn Adapter) = (Harness::Grok, &Grok); @@ -46,8 +46,9 @@ impl Adapter for Grok { } fn session(&self, choice: &Choice, resume: Option<&str>, prompt: &str) -> Invocation { let mut args: Vec = [ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", ] @@ -62,7 +63,7 @@ impl Adapter for Grok { if let Some(id) = resume { args.extend(["-r".to_string(), id.to_string()]); } - args.push(self.skill_loading().prompt(prompt)); + args.extend(["-p".to_string(), self.skill_loading().prompt(prompt)]); Invocation { args, stdin: None } } fn check(&self, choice: &mut Choice) -> Result<()> { @@ -89,25 +90,30 @@ impl Adapter for Grok { self.product_name(), catalog.listed().join(", ") )); - let model = loop { - let model = ask_setting(outside, "Model", Harness::Grok, current.model.as_deref())?; - match catalog.settle(model.as_deref(), None) { - Ok(settled) => break settled.model, - Err(error) => outside.say(format!("{error:#}")), - } - }; + let model = ask_checked_setting( + outside, + "Model", + Harness::Grok, + current.model.as_deref(), + |model| catalog.settle(model, None).map(|settled| settled.model), + )?; outside.say(format!( "Efforts {} supports: {}", model.as_deref().unwrap_or("Grok Build's default Model"), catalog.efforts(model.as_deref()).join(", ") )); - loop { - let effort = ask_setting(outside, "Effort", Harness::Grok, current.effort.as_deref())?; - match catalog.settle(model.as_deref(), effort.as_deref()) { - Ok(settled) => return Ok(Some(settled)), - Err(error) => outside.say(format!("{error:#}")), - } - } + let effort = ask_checked_setting( + outside, + "Effort", + Harness::Grok, + current.effort.as_deref(), + |effort| { + catalog + .settle(model.as_deref(), effort) + .map(|settled| settled.effort) + }, + )?; + Ok(Some(ModelAndEffort { model, effort })) } fn stream(&self, _worktree: &Path) -> Box { Box::new(stream::GrokProgress::default()) diff --git a/src/harness/grok/stream.rs b/src/harness/grok/stream.rs index 59a000cc7..a172ed6e5 100644 --- a/src/harness/grok/stream.rs +++ b/src/harness/grok/stream.rs @@ -12,6 +12,7 @@ pub struct GrokProgress { error: Option, session_id: Option, usage: Option, + cost_known: bool, } impl Stream for GrokProgress { @@ -28,6 +29,11 @@ impl Stream for GrokProgress { self.error = event["message"].as_str().map(String::from); } Some("result") => { + // Messages streams use zero as a placeholder for unknown cost. + self.cost_known = event["total_cost_usd"] + .as_f64() + .is_some_and(|cost| cost > 0.0); + self.usage = None; self.failed = event["is_error"] == true || event["subtype"] != "success"; let errors: Vec<_> = event["errors"] .as_array() @@ -82,7 +88,11 @@ impl Stream for GrokProgress { self.claude.condense(&event.to_string()) } fn summary(&self) -> Option { - self.claude.summary().or_else(|| self.usage.clone()) + if self.cost_known { + self.claude.summary().or_else(|| self.usage.clone()) + } else { + self.usage.clone() + } } fn final_message(&self) -> Option<&str> { self.claude.final_message() @@ -124,7 +134,7 @@ mod tests { #[test] fn usage_without_a_complete_cost_and_a_result_without_init_are_still_read() { let mut stream = GrokProgress::default(); - stream.condense(r#"{"type":"result","subtype":"success","result":"done","usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); + stream.condense(r#"{"type":"result","subtype":"success","result":"done","num_turns":1,"total_cost_usd":0,"usage":{"input_tokens":812,"output_tokens":210,"cache_read_input_tokens":45,"cache_creation_input_tokens":12},"session_id":"abc123"}"#); assert_eq!(stream.session_id(), Some("abc123")); assert_eq!( stream.summary().as_deref(), diff --git a/src/main.rs b/src/main.rs index 2f790f7cf..e3809357f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -90,7 +90,7 @@ Before any work, the Harness's CLI must be on PATH. On claude, a named Model get test call with its Effort, which must succeed. On codex, a named Model and Effort must be in codex debug models, matched regardless of case, a Model by slug or display name, and are passed on as Codex names them. Grok Build checks grok models and its refreshed effort cache, -then runs grok -p --always-approve --output-format streaming-messages-json with null stdin, +then runs grok -p --always-approve --sandbox off --output-format streaming-messages-json with null stdin, GROK_DISABLE_AUTOUPDATER=1 and GROK_FOLDER_TRUST=0. Resume uses -r . A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json diff --git a/src/setup/questions.rs b/src/setup/questions.rs index af44b79e1..bfabbe708 100644 --- a/src/setup/questions.rs +++ b/src/setup/questions.rs @@ -166,7 +166,7 @@ pub fn ask( /// checks them, through the chosen adapter. The Harness's /// default is the `current` one if it's installed, else the first one /// installed, so `claude` when it is. With no -/// Harness to choose, or Codex chosen and its catalog unreadable, the answer +/// Harness to choose, or a chosen Harness with an unreadable catalog, the answer /// is `None`. fn ask_harness( outside: &mut impl Outside, @@ -240,6 +240,24 @@ pub(crate) fn ask_setting( Ok(Some(answer).filter(|answer| !answer.is_empty() && answer != "-")) } +/// Ask one catalog-backed setting, retrying refusals with the catalog's +/// valid choices and returning the name the Harness takes. +pub(crate) fn ask_checked_setting( + outside: &mut (impl Outside + ?Sized), + setting: &str, + harness: Harness, + current: Option<&str>, + settle: impl Fn(Option<&str>) -> Result>, +) -> Result> { + loop { + let answer = ask_setting(outside, setting, harness, current)?; + match settle(answer.as_deref()) { + Ok(settled) => return Ok(settled), + Err(error) => outside.say(format!("{error:#}")), + } + } +} + /// Ask `question` until the answer is yes, no or nothing, which is `default`. fn yes_or_no(outside: &mut impl Outside, question: &str, default: bool) -> Result { let choices = if default { "[Y/n]" } else { "[y/N]" }; diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs index 94dd5b03d..a9cb20612 100644 --- a/tests/fakes/grok.rs +++ b/tests/fakes/grok.rs @@ -17,6 +17,15 @@ pub fn main(argv: Vec) { if !catalog && !argv.iter().any(|arg| arg == "-p") { crate::die("fake grok: expected models or -p", 2); } + if !catalog { + let prompt = argv + .iter() + .position(|arg| arg == "-p") + .and_then(|at| argv.get(at + 1)); + if prompt.is_none_or(|prompt| prompt.starts_with('-')) { + crate::die("a value is required for --single ", 2); + } + } let record_path = crate::env_path("FAKE_GROK_RECORD"); let lock = crate::lock_beside(&record_path); let (branch, _) = git_here(&["branch", "--show-current"]); diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs index 05f38d606..b8966dc85 100644 --- a/tests/grok_sessions.rs +++ b/tests/grok_sessions.rs @@ -87,6 +87,7 @@ echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"ru &[ ("GROK_DISABLE_AUTOUPDATER", "0"), ("GROK_FOLDER_TRUST", "1"), + ("GROK_SANDBOX", "workspace"), ], ); assert_eq!(result.code, Some(0), "{}", result.stderr); @@ -112,14 +113,16 @@ echo '{{"type":"assistant","message":{{"content":[{{"type":"tool_use","name":"ru assert_eq!( session["argv"], serde_json::json!([ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", "-m", "grok-4.7", "--reasoning-effort", "high", + "-p", prompt ]) ); @@ -198,8 +201,9 @@ echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status assert_eq!( calls[2]["argv"], serde_json::json!([ - "-p", "--always-approve", + "--sandbox", + "off", "--output-format", "streaming-messages-json", "-m", @@ -208,6 +212,7 @@ echo '{{"type":"system","subtype":"task_notification","task_id":"task-1","status "low", "-r", "fake-grok-1", + "-p", prompt ]) ); From 78403c0797d64ef3563eaccbc5f27a55187df007 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Tue, 6 Oct 2026 13:23:58 -0400 Subject: [PATCH 3/3] Emit one terminal result in Grok failure scenarios --- README.md | 2 +- tests/fakes/grok.rs | 70 +++++++++++++++++++++--------------------- tests/grok_sessions.rs | 14 ++++++++- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e6c92a6cd..154f803dc 100644 --- a/README.md +++ b/README.md @@ -884,7 +884,7 @@ Editing a skill in `skills/` has no effect until you rebuild and reinstall ([ADR The [Prompts and skills page](https://thirdshift.app/prompts/) is generated from the prompts, the `claude` arguments and the skills, and each Session prompt is also generated as a Markdown file in [`prompts/`](prompts/), with a [`README.md`](prompts/README.md) that lists them with the `claude` command lines. `cargo test` fails until they are regenerated after a change to any of them. Regenerate the page and `prompts/` with `UPDATE_PROMPTS=1 cargo test prompts_page`, which also deletes any file in `prompts/` that it doesn't generate. -The integration tests swap in a fake `gh`, `claude`, `codex` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. +The integration tests swap in a fake `gh`, `claude`, `codex`, `agy` and `grok`, from [`tests/fakes.rs`](tests/fakes.rs). The first test that needs them builds them with `rustc` (the one on `PATH`, or `$RUSTC`), so the suite needs nothing else on `PATH` beyond `git` and `bash`. ### The Rust version diff --git a/tests/fakes/grok.rs b/tests/fakes/grok.rs index a9cb20612..76b7b6d3a 100644 --- a/tests/fakes/grok.rs +++ b/tests/fakes/grok.rs @@ -2,7 +2,6 @@ use std::fs; use std::io::Write; -use std::path::Path; use std::process::Command; use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; @@ -94,47 +93,48 @@ pub fn main(argv: Vec) { ); std::io::stdout().flush().unwrap(); let final_message = format!("{}.final-message.{session}", record_path.display()); - let after_result = format!("{}.after-result.{session}", record_path.display()); + let result = format!("{}.result.{session}", record_path.display()); let status = Command::new("bash") .arg("-e") .arg(script) .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) - .env("FAKE_CLAUDE_AFTER_RESULT", &after_result) + .env("FAKE_GROK_RESULT", &result) .status() .unwrap(); let code = exit_code(status); - println!( - "{}", - object([ - ("type", string("result")), - ( - "subtype", - string(if code == 0 { - "success" - } else { - "error_during_execution" - }) - ), - ("is_error", Bool(code != 0)), - ("session_id", string(id)), - ( - "result", - string(fs::read_to_string(&final_message).unwrap_or_default()) - ), - ("num_turns", crate::json::number(2)), - ("total_cost_usd", crate::json::number(0.0127)), - ( - "usage", - object([ - ("input_tokens", crate::json::number(812)), - ("output_tokens", crate::json::number(210)), - ("cache_read_input_tokens", crate::json::number(0)), - ]) - ), - ]) - ); - if let Ok(lines) = fs::read_to_string(Path::new(&after_result)) { - print!("{lines}"); + if let Ok(result) = fs::read_to_string(result) { + print!("{result}"); + } else { + println!( + "{}", + object([ + ("type", string("result")), + ( + "subtype", + string(if code == 0 { + "success" + } else { + "error_during_execution" + }) + ), + ("is_error", Bool(code != 0)), + ("session_id", string(id)), + ( + "result", + string(fs::read_to_string(&final_message).unwrap_or_default()) + ), + ("num_turns", crate::json::number(2)), + ("total_cost_usd", crate::json::number(0.0127)), + ( + "usage", + object([ + ("input_tokens", crate::json::number(812)), + ("output_tokens", crate::json::number(210)), + ("cache_read_input_tokens", crate::json::number(0)), + ]) + ), + ]) + ); } crate::exit(code); } diff --git a/tests/grok_sessions.rs b/tests/grok_sessions.rs index 9e9047dd1..71a149a93 100644 --- a/tests/grok_sessions.rs +++ b/tests/grok_sessions.rs @@ -165,7 +165,7 @@ fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { ] { let scenario = Scenario::new(); let script = if exit.is_empty() { - format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_CLAUDE_AFTER_RESULT\"\n") + format!("{AGENT_OPENS_PR}\necho '{event}' > \"$FAKE_GROK_RESULT\"\n") } else { format!("echo '{event}'\n{exit}") }; @@ -173,6 +173,18 @@ fn groks_error_is_in_the_failure_cause_even_when_the_cli_exits_zero() { let result = scenario.run(&["harness", "grok", &scenario.issue_url(7)]); assert_eq!(result.code, Some(1), "{}", result.stderr); assert!(result.stderr.contains(cause), "{}", result.stderr); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/sessions"); + let log = std::fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/sessions/{}", + logs[0] + ))) + .unwrap(); + let results = log + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["type"] == "result") + .count(); + assert_eq!(results, 1, "{log}"); } }