From 479a1fb355138347de07c72ba19804e7e6f87cc5 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 5 Oct 2026 00:06:59 -0400 Subject: [PATCH 01/14] Load the Factory skills from the worktree as thirdshift- (#391) The Factory skills stop being a Claude plugin. Each is renamed thirdshift- in skills/, and names the others in plain prose. thirdshift writes them out once per Command and links each into the worktree's .claude/skills/, kept out of git by a /.claude/skills/thirdshift-* entry in .git/info/exclude, added once. No session gets --plugin-dir, and each prompt's first line is /thirdshift-. Restores resolving-merge-conflicts, dropped from skills/ by a skills update though the conflict Repair prompts still invoke it. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +- .../0001-rust-binary-with-embedded-skills.md | 2 +- prompts/README.md | 8 +- prompts/architecture-review.md | 6 +- prompts/conflict-repair.md | 2 +- prompts/continuation-pr.md | 6 +- prompts/continuation.md | 6 +- prompts/foreign-conflict-repair.md | 2 +- prompts/fresh.md | 6 +- prompts/review-repair.md | 4 +- prompts/spec-review.md | 8 +- site/index.html | 12 +- site/prompts/index.html | 176 +++++++++--------- .../SKILL.md | 2 +- .../agents/openai.yaml | 0 .../DEEPENING.md | 0 .../DESIGN-IT-TWICE.md | 0 .../SKILL.md | 2 +- .../agents/openai.yaml | 0 .../SKILL.md | 6 +- .../agents/openai.yaml | 0 .../REPORT.md | 6 +- .../SKILL.md | 12 +- .../agents/openai.yaml | 0 skills/{pr => thirdshift-pr}/CREDITS.md | 0 skills/{pr => thirdshift-pr}/SKILL.md | 2 +- .../{pr => thirdshift-pr}/agents/openai.yaml | 0 .../SKILL.md | 14 ++ .../agents/openai.yaml | 3 + skills/{tdd => thirdshift-tdd}/SKILL.md | 6 +- .../agents/openai.yaml | 0 skills/{tdd => thirdshift-tdd}/mocking.md | 0 skills/{tdd => thirdshift-tdd}/tests.md | 0 .../{to-spec => thirdshift-to-spec}/SKILL.md | 2 +- .../agents/openai.yaml | 0 .../SKILL.md | 2 +- .../agents/openai.yaml | 0 src/architect.rs | 2 +- src/command.rs | 10 +- src/main.rs | 2 +- src/plugin.rs | 49 ----- src/progress.rs | 4 +- src/prompt.rs | 34 ++-- src/prompts_page.rs | 33 ++-- src/run.rs | 4 +- src/session.rs | 74 +++----- src/skills.rs | 113 +++++++++++ tests/architect_run.rs | 14 +- tests/background_work.rs | 1 + tests/continuation.rs | 2 +- tests/fakes/claude.rs | 89 ++++++--- tests/fresh_run.rs | 99 ++++++---- tests/merge_run.rs | 6 +- tests/mergeable.rs | 5 +- tests/numbered_branches.rs | 2 +- tests/packaging.rs | 18 +- tests/pickup_run.rs | 2 +- tests/progress_lines.rs | 4 +- tests/spec_run.rs | 8 +- tests/support/mod.rs | 31 +++ 60 files changed, 540 insertions(+), 373 deletions(-) rename skills/{code-review => thirdshift-code-review}/SKILL.md (99%) rename skills/{code-review => thirdshift-code-review}/agents/openai.yaml (100%) rename skills/{codebase-design => thirdshift-codebase-design}/DEEPENING.md (100%) rename skills/{codebase-design => thirdshift-codebase-design}/DESIGN-IT-TWICE.md (100%) rename skills/{codebase-design => thirdshift-codebase-design}/SKILL.md (99%) rename skills/{codebase-design => thirdshift-codebase-design}/agents/openai.yaml (100%) rename skills/{implement => thirdshift-implement}/SKILL.md (66%) rename skills/{implement => thirdshift-implement}/agents/openai.yaml (100%) rename skills/{improve-codebase-architecture => thirdshift-improve-codebase-architecture}/REPORT.md (93%) rename skills/{improve-codebase-architecture => thirdshift-improve-codebase-architecture}/SKILL.md (89%) rename skills/{improve-codebase-architecture => thirdshift-improve-codebase-architecture}/agents/openai.yaml (100%) rename skills/{pr => thirdshift-pr}/CREDITS.md (100%) rename skills/{pr => thirdshift-pr}/SKILL.md (99%) rename skills/{pr => thirdshift-pr}/agents/openai.yaml (100%) create mode 100644 skills/thirdshift-resolving-merge-conflicts/SKILL.md create mode 100644 skills/thirdshift-resolving-merge-conflicts/agents/openai.yaml rename skills/{tdd => thirdshift-tdd}/SKILL.md (94%) rename skills/{tdd => thirdshift-tdd}/agents/openai.yaml (100%) rename skills/{tdd => thirdshift-tdd}/mocking.md (100%) rename skills/{tdd => thirdshift-tdd}/tests.md (100%) rename skills/{to-spec => thirdshift-to-spec}/SKILL.md (99%) rename skills/{to-spec => thirdshift-to-spec}/agents/openai.yaml (100%) rename skills/{to-tickets => thirdshift-to-tickets}/SKILL.md (99%) rename skills/{to-tickets => thirdshift-to-tickets}/agents/openai.yaml (100%) delete mode 100644 src/plugin.rs create mode 100644 src/skills.rs diff --git a/README.md b/README.md index d7fc5da02..125011842 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ From a clone of the issue's repository, `thirdshift `: Any failed check stops the Run before any work happens. 2. Picks the **Issue branch** (`issue-`, or `issue--branch-` once earlier ones are finished) and the **Base branch**, normally the branch you have checked out. Then it makes the [Claim](#the-claim): it labels the issue `in-progress`, in place of `ready-for-agent` if it has that. 3. Creates a git worktree next to your clone, named `-`, so your own checkout is never touched. -4. Runs a headless Claude Code session in that worktree with the **Factory skills** in [`skills/`](skills/) loaded, started with one of the **Session prompts** in [`prompts/`](prompts/). The agent implements the issue, reviews its work against the Base branch, addresses the **Standards findings** and **Spec findings** it agrees with, and opens a ready-for-review pull request that lists every **Unaddressed finding** with a reason and closes the issue. +4. Runs a headless Claude Code session in that worktree with the **Factory skills** in [`skills/`](skills/) linked into its `.claude/skills/` as `thirdshift-` and kept out of git by a `.git/info/exclude` entry ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)), started with one of the **Session prompts** in [`prompts/`](prompts/). The agent implements the issue, reviews its work against the Base branch, addresses the **Standards findings** and **Spec findings** it agrees with, and opens a ready-for-review pull request that lists every **Unaddressed finding** with a reason and closes the issue. 5. Takes over deterministically: pushes the Issue branch, skipping your repo's git hooks since the session runs the tests and CI gates the pull request, checks through `gh` that the pull request exists, is open and targets the Base branch, and marks it ready for review (`gh pr ready`) if the agent left it as a draft. 6. Keeps the pull request mergeable and green: merges the Base branch in and watches CI, starting a **Repair** session for a merge conflict or failing checks, at most 5 per Run. A failed check that also failed, under the same name, on the Base branch commit the head last merged in is an **Inherited failure**, not the branch's to fix: the CI-fix Repair is given only the branch's own failures, with the Inherited failures listed as not to fix. If every failed check is an Inherited failure, no Repair starts: the Run merges the Base branch again if it has moved since, and otherwise ends as a [Failed run](#failed-runs) that says to fix the Base branch first, unless it was given `base-fix` or the [User config](#user-config) sets `base.fix`, when it starts a [Base fix](#base-fix) first. A check that is pending, passed or missing on that Base branch commit is the branch's own, and thirdshift never triggers or waits for the Base branch's CI. If a CI-fix Repair leaves the head unchanged, with no commit of its own and no Base branch move to merge, thirdshift does a **Check re-run**, once for that head: it asks GitHub to re-run the branch's own failed checks (`gh run rerun --failed`, once per workflow run), says so on stderr as `re-running the failed checks on : [, …]`, and watches CI on that head again, from the new attempt on. A check that failed for a reason that does not repeat, such as a flaky test or a runner fault, then passes, and the Run goes on as for any green head. If CI is red again, the Run ends as a Failed run, with no second Repair or re-run for that head. It ends the same way, with nothing re-run, if one of those checks can't be re-run (a commit status, or a check run that is not a GitHub Actions job), since the head could not go green, or if GitHub refuses the re-run, which stderr shows. What the Repair concluded is never read, Inherited failures are never re-run on their own account, and a Check re-run is no Repair, so it counts against no budget ([ADR-0010](docs/adr/0010-one-check-re-run-before-a-declined-ci-fix.md)). If the Base branch moves while CI runs, it merges it again and goes round, at most 5 times per Run. In a Merge run, each round also takes in **Foreign commits** first: see [Foreign commits in a Merge run](#foreign-commits-in-a-merge-run). 7. In a **Merge run** (`thirdshift merge `), does the **Self-merge**: once the pull request is open, ready for review, mergeable and green, thirdshift merges it into the Base branch with a merge commit, on exactly the head commit whose CI it watched (`gh pr merge --merge --match-head-commit `). It never uses GitHub's auto-merge ([ADR-0004](docs/adr/0004-self-merge-by-thirdshift-not-github-auto-merge.md)). A merge that fails goes back round step 6, within the same budgets, and is tried again on the new green head. If that round finds nothing to fix, the refusal is a **policy refusal**, such as merge commits being disallowed or a review being required. After the merge, thirdshift deletes the Issue branch on `origin`, and closes the issue if it is still open, with the comment `Closed by #, merged into by a thirdshift Merge run.` GitHub closes it on its own only for a merge into the repository's default branch, and then thirdshift leaves it alone. Once the issue is closed, either way, thirdshift takes `in-progress` off it, [removing the Claim](#when-the-claim-ends). -8. Cleans up: removes the worktree, the local Issue branch and the temporary plugin directory, whether the Run succeeded or not. The one exception is a **Failed run** whose work could not be pushed: see below. +8. Cleans up: removes the worktree, the local Issue branch and the temporary directory the Factory skills were written out to, whether the Run succeeded or not. The one exception is a **Failed run** whose work could not be pushed: see below. ### The Claim @@ -108,7 +108,7 @@ Sessions run headless in Claude Code's auto mode (`claude -p --permission-mode a ### What sessions leave behind -Cleanup removes only thirdshift's own worktree, local Issue branch and temporary plugin directory. Anything else an agent does as your user stays. For example, one Run downloaded a JDK to `~/.local/jdk/` because the server had no Java, installed Playwright in `/tmp/pw`, and left a pull request body draft in `/tmp/`. This is by design, but worth knowing: +Cleanup removes only thirdshift's own worktree, local Issue branch and the temporary directory it wrote the Factory skills out to. The `.git/info/exclude` entry that keeps the linked skills out of git, `/.claude/skills/thirdshift-*`, is added once and left in place, as every worktree of the repository shares the file. Anything else an agent does as your user stays. For example, one Run downloaded a JDK to `~/.local/jdk/` because the server had no Java, installed Playwright in `/tmp/pw`, and left a pull request body draft in `/tmp/`. This is by design, but worth knowing: - **Install the target repository's toolchains up front** (language runtimes, package managers, test tools), so the agents don't download their own on every Run. - **For UI work, install the headless browser dependencies**, e.g. `libgbm1` on Ubuntu server, which headless Chromium needs. Without it, agents can't take screenshots to check a UI change and fall back to reading the code. @@ -359,7 +359,7 @@ With `quiet_skips = true` in the `[activity]` section of the [User config](#user A Merge run merges only code an agent wrote or reviewed. If someone else pushes to the Issue branch during the Run, their **Foreign commits** are reviewed before they can be merged: 1. Each round of step 6 starts by fetching the Issue branch from `origin`. Any new commits there are merged into the Run's branch, by fast-forward or a merge commit, never a rebase, each logged on stderr as `merging new commit from origin/`. Foreign commits that arrive while CI runs on a green head send the Run round again rather than on to the merge. A merge that conflicts with the Run's own work gets a conflict Repair. -2. A **review Repair** then runs `/thirdshift:code-review` with the head thirdshift last knew as the Run's own as the fixed point. The agent fixes the findings it agrees with, adds the rest to the pull request body's "Unaddressed findings" section marked as coming from the Foreign commits, and pushes. +2. A **review Repair** then runs `/thirdshift-code-review` with the head thirdshift last knew as the Run's own as the fixed point. The agent fixes the findings it agrees with, adds the rest to the pull request body's "Unaddressed findings" section marked as coming from the Foreign commits, and pushes. 3. The round goes on as usual: the Base branch is merged in, CI watched, and the merge tried once the head is green. Review Repairs are logged as `repair-`, numbered with the other Repairs, count against the cap of 5 Repairs, and get a Resume like any session. A round that picks up Foreign commits counts against the same 5, like a Base branch move. A Run that spends either budget fails. @@ -512,7 +512,7 @@ An Architect run: 1. Makes the checks a Run makes that don't need an issue, before creating anything: `origin` is a GitHub repository, git has a `user.name` and `user.email`, HEAD is not detached, and the Base branch exists on `origin` with your local copy not ahead of it. With `base `, a detached HEAD is fine, since the checkout no longer picks the Base branch, and the other two checks are made on ``: `base branch does not exist on origin; push it first`, or `local is commit(s) ahead of origin/; push them first`. There is no **Origin match**, since there is no Issue URL: the repository is the one `origin` names. If another Architect run on the repository, or a [Pickup run](#pickup-runs), is still running, it is [skipped](#one-at-a-time) here, and if not, it is skipped when an earlier [Architect plan is still open](#one-architect-plan-at-a-time), and if not, when an [Architect idea waits for triage](#an-architect-idea-waits-for-triage), and if not, when the repository has a [Ready issue](#a-ready-issue-goes-first). With `launch.pull = true` in the [User config](#user-config), it then fast-forwards your checkout of the Base branch, as a Run does, and so only when the Base branch is the branch checked out: with `base ` from a clone on another branch, your checkout is left alone. 2. Creates a git worktree next to your clone, named `-architect`, detached at the head of the Base branch on `origin`, with no **Issue branch**. Your checkout, its uncommitted changes and its untracked files are never touched or scanned. 3. Runs the **Architecture review** there: a headless Claude Code session with the **Factory skills** loaded, started with the [Architecture review prompt](prompts/architecture-review.md). It looks for deepening opportunities, skips any an open issue already covers, and takes the top recommendation. If it is Strong, the review publishes it as the plan, labelled `needs-triage`: a **Spec** with **Tickets**, or a single Ticket when one session is enough. It may edit files in the worktree to check an idea, but commits and pushes nothing. It ends its final message with one line naming the plan: `Architecture review plan: `. thirdshift reads only that line. With [no Strong candidate](#no-strong-candidate) the line names another issue, which thirdshift labels an **Architect idea**, and steps 5 to 7 are skipped. -4. Removes the worktree and the temporary plugin directory once the session ends, whatever the outcome. +4. Removes the worktree once the session ends, whatever the outcome. The temporary directory the Factory skills were written out to goes as the Architect run ends. 5. Checks the plan: the issue is in this repository, is open, was created after the Architect run started, and carries no other label that says it is not agent work (`ready-for-human`, `needs-info` or `wontfix`). 6. Marks the plan ready, in one request: `needs-triage` is swapped for `ready-for-agent`, the plan is labelled `architect-plan`, the mark a later Architect run finds an open **Architect plan** by, and its other labels are kept. thirdshift adds that label, never the agent, and first creates it if the repository has none. A Spec's Tickets are left as the review labelled them, without `architect-plan`. 7. Dispatches the plan, unless given `--plan-only`, exactly as `thirdshift ` would from the same clone on the Architect run's Base branch: a [Spec run](#spec-runs) when the plan has sub-issues, a Run otherwise. Its Base branch is the Architect run's, so with `base ` it is ``, not the branch checked out. `merge` and `no-merge` apply to the Spec PR or the Run's pull request, `parallel ` to the Spec run, and `base-fix` and `no-base-fix` to whether that run may start a [Base fix](#base-fix), as if given to that command, and the [User config](#user-config) sets what they leave unsaid: `merge.always`, `spec.parallel`, `base.fix`, `launch.pull` and `logs.dir`. The Architecture review itself never watches CI, so a Base fix can only happen in the dispatched run. It makes the [Claim](#the-claim) as that command would too: once its checks pass, the plan's `ready-for-agent` is swapped for `in-progress`, and the plan keeps `architect-plan`. The one difference is that it sends no [Run notification](#run-notifications) of its own, whatever `email.always` says: the Architect run sends [the one](#one-run-notification). @@ -848,4 +848,4 @@ The tag starts the release workflow, generated by [`dist`](https://github.com/ax thirdshift is released under the [MIT License](LICENSE). -The Factory skills in `skills/` are adapted from Matt Pocock's [mattpocock/skills](https://github.com/mattpocock/skills), changed to run headless with no human in the loop. They are used under the MIT License; its copyright notice is kept in [`skills/LICENSE`](skills/LICENSE) and is embedded in the binary and written out with the skills on every Run. The `pr` skill also reproduces part of Dex Horthy's `show-me` skill; see [`skills/pr/CREDITS.md`](skills/pr/CREDITS.md). +The Factory skills in `skills/` are adapted from Matt Pocock's [mattpocock/skills](https://github.com/mattpocock/skills), changed to run headless with no human in the loop. They are used under the MIT License; its copyright notice is kept in [`skills/LICENSE`](skills/LICENSE) and is embedded in the binary and written out with the skills on every Run. The `thirdshift-pr` skill also reproduces part of Dex Horthy's `show-me` skill; see [`skills/thirdshift-pr/CREDITS.md`](skills/thirdshift-pr/CREDITS.md). diff --git a/docs/adr/0001-rust-binary-with-embedded-skills.md b/docs/adr/0001-rust-binary-with-embedded-skills.md index 5bcdb159d..20568bca0 100644 --- a/docs/adr/0001-rust-binary-with-embedded-skills.md +++ b/docs/adr/0001-rust-binary-with-embedded-skills.md @@ -5,4 +5,4 @@ The factory is a Rust binary that compiles `skills/` into itself (`include_dir!` ## Consequences - Editing a skill has no effect until the binary is rebuilt and reinstalled (`cargo install --path .`). -- The skills are loaded as a plugin, so they are namespaced (`/thirdshift:implement`), and their references to each other must use that namespace. +- ~~The skills are loaded as a plugin, so they are namespaced (`/thirdshift:implement`), and their references to each other must use that namespace.~~ Replaced by [ADR 0012](0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md): the skills are linked into each worktree, named `thirdshift-`, and name each other in plain prose. diff --git a/prompts/README.md b/prompts/README.md index 67f969a22..b1012d114 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -4,16 +4,16 @@ Every Session prompt thirdshift can start or resume an agent session with, one file each, with placeholders such as `` where each Run fills in its own values. The [Prompts and skills page](https://thirdshift.app/prompts/) shows the same prompts. -Every session runs `claude` headless in the Run's worktree, in auto permission mode, with the Factory skills loaded as a plugin from a temporary directory: +Every session runs `claude` headless in the Run's worktree, in auto permission mode. The Factory skills are linked into the worktree's `.claude/skills/`, where Claude Code finds them, and kept out of git: ```sh -claude -p --permission-mode auto --plugin-dir --output-format stream-json --verbose +claude -p --permission-mode auto --output-format stream-json --verbose ``` A [Resume](resume.md) continues the session that ended: ```sh -claude -p --permission-mode auto --plugin-dir --output-format stream-json --verbose --resume +claude -p --permission-mode auto --output-format stream-json --verbose --resume ``` | Prompt | When it is sent | File | @@ -30,4 +30,4 @@ claude -p --permission-mode auto --plugin-dir --output-format strea | CI-fix Repair, with Inherited failures | Starts a Repair session when CI fails on the pull request's head commit and some of the failed checks, but not all, are Inherited failures: they also failed on the Base branch commit the head last merged in, so they are listed apart as not to fix. | [ci-fix-repair-inherited.md](ci-fix-repair-inherited.md) | | Resume | Continues any session, once, that ended its turn with background work still running, which was killed with it. | [resume.md](resume.md) | -The Factory skills the prompts name are in [`skills/`](../skills/), as the plugin writes them out. +The Factory skills the prompts name are in [`skills/`](../skills/), as thirdshift writes them out. diff --git a/prompts/architecture-review.md b/prompts/architecture-review.md index 331775f7c..b3f32c7fa 100644 --- a/prompts/architecture-review.md +++ b/prompts/architecture-review.md @@ -5,15 +5,15 @@ Starts the Architecture review, the session an Architect run opens with, in a worktree at the head of the Base branch. The line naming the focus is left out when the command gives none. ``` -/thirdshift:improve-codebase-architecture +/thirdshift-improve-codebase-architecture This is an Architecture review of the base branch : this worktree is checked out at its head on origin, on no branch. Focus the review on: -Find the deepening opportunities with /thirdshift:improve-codebase-architecture, using /thirdshift:codebase-design for the vocabulary. Skip any that an open issue already covers, and take the top recommendation. +Find the deepening opportunities with the `thirdshift-improve-codebase-architecture` skill, using the `thirdshift-codebase-design` skill for the vocabulary. Skip any that an open issue already covers, and take the top recommendation. -If it is Strong, settle its design yourself and publish it as the plan with /thirdshift:to-spec and /thirdshift:to-tickets: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `needs-triage`, not `ready-for-agent`: thirdshift marks it ready once you are done. +If it is Strong, settle its design yourself and publish it as the plan with the `thirdshift-to-spec` and `thirdshift-to-tickets` skills: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `needs-triage`, not `ready-for-agent`: thirdshift marks it ready once you are done. If it is not Strong, file it as one issue labelled `needs-triage`, unless an open issue already covers it. diff --git a/prompts/conflict-repair.md b/prompts/conflict-repair.md index 15b538b28..150c3506c 100644 --- a/prompts/conflict-repair.md +++ b/prompts/conflict-repair.md @@ -5,7 +5,7 @@ Starts a Repair session when merging the Base branch into the Issue branch leaves conflicts. ``` -/thirdshift:resolving-merge-conflicts +/thirdshift-resolving-merge-conflicts A merge of origin/ into is in progress in this worktree and has conflicts. implements ; its pull request is . diff --git a/prompts/continuation-pr.md b/prompts/continuation-pr.md index 20b831ab9..33e37eec6 100644 --- a/prompts/continuation-pr.md +++ b/prompts/continuation-pr.md @@ -5,17 +5,17 @@ Starts the implement session when the Run is a Continuation of an Issue branch whose pull request is open. ``` -/thirdshift:implement +/thirdshift-implement You are continuing work on branch , which already has commits (see git log ..HEAD). Build on them; don't start over. -The base branch is . Review with /thirdshift:code-review using as the fixed point. +The base branch is . Review with the `thirdshift-code-review` skill using as the fixed point. Address the Standards and Spec findings you agree with. Push branch . -Update PR using /thirdshift:pr, rewriting its body to cover the whole branch, marked ready for review. +Update PR using the `thirdshift-pr` skill, rewriting its body to cover the whole branch, marked ready for review. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. diff --git a/prompts/continuation.md b/prompts/continuation.md index fca01aa64..8bde6ee19 100644 --- a/prompts/continuation.md +++ b/prompts/continuation.md @@ -5,17 +5,17 @@ Starts the implement session when the Run is a Continuation of an Issue branch that has no pull request. ``` -/thirdshift:implement +/thirdshift-implement You are continuing work on branch , which already has commits (see git log ..HEAD). Build on them; don't start over. -The base branch is . Review with /thirdshift:code-review using as the fixed point. +The base branch is . Review with the `thirdshift-code-review` skill using as the fixed point. Address the Standards and Spec findings you agree with. Push branch . -Create a pull request against using /thirdshift:pr, marked ready for review. +Create a pull request against using the `thirdshift-pr` skill, marked ready for review. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. diff --git a/prompts/foreign-conflict-repair.md b/prompts/foreign-conflict-repair.md index b58bc2808..99a90c54a 100644 --- a/prompts/foreign-conflict-repair.md +++ b/prompts/foreign-conflict-repair.md @@ -5,7 +5,7 @@ In a Merge run, starts a Repair session when merging Foreign commits from the Issue branch on origin into the local one leaves conflicts. ``` -/thirdshift:resolving-merge-conflicts +/thirdshift-resolving-merge-conflicts A merge of origin/ into is in progress in this worktree and has conflicts. implements ; its pull request is . diff --git a/prompts/fresh.md b/prompts/fresh.md index 788113188..b21fce162 100644 --- a/prompts/fresh.md +++ b/prompts/fresh.md @@ -5,10 +5,10 @@ Starts the implement session, which goes on to review and open the pull request, when the Run starts a new Issue branch. ``` -/thirdshift:implement -The base branch is . Review with /thirdshift:code-review using as the fixed point. +/thirdshift-implement +The base branch is . Review with the `thirdshift-code-review` skill using as the fixed point. Address the Standards and Spec findings you agree with. -Push branch and create a pull request against using /thirdshift:pr, marked ready for review. +Push branch and create a pull request against using the `thirdshift-pr` skill, marked ready for review. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. Include "Closes #" in the PR body. You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. diff --git a/prompts/review-repair.md b/prompts/review-repair.md index 76f9b48f1..3e230acd8 100644 --- a/prompts/review-repair.md +++ b/prompts/review-repair.md @@ -5,11 +5,11 @@ In a Merge run, starts a Repair session once Foreign commits are merged into the Issue branch, to review them from the head the Run last knew as its own before they can be merged. ``` -/thirdshift:code-review +/thirdshift-code-review Someone else pushed commits to while it was being worked on, and they have been merged into in this worktree. implements ; its pull request is . They are merged into the base branch only once you have reviewed them. -Review with /thirdshift:code-review using as the fixed point: 's head before their commits were merged in. +Review with the `thirdshift-code-review` skill using as the fixed point: 's head before their commits were merged in. Address the Standards and Spec findings you agree with. diff --git a/prompts/spec-review.md b/prompts/spec-review.md index 0ac7de4dd..a22dd00aa 100644 --- a/prompts/spec-review.md +++ b/prompts/spec-review.md @@ -5,17 +5,17 @@ In a Spec run, starts the Spec review once every Ticket has landed on the Spec branch, before the Spec PR, a draft until then, is marked ready. ``` -/thirdshift:code-review , with the Spec as the spec +/thirdshift-code-review , with the Spec as the spec Every Ticket of the Spec has landed on branch , its Spec branch (see git log ..HEAD). Review the Spec as a whole, including how the Tickets' work fits together. -Review with /thirdshift:code-review using as the fixed point and as the spec. +Review with the `thirdshift-code-review` skill using as the fixed point and as the spec. -Address the Standards and Spec findings you agree with, using /thirdshift:tdd where it fits, and commit. +Address the Standards and Spec findings you agree with, using the `thirdshift-tdd` skill where it fits, and commit. Push branch . Do not rebase or force-push. -Update PR using /thirdshift:pr, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done. +Update PR using the `thirdshift-pr` skill, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. diff --git a/site/index.html b/site/index.html index 802452371..9b51ff998 100644 --- a/site/index.html +++ b/site/index.html @@ -76,7 +76,7 @@

Your issues go to press overnight.

  1. /grill-with-docs
  2. /to-spec
  3. /to-tickets

The human half of the workflow: shaping work into issues the factory can take on, by grilling the idea with /grill-with-docs, writing the spec with /to-spec and cutting it into tickets with /to-tickets, and reviewing the pull requests it delivers.

Third shift

Agents, with headless copies of the same skills

-
  1. /implement
  2. /code-review
  3. /pr
+
  1. /thirdshift-implement
  2. /thirdshift-code-review
  3. /thirdshift-pr

thirdshift takes one Issue URL per Run. Its Factory skills, adapted to run with no human in the loop, implement the issue with /implement, review the work with /code-review and open a pull request that is ready for review with /pr.

Read every prompt and skill the agents get →

@@ -196,7 +196,7 @@

1 · C · Prepare

2 · M · Implement

A headless Claude Code session, with the Factory skills loaded, implements the issue in the worktree, test-first where it can, and commits.

- +
  1. thirdshift: logging the session to /home/jacob/.thirdshift/logs/JacobStephens2-thirdshift-issue-24-20260925T134143Z-implement.jsonl
  2. thirdshift: implement: session started
  3. @@ -216,9 +216,9 @@

    2 · M · Implement<

    3 · Y · Review

    The same session reviews its work against the Base branch, fixes the Standards findings and Spec findings it agrees with, and lists the rest as Unaddressed findings in the pull request body. Then it opens the pull request, ready for review.

    - +
      -
    1. thirdshift: implement: skill thirdshift:code-review
    2. +
    3. thirdshift: implement: skill thirdshift-code-review
    4. thirdshift: implement: $ git rev-parse main && git log main..HEAD --oneline && git diff main...HEAD --stat; ls .agents/skills…
    5. thirdshift: implement: Agent Standards review
    6. thirdshift: implement: $ git diff main...HEAD && cat CODING_STANDARDS.md && ls .agents/skills/codebase-design/ && cat .agents…
    7. @@ -233,7 +233,7 @@

      3 · Y · Review

      thirdshift: implement: $ python3 - <<'EOF'
    8. thirdshift: implement: SubagentHandback
    9. thirdshift: implement: commit and push
    10. -
    11. thirdshift: implement: skill thirdshift:pr
    12. +
    13. thirdshift: implement: skill thirdshift-pr
    14. thirdshift: implement: $ cat > /tmp/pr24.md <<'EOF'
    15. thirdshift: implement: session ended after 3m 40s: 5 turns, $1.06 at API prices
    @@ -241,7 +241,7 @@

    3 · Y · Review

    4 · K · Finish

    thirdshift pushes, checks the pull request, merges the Base branch in and watches CI. A merge conflict or failing CI starts a Repair, at most 5. A Resume can happen in any session and isn't a Repair. The Run ends once the pull request is open, ready, mergeable and passing CI. This Run needed no Repair.

    - +
    1. thirdshift: pushing issue-24
    2. thirdshift: checking the PR
    3. diff --git a/site/prompts/index.html b/site/prompts/index.html index a058eede1..7607c4f82 100644 --- a/site/prompts/index.html +++ b/site/prompts/index.html @@ -34,10 +34,10 @@

      Prompts and skills

      The command line

      How each session starts

      -

      Every session runs claude headless in the Run's worktree, in auto permission mode, with the Factory skills loaded as a plugin from a temporary directory. The prompt is one of the templates below.

      -
      claude -p --permission-mode auto --plugin-dir <plugin dir> --output-format stream-json --verbose <prompt>
      +

      Every session runs claude headless in the Run's worktree, in auto permission mode. The Factory skills are linked into the worktree's .claude/skills/, where Claude Code finds them, and kept out of git. The prompt is one of the templates below.

      +
      claude -p --permission-mode auto --output-format stream-json --verbose <prompt>

      A Resume continues the session that ended:

      -
      claude -p --permission-mode auto --plugin-dir <plugin dir> --output-format stream-json --verbose --resume <session id> <prompt>
      +
      claude -p --permission-mode auto --output-format stream-json --verbose --resume <session id> <prompt>
      @@ -49,10 +49,10 @@

      Prompts and skills

      Fresh

      Starts the implement session, which goes on to review and open the pull request, when the Run starts a new Issue branch.

      Sent by 2 · M · Implement

      -
      /thirdshift:implement <Issue URL>
      -The base branch is <base>. Review with /thirdshift:code-review using <base> as the fixed point.
      +        
      /thirdshift-implement <Issue URL>
      +The base branch is <base>. Review with the `thirdshift-code-review` skill using <base> as the fixed point.
       Address the Standards and Spec findings you agree with.
      -Push branch <branch> and create a pull request against <base> using /thirdshift:pr, marked ready for review.
      +Push branch <branch> and create a pull request against <base> using the `thirdshift-pr` skill, marked ready for review.
       In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason.
       Include "Closes #<n>" in the PR body.
       You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.
      @@ -62,17 +62,17 @@ 

      Fresh

      Continuation, with no pull request

      Starts the implement session when the Run is a Continuation of an Issue branch that has no pull request.

      Sent by 2 · M · Implement

      -
      /thirdshift:implement <Issue URL>
      +        
      /thirdshift-implement <Issue URL>
       
       You are continuing work on branch <branch>, which already has commits (see git log <base>..HEAD). Build on them; don't start over.
       
      -The base branch is <base>. Review with /thirdshift:code-review using <base> as the fixed point.
      +The base branch is <base>. Review with the `thirdshift-code-review` skill using <base> as the fixed point.
       
       Address the Standards and Spec findings you agree with.
       
       Push branch <branch>.
       
      -Create a pull request against <base> using /thirdshift:pr, marked ready for review.
      +Create a pull request against <base> using the `thirdshift-pr` skill, marked ready for review.
       
       In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason.
       
      @@ -85,17 +85,17 @@ 

      Continuation, with no pull request

      Continuation, with an open pull request

      Starts the implement session when the Run is a Continuation of an Issue branch whose pull request is open.

      Sent by 2 · M · Implement

      -
      /thirdshift:implement <Issue URL>
      +        
      /thirdshift-implement <Issue URL>
       
       You are continuing work on branch <branch>, which already has commits (see git log <base>..HEAD). Build on them; don't start over.
       
      -The base branch is <base>. Review with /thirdshift:code-review using <base> as the fixed point.
      +The base branch is <base>. Review with the `thirdshift-code-review` skill using <base> as the fixed point.
       
       Address the Standards and Spec findings you agree with.
       
       Push branch <branch>.
       
      -Update PR <pull request URL> using /thirdshift:pr, rewriting its body to cover the whole branch, marked ready for review.
      +Update PR <pull request URL> using the `thirdshift-pr` skill, rewriting its body to cover the whole branch, marked ready for review.
       
       In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason.
       
      @@ -108,17 +108,17 @@ 

      Continuation, with an open pull requestSpec review

      In a Spec run, starts the Spec review once every Ticket has landed on the Spec branch, before the Spec PR, a draft until then, is marked ready.

      Sent by 3 · Y · Review

      -
      /thirdshift:code-review <base>, with the Spec <Spec URL> as the spec
      +        
      /thirdshift-code-review <base>, with the Spec <Spec URL> as the spec
       
       Every Ticket of the Spec <Spec URL> has landed on branch <Spec branch>, its Spec branch (see git log <base>..HEAD). Review the Spec as a whole, including how the Tickets' work fits together.
       
      -Review with /thirdshift:code-review using <base> as the fixed point and <Spec URL> as the spec.
      +Review with the `thirdshift-code-review` skill using <base> as the fixed point and <Spec URL> as the spec.
       
      -Address the Standards and Spec findings you agree with, using /thirdshift:tdd where it fits, and commit.
      +Address the Standards and Spec findings you agree with, using the `thirdshift-tdd` skill where it fits, and commit.
       
       Push branch <Spec branch>. Do not rebase or force-push.
       
      -Update PR <pull request URL> using /thirdshift:pr, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done.
      +Update PR <pull request URL> using the `thirdshift-pr` skill, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done.
       
       In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason.
       
      @@ -131,15 +131,15 @@ 

      Spec review

      Architecture review

      Starts the Architecture review, the session an Architect run opens with, in a worktree at the head of the Base branch. The line naming the focus is left out when the command gives none.

      Sent by an Architect run, before any unit

      -
      /thirdshift:improve-codebase-architecture
      +        
      /thirdshift-improve-codebase-architecture
       
       This is an Architecture review of the base branch <base>: this worktree is checked out at its head on origin, on no branch.
       
       Focus the review on: <focus>
       
      -Find the deepening opportunities with /thirdshift:improve-codebase-architecture, using /thirdshift:codebase-design for the vocabulary. Skip any that an open issue already covers, and take the top recommendation.
      +Find the deepening opportunities with the `thirdshift-improve-codebase-architecture` skill, using the `thirdshift-codebase-design` skill for the vocabulary. Skip any that an open issue already covers, and take the top recommendation.
       
      -If it is Strong, settle its design yourself and publish it as the plan with /thirdshift:to-spec and /thirdshift:to-tickets: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `needs-triage`, not `ready-for-agent`: thirdshift marks it ready once you are done.
      +If it is Strong, settle its design yourself and publish it as the plan with the `thirdshift-to-spec` and `thirdshift-to-tickets` skills: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `needs-triage`, not `ready-for-agent`: thirdshift marks it ready once you are done.
       
       If it is not Strong, file it as one issue labelled `needs-triage`, unless an open issue already covers it.
       
      @@ -157,7 +157,7 @@ 

      Architecture review

      Conflict Repair

      Starts a Repair session when merging the Base branch into the Issue branch leaves conflicts.

      Sent by 4 · K · Finish

      -
      /thirdshift:resolving-merge-conflicts
      +        
      /thirdshift-resolving-merge-conflicts
       
       A merge of origin/<base> into <branch> is in progress in this worktree and has conflicts.
       <branch> implements <Issue URL>; its pull request is <pull request URL>.
      @@ -171,7 +171,7 @@ 

      Conflict Repair

      Conflict Repair, on Foreign commits

      In a Merge run, starts a Repair session when merging Foreign commits from the Issue branch on origin into the local one leaves conflicts.

      Sent by 4 · K · Finish

      -
      /thirdshift:resolving-merge-conflicts
      +        
      /thirdshift-resolving-merge-conflicts
       
       A merge of origin/<branch> into <branch> is in progress in this worktree and has conflicts.
       <branch> implements <Issue URL>; its pull request is <pull request URL>.
      @@ -185,11 +185,11 @@ 

      Conflict Repair, on Foreign commit

      Review Repair

      In a Merge run, starts a Repair session once Foreign commits are merged into the Issue branch, to review them from the head the Run last knew as its own before they can be merged.

      Sent by 4 · K · Finish

      -
      /thirdshift:code-review <own head>
      +        
      /thirdshift-code-review <own head>
       
       Someone else pushed commits to <branch> while it was being worked on, and they have been merged into <branch> in this worktree. <branch> implements <Issue URL>; its pull request is <pull request URL>. They are merged into the base branch only once you have reviewed them.
       
      -Review with /thirdshift:code-review using <own head> as the fixed point: <branch>'s head before their commits were merged in.
      +Review with the `thirdshift-code-review` skill using <own head> as the fixed point: <branch>'s head before their commits were merged in.
       
       Address the Standards and Spec findings you agree with.
       
      @@ -254,14 +254,14 @@ 

      Resume

      Factory skills

      The skills, as the agent reads them

      -

      Every Factory skill and supporting file, raw, exactly as the plugin writes it out.

      +

      Every Factory skill and supporting file, raw, exactly as thirdshift writes it out.

      -
      -

      code-review

      +
      +

      thirdshift-code-review

      Used by 3 · Y · Review

      -

      code-review/SKILL.md

      +

      thirdshift-code-review/SKILL.md

      ---
      -name: code-review
      +name: thirdshift-code-review
       description: "Review the changes since a fixed point (commit, branch, tag, or merge-base) along two axes: Standards (does the code follow this repo's documented coding standards?) and Spec (does the code match what the originating issue/spec asked for?). Runs both reviews in parallel sub-agents and reports them side by side. Use when the user wants to review a branch, a PR, work-in-progress changes, or asks to \"review since X\"."
       ---
       
      @@ -348,18 +348,18 @@ 

      code-review/SKILL.md

      Reporting them separately stops one axis from masking the other.
      -

      code-review/agents/openai.yaml

      +

      thirdshift-code-review/agents/openai.yaml

      interface:
         display_name: "Code Review"
         short_description: "Review a diff on standards and spec"
       
      -
      -

      codebase-design

      +
      +

      thirdshift-codebase-design

      Used by the Architecture review, in an Architect run

      -

      codebase-design/SKILL.md

      +

      thirdshift-codebase-design/SKILL.md

      ---
      -name: codebase-design
      +name: thirdshift-codebase-design
       description: Shared vocabulary for designing deep modules. Use when designing or improving a module's interface, finding deepening opportunities, deciding where a seam goes, making code more testable or AI-navigable, or when another skill needs the deep-module vocabulary.
       ---
       
      @@ -473,7 +473,7 @@ 

      codebase-design/SKILL.md

      -

      codebase-design/DEEPENING.md

      +

      thirdshift-codebase-design/DEEPENING.md

      # Deepening
       
       How to deepen a cluster of shallow modules safely, given its dependencies. Assumes the vocabulary in [SKILL.md](SKILL.md): **module**, **interface**, **seam**, **adapter**.
      @@ -512,7 +512,7 @@ 

      codebase-design/DEEPENING.md

      -

      codebase-design/DESIGN-IT-TWICE.md

      +

      thirdshift-codebase-design/DESIGN-IT-TWICE.md

      # Design It Twice
       
       To explore alternative interfaces for a chosen deepening candidate, use this parallel sub-agent pattern. Based on "Design It Twice" (Ousterhout): your first idea is unlikely to be the best.
      @@ -558,33 +558,33 @@ 

      codebase-design/DESIGN-IT After comparing, decide: which design is strongest and why. If elements from different designs would combine well, take a hybrid. Be opinionated: the choice is yours to make, and nobody picks from a menu. Record the decision and its reason, with the designs you turned down and why.

      -

      codebase-design/agents/openai.yaml

      +

      thirdshift-codebase-design/agents/openai.yaml

      interface:
         display_name: "Codebase Design"
         short_description: "Vocabulary for deep-module design"
       
      -
      -

      implement

      +
      +

      thirdshift-implement

      Used by 2 · M · Implement

      -

      implement/SKILL.md

      +

      thirdshift-implement/SKILL.md

      ---
      -name: implement
      +name: thirdshift-implement
       description: "Implement a piece of work based on a spec or set of tickets."
       disable-model-invocation: false
       ---
       
       Implement the work described by the user in the spec or tickets.
       
      -Use /thirdshift:tdd where possible, at pre-agreed seams.
      +Use the `thirdshift-tdd` skill where possible, at pre-agreed seams.
       
       Run typechecking regularly, single test files regularly, and the full test suite once at the end.
       
      -Once done, use /thirdshift:code-review to review the work.
      +Once done, use the `thirdshift-code-review` skill to review the work.
       
       Commit your work to the current branch.
       
      -

      implement/agents/openai.yaml

      +

      thirdshift-implement/agents/openai.yaml

      interface:
         display_name: "Implement"
         short_description: "Build work from a spec or tickets"
      @@ -592,12 +592,12 @@ 

      implement/agents/openai.yaml

      -
      -

      improve-codebase-architecture

      +
      +

      thirdshift-improve-codebase-architecture

      Used by the Architecture review, in an Architect run

      -

      improve-codebase-architecture/SKILL.md

      +

      thirdshift-improve-codebase-architecture/SKILL.md

      ---
      -name: improve-codebase-architecture
      +name: thirdshift-improve-codebase-architecture
       description: "Scan a codebase for deepening opportunities, take the top recommendation, and publish it to the issue tracker: as a plan when it is Strong, as an idea otherwise. Only for an Architecture review."
       disable-model-invocation: false
       ---
      @@ -612,7 +612,7 @@ 

      improve-codebase-arch This command is _informed_ by the project's domain model and built on a shared design vocabulary: -- Call the Skill tool with "thirdshift:codebase-design" for the architecture vocabulary (**module**, **interface**, **depth**, **seam**, **adapter**, **leverage**, **locality**) and its principles (the deletion test, "the interface is the test surface", "one adapter = hypothetical seam, two = real"). Use these terms exactly in every suggestion, and don't drift into "component," "service," "API," or "boundary." +- Use the `thirdshift-codebase-design` skill for the architecture vocabulary (**module**, **interface**, **depth**, **seam**, **adapter**, **leverage**, **locality**) and its principles (the deletion test, "the interface is the test surface", "one adapter = hypothetical seam, two = real"). Use these terms exactly in every suggestion, and don't drift into "component," "service," "API," or "boundary." - The domain language in `CONTEXT.md` gives names to good seams; ADRs in `docs/adr/` record decisions this command should not re-litigate. The issue tracker and triage label vocabulary should have been provided to you. If `docs/agents/issue-tracker.md` is missing, fall back to the `gh` CLI. If `docs/agents/triage-labels.md` is missing, use the label names as written here, creating a label the tracker lacks (`gh label create`). @@ -669,7 +669,7 @@

      improve-codebase-arch If the scan surfaced no candidate at all, file nothing, and say so plainly in place of the final line of step 6: there is no issue for it to name. -**Use CONTEXT.md vocabulary for the domain, and the `/thirdshift:codebase-design` vocabulary for the architecture.** If `CONTEXT.md` defines "Order," talk about "the Order intake module," not "the FooBarHandler," and not "the Order service." +**Use CONTEXT.md vocabulary for the domain, and the `thirdshift-codebase-design` skill's vocabulary for the architecture.** If `CONTEXT.md` defines "Order," talk about "the Order intake module," not "the FooBarHandler," and not "the Order service." **ADR conflicts**: if a candidate contradicts an existing ADR, only keep it when the friction is real enough to warrant revisiting the ADR. Mark it clearly in the report (e.g. a warning callout: _"contradicts ADR-0007, but worth reopening because…"_). Don't raise every theoretical refactor an ADR forbids. @@ -690,7 +690,7 @@

      improve-codebase-arch - **Naming a deepened module after a concept not in `CONTEXT.md`?** Record the term and its definition as work for a Ticket. - **Sharpening a fuzzy term as you decide?** Record the sharper definition as work for a Ticket. - **Reopening an ADR, or making a decision worth one?** Record the new or changed ADR as work for a Ticket. A decision is worth an ADR only when it is hard to reverse, surprising without context, and the result of a real trade-off. -- **Want to explore alternative interfaces for the deepened module?** Call the Skill tool with "thirdshift:codebase-design" and use its design-it-twice parallel sub-agent pattern. +- **Want to explore alternative interfaces for the deepened module?** Use the `thirdshift-codebase-design` skill and its design-it-twice parallel sub-agent pattern. ### 4. Publish the plan @@ -698,8 +698,8 @@

      improve-codebase-arch Decide the plan's size. If one session, a single fresh context window, is enough for the whole change, the plan is a single **Ticket**. Otherwise it is a **Spec** with **Tickets**. Don't force a small refactor into a Spec. -- **A Spec with Tickets**: call the Skill tool with "thirdshift:to-spec" to publish the Spec, then with "thirdshift:to-tickets" to publish its Tickets as the Spec's sub-issues. -- **A single Ticket**: call the Skill tool with "thirdshift:to-tickets" to publish one standalone Ticket, with no parent. +- **A Spec with Tickets**: use the `thirdshift-to-spec` skill to publish the Spec, then the `thirdshift-to-tickets` skill to publish its Tickets as the Spec's sub-issues. +- **A single Ticket**: use the `thirdshift-to-tickets` skill to publish one standalone Ticket, with no parent. The plan's top issue, the Spec or the single Ticket, is labelled `needs-triage` and nothing else. thirdshift marks it ready once the session has ended and the plan passes its checks; you never do. Tickets under a Spec are labelled `ready-for-agent`. @@ -728,7 +728,7 @@

      improve-codebase-arch thirdshift reads only that line, so everything else you want read belongs in the issue.

      -

      improve-codebase-architecture/REPORT.md

      +

      thirdshift-improve-codebase-architecture/REPORT.md

      # Report Format
       
       The Architecture review's report is Markdown, written into the issue it publishes: the plan's top issue, or the idea issue. GitHub renders Mermaid in a fenced `mermaid` block, so the diagrams live where the issue is read. Write no file, and open nothing.
      @@ -737,7 +737,7 @@ 

      improve-codebase-arc ## Candidate card -The diagrams carry the weight. Prose is sparse, plain, and uses the glossary terms (from the `/thirdshift:codebase-design` skill) without ceremony. +The diagrams carry the weight. Prose is sparse, plain, and uses the glossary terms (from the `thirdshift-codebase-design` skill) without ceremony. ```md **Strength**: Strong · **Dependencies**: in-process @@ -824,7 +824,7 @@

      improve-codebase-arc ## Tone -Plain English, concise, but the architectural nouns and verbs come straight from the `/thirdshift:codebase-design` skill. Concision is not an excuse to drift. +Plain English, concise, but the architectural nouns and verbs come straight from the `thirdshift-codebase-design` skill. Concision is not an excuse to drift. **Use exactly:** module, interface, implementation, depth, deep, shallow, seam, adapter, leverage, locality. @@ -839,9 +839,9 @@

      improve-codebase-arc **Wins bullets** name the gain in glossary terms: *"locality: bugs concentrate in one module"*, *"leverage: one interface, N call sites"*, *"interface shrinks; implementation absorbs the wrappers"*. Don't write *"easier to maintain"* or *"cleaner code"*, because those terms aren't in the glossary and don't earn their place. -No hedging, no throat-clearing, no "it's worth noting that…". If a sentence could be a bullet, make it a bullet. If a bullet could be cut, cut it. If a term isn't in the `/thirdshift:codebase-design` glossary, reach for one that is before inventing a new one. +No hedging, no throat-clearing, no "it's worth noting that…". If a sentence could be a bullet, make it a bullet. If a bullet could be cut, cut it. If a term isn't in the `thirdshift-codebase-design` skill's glossary, reach for one that is before inventing a new one.

      -

      improve-codebase-architecture/agents/openai.yaml

      +

      thirdshift-improve-codebase-architecture/agents/openai.yaml

      interface:
         display_name: "Improve Codebase Architecture"
         short_description: "Find and publish the top architecture improvement"
      @@ -849,12 +849,12 @@ 

      improve-cod allow_implicit_invocation: true

      -
      -

      pr

      +
      +

      thirdshift-pr

      Used by 3 · Y · Review

      -

      pr/SKILL.md

      +

      thirdshift-pr/SKILL.md

      ---
      -name: pr
      +name: thirdshift-pr
       description: "Use when writing a PR body."
       metadata:
         credits:
      @@ -1022,23 +1022,23 @@ 

      pr/SKILL.md

      The blast radius is the potential impact or scope of the changes introduced by this PR. Consider all possibilities. Examples are layout shift, breakages for consumers, mobile responsiveness, etc.
      -

      pr/CREDITS.md

      +

      thirdshift-pr/CREDITS.md

      # Credits
       
       "The shape of the change" section is [Dex Horthy](https://github.com/dexhorthy)'s [`show-me`](https://github.com/humanlayer/humanlayer) skill, reproduced almost word for word, aimed at a diff instead of a live conversation. `pr` does not depend on `show-me` as a skill (it isn't part of this repo, and a hard dependency would break standalone installs), so the content is copied in rather than pointed at; this file is the attribution a dependency would otherwise have carried.
       
      -

      pr/agents/openai.yaml

      +

      thirdshift-pr/agents/openai.yaml

      interface:
         display_name: "PR"
         short_description: "Write a PR body that's fast to review"
       
      -
      -

      resolving-merge-conflicts

      +
      +

      thirdshift-resolving-merge-conflicts

      Used by 4 · K · Finish, in a Repair

      -

      resolving-merge-conflicts/SKILL.md

      +

      thirdshift-resolving-merge-conflicts/SKILL.md

      ---
      -name: resolving-merge-conflicts
      +name: thirdshift-resolving-merge-conflicts
       description: "Use when you need to resolve an in-progress git merge/rebase conflict."
       ---
       
      @@ -1051,18 +1051,18 @@ 

      resolving-merge-conflicts 4. Discover the project's **automated checks** and run them, typically typecheck, then tests, then format. Fix anything the merge broke. 5. **Finish the merge/rebase.** Stage everything, commit, and push. If rebasing, continue the rebase process until all commits are rebased.

      -

      resolving-merge-conflicts/agents/openai.yaml

      +

      thirdshift-resolving-merge-conflicts/agents/openai.yaml

      interface:
         display_name: "Resolving Merge Conflicts"
         short_description: "Resolve merge and rebase conflicts"
       
      -
      -

      tdd

      +
      +

      thirdshift-tdd

      Used by 2 · M · Implement

      -

      tdd/SKILL.md

      +

      thirdshift-tdd/SKILL.md

      ---
      -name: tdd
      +name: thirdshift-tdd
       description: Test-driven development. Use when the user wants to build features or fix bugs test-first, mentions "red-green-refactor", or wants integration tests.
       ---
       
      @@ -1086,7 +1086,7 @@ 

      tdd/SKILL.md

      Decide: "What's the public interface, and which seams should we test?" -When the shape of that interface is itself in question (how deep the module is, where the seam belongs, what the interface should expose), call the Skill tool with "codebase-design" for the vocabulary. It is the shared source of the module, interface, depth, seam, adapter, leverage and locality terms, and it is a reference to consult, not a session to run. +When the shape of that interface is itself in question (how deep the module is, where the seam belongs, what the interface should expose), use the `thirdshift-codebase-design` skill for the vocabulary. It is the shared source of the module, interface, depth, seam, adapter, leverage and locality terms, and it is a reference to consult, not a session to run. ## Anti-patterns @@ -1098,14 +1098,14 @@

      tdd/SKILL.md

      - **Red before green.** Write the failing test first, then only enough code to pass it. Don't anticipate future tests or add speculative features. - **One slice at a time.** One seam, one test, one minimal implementation per cycle. -- **Refactoring is not part of the loop.** It belongs to the review stage (see the `code-review` skill), not the red → green implementation cycle. +- **Refactoring is not part of the loop.** It belongs to the review stage (see the `thirdshift-code-review` skill), not the red → green implementation cycle.
      -

      tdd/agents/openai.yaml

      +

      thirdshift-tdd/agents/openai.yaml

      interface:
         display_name: "TDD"
         short_description: "Test-driven red-green-refactor"
       
      -

      tdd/mocking.md

      +

      thirdshift-tdd/mocking.md

      # When to Mock
       
       Mock at **system boundaries** only:
      @@ -1166,7 +1166,7 @@ 

      tdd/mocking.md

      - Easier to see which endpoints a test exercises - Type safety per endpoint
      -

      tdd/tests.md

      +

      thirdshift-tdd/tests.md

      # Good and Bad Tests
       
       ## Good Tests
      @@ -1246,12 +1246,12 @@ 

      tdd/tests.md

      ```
      -
      -

      to-spec

      +
      +

      thirdshift-to-spec

      Used by the Architecture review, in an Architect run

      -

      to-spec/SKILL.md

      +

      thirdshift-to-spec/SKILL.md

      ---
      -name: to-spec
      +name: thirdshift-to-spec
       description: "Turn the current session into a spec and publish it to the project issue tracker: no interview, just synthesis of what you've already settled. Only for an Architecture review."
       disable-model-invocation: false
       ---
      @@ -1328,7 +1328,7 @@ 

      to-spec/SKILL.md

      </spec-template>
      -

      to-spec/agents/openai.yaml

      +

      thirdshift-to-spec/agents/openai.yaml

      interface:
         display_name: "To Spec"
         short_description: "Turn a session into a spec"
      @@ -1336,12 +1336,12 @@ 

      to-spec/agents/openai.yaml allow_implicit_invocation: true

      -
      -

      to-tickets

      +
      +

      thirdshift-to-tickets

      Used by the Architecture review, in an Architect run

      -

      to-tickets/SKILL.md

      +

      thirdshift-to-tickets/SKILL.md

      ---
      -name: to-tickets
      +name: thirdshift-to-tickets
       description: Break a plan, spec, or the current session into a set of tracer-bullet tickets, each declaring its blocking edges, published to the issue tracker with native sub-issue and blocking links. Only for an Architecture review.
       disable-model-invocation: false
       ---
      @@ -1441,7 +1441,7 @@ 

      to-tickets/SKILL.md

      Avoid specific file paths or code snippets: they go stale fast. Exception: if a prototype produced a snippet that encodes a decision more precisely than prose can (state machine, reducer, schema, type shape), inline it and note briefly that it came from a prototype. Trim to the decision-rich parts, not a working demo, just the important bits.
      -

      to-tickets/agents/openai.yaml

      +

      thirdshift-to-tickets/agents/openai.yaml

      interface:
         display_name: "To Tickets"
         short_description: "Split a plan into tracer-bullet tickets"
      @@ -1455,7 +1455,7 @@ 

      to-tickets/agents/openai.yaml<

      Licence and credits

      Whose skills these are

      -

      The Factory skills are adapted from Matt Pocock's skills, under this licence. The pr skill credits Dex Horthy's show-me skill in its CREDITS.md.

      +

      The Factory skills are adapted from Matt Pocock's skills, under this licence. The thirdshift-pr skill credits Dex Horthy's show-me skill in its CREDITS.md.

      LICENSE

      MIT License
       
      diff --git a/skills/code-review/SKILL.md b/skills/thirdshift-code-review/SKILL.md
      similarity index 99%
      rename from skills/code-review/SKILL.md
      rename to skills/thirdshift-code-review/SKILL.md
      index d701b736e..e2f2e8ce2 100644
      --- a/skills/code-review/SKILL.md
      +++ b/skills/thirdshift-code-review/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: code-review
      +name: thirdshift-code-review
       description: "Review the changes since a fixed point (commit, branch, tag, or merge-base) along two axes: Standards (does the code follow this repo's documented coding standards?) and Spec (does the code match what the originating issue/spec asked for?). Runs both reviews in parallel sub-agents and reports them side by side. Use when the user wants to review a branch, a PR, work-in-progress changes, or asks to \"review since X\"."
       ---
       
      diff --git a/skills/code-review/agents/openai.yaml b/skills/thirdshift-code-review/agents/openai.yaml
      similarity index 100%
      rename from skills/code-review/agents/openai.yaml
      rename to skills/thirdshift-code-review/agents/openai.yaml
      diff --git a/skills/codebase-design/DEEPENING.md b/skills/thirdshift-codebase-design/DEEPENING.md
      similarity index 100%
      rename from skills/codebase-design/DEEPENING.md
      rename to skills/thirdshift-codebase-design/DEEPENING.md
      diff --git a/skills/codebase-design/DESIGN-IT-TWICE.md b/skills/thirdshift-codebase-design/DESIGN-IT-TWICE.md
      similarity index 100%
      rename from skills/codebase-design/DESIGN-IT-TWICE.md
      rename to skills/thirdshift-codebase-design/DESIGN-IT-TWICE.md
      diff --git a/skills/codebase-design/SKILL.md b/skills/thirdshift-codebase-design/SKILL.md
      similarity index 99%
      rename from skills/codebase-design/SKILL.md
      rename to skills/thirdshift-codebase-design/SKILL.md
      index 37e47d19e..c0daf2e9f 100644
      --- a/skills/codebase-design/SKILL.md
      +++ b/skills/thirdshift-codebase-design/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: codebase-design
      +name: thirdshift-codebase-design
       description: Shared vocabulary for designing deep modules. Use when designing or improving a module's interface, finding deepening opportunities, deciding where a seam goes, making code more testable or AI-navigable, or when another skill needs the deep-module vocabulary.
       ---
       
      diff --git a/skills/codebase-design/agents/openai.yaml b/skills/thirdshift-codebase-design/agents/openai.yaml
      similarity index 100%
      rename from skills/codebase-design/agents/openai.yaml
      rename to skills/thirdshift-codebase-design/agents/openai.yaml
      diff --git a/skills/implement/SKILL.md b/skills/thirdshift-implement/SKILL.md
      similarity index 66%
      rename from skills/implement/SKILL.md
      rename to skills/thirdshift-implement/SKILL.md
      index 9c5da65a7..adfdbfbf3 100644
      --- a/skills/implement/SKILL.md
      +++ b/skills/thirdshift-implement/SKILL.md
      @@ -1,15 +1,15 @@
       ---
      -name: implement
      +name: thirdshift-implement
       description: "Implement a piece of work based on a spec or set of tickets."
       disable-model-invocation: false
       ---
       
       Implement the work described by the user in the spec or tickets.
       
      -Use /thirdshift:tdd where possible, at pre-agreed seams.
      +Use the `thirdshift-tdd` skill where possible, at pre-agreed seams.
       
       Run typechecking regularly, single test files regularly, and the full test suite once at the end.
       
      -Once done, use /thirdshift:code-review to review the work.
      +Once done, use the `thirdshift-code-review` skill to review the work.
       
       Commit your work to the current branch.
      diff --git a/skills/implement/agents/openai.yaml b/skills/thirdshift-implement/agents/openai.yaml
      similarity index 100%
      rename from skills/implement/agents/openai.yaml
      rename to skills/thirdshift-implement/agents/openai.yaml
      diff --git a/skills/improve-codebase-architecture/REPORT.md b/skills/thirdshift-improve-codebase-architecture/REPORT.md
      similarity index 93%
      rename from skills/improve-codebase-architecture/REPORT.md
      rename to skills/thirdshift-improve-codebase-architecture/REPORT.md
      index 44224c032..6d967aa67 100644
      --- a/skills/improve-codebase-architecture/REPORT.md
      +++ b/skills/thirdshift-improve-codebase-architecture/REPORT.md
      @@ -6,7 +6,7 @@ The report is one candidate's card: the top recommendation's. The candidates tha
       
       ## Candidate card
       
      -The diagrams carry the weight. Prose is sparse, plain, and uses the glossary terms (from the `/thirdshift:codebase-design` skill) without ceremony.
      +The diagrams carry the weight. Prose is sparse, plain, and uses the glossary terms (from the `thirdshift-codebase-design` skill) without ceremony.
       
       ```md
       **Strength**: Strong · **Dependencies**: in-process
      @@ -93,7 +93,7 @@ flowchart LR
       
       ## Tone
       
      -Plain English, concise, but the architectural nouns and verbs come straight from the `/thirdshift:codebase-design` skill. Concision is not an excuse to drift.
      +Plain English, concise, but the architectural nouns and verbs come straight from the `thirdshift-codebase-design` skill. Concision is not an excuse to drift.
       
       **Use exactly:** module, interface, implementation, depth, deep, shallow, seam, adapter, leverage, locality.
       
      @@ -108,4 +108,4 @@ Plain English, concise, but the architectural nouns and verbs come straight from
       
       **Wins bullets** name the gain in glossary terms: *"locality: bugs concentrate in one module"*, *"leverage: one interface, N call sites"*, *"interface shrinks; implementation absorbs the wrappers"*. Don't write *"easier to maintain"* or *"cleaner code"*, because those terms aren't in the glossary and don't earn their place.
       
      -No hedging, no throat-clearing, no "it's worth noting that…". If a sentence could be a bullet, make it a bullet. If a bullet could be cut, cut it. If a term isn't in the `/thirdshift:codebase-design` glossary, reach for one that is before inventing a new one.
      +No hedging, no throat-clearing, no "it's worth noting that…". If a sentence could be a bullet, make it a bullet. If a bullet could be cut, cut it. If a term isn't in the `thirdshift-codebase-design` skill's glossary, reach for one that is before inventing a new one.
      diff --git a/skills/improve-codebase-architecture/SKILL.md b/skills/thirdshift-improve-codebase-architecture/SKILL.md
      similarity index 89%
      rename from skills/improve-codebase-architecture/SKILL.md
      rename to skills/thirdshift-improve-codebase-architecture/SKILL.md
      index 571918d39..af6fb068c 100644
      --- a/skills/improve-codebase-architecture/SKILL.md
      +++ b/skills/thirdshift-improve-codebase-architecture/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: improve-codebase-architecture
      +name: thirdshift-improve-codebase-architecture
       description: "Scan a codebase for deepening opportunities, take the top recommendation, and publish it to the issue tracker: as a plan when it is Strong, as an idea otherwise. Only for an Architecture review."
       disable-model-invocation: false
       ---
      @@ -14,7 +14,7 @@ It changes nothing in the repository. Never commit, push, create a branch or ope
       
       This command is _informed_ by the project's domain model and built on a shared design vocabulary:
       
      -- Call the Skill tool with "thirdshift:codebase-design" for the architecture vocabulary (**module**, **interface**, **depth**, **seam**, **adapter**, **leverage**, **locality**) and its principles (the deletion test, "the interface is the test surface", "one adapter = hypothetical seam, two = real"). Use these terms exactly in every suggestion, and don't drift into "component," "service," "API," or "boundary."
      +- Use the `thirdshift-codebase-design` skill for the architecture vocabulary (**module**, **interface**, **depth**, **seam**, **adapter**, **leverage**, **locality**) and its principles (the deletion test, "the interface is the test surface", "one adapter = hypothetical seam, two = real"). Use these terms exactly in every suggestion, and don't drift into "component," "service," "API," or "boundary."
       - The domain language in `CONTEXT.md` gives names to good seams; ADRs in `docs/adr/` record decisions this command should not re-litigate.
       
       The issue tracker and triage label vocabulary should have been provided to you. If `docs/agents/issue-tracker.md` is missing, fall back to the `gh` CLI. If `docs/agents/triage-labels.md` is missing, use the label names as written here, creating a label the tracker lacks (`gh label create`).
      @@ -71,7 +71,7 @@ Every other candidate is dropped. Don't file it, and don't list it in the plan o
       
       If the scan surfaced no candidate at all, file nothing, and say so plainly in place of the final line of step 6: there is no issue for it to name.
       
      -**Use CONTEXT.md vocabulary for the domain, and the `/thirdshift:codebase-design` vocabulary for the architecture.** If `CONTEXT.md` defines "Order," talk about "the Order intake module," not "the FooBarHandler," and not "the Order service."
      +**Use CONTEXT.md vocabulary for the domain, and the `thirdshift-codebase-design` skill's vocabulary for the architecture.** If `CONTEXT.md` defines "Order," talk about "the Order intake module," not "the FooBarHandler," and not "the Order service."
       
       **ADR conflicts**: if a candidate contradicts an existing ADR, only keep it when the friction is real enough to warrant revisiting the ADR. Mark it clearly in the report (e.g. a warning callout: _"contradicts ADR-0007, but worth reopening because…"_). Don't raise every theoretical refactor an ADR forbids.
       
      @@ -92,7 +92,7 @@ Side effects are recorded, never written:
       - **Naming a deepened module after a concept not in `CONTEXT.md`?** Record the term and its definition as work for a Ticket.
       - **Sharpening a fuzzy term as you decide?** Record the sharper definition as work for a Ticket.
       - **Reopening an ADR, or making a decision worth one?** Record the new or changed ADR as work for a Ticket. A decision is worth an ADR only when it is hard to reverse, surprising without context, and the result of a real trade-off.
      -- **Want to explore alternative interfaces for the deepened module?** Call the Skill tool with "thirdshift:codebase-design" and use its design-it-twice parallel sub-agent pattern.
      +- **Want to explore alternative interfaces for the deepened module?** Use the `thirdshift-codebase-design` skill and its design-it-twice parallel sub-agent pattern.
       
       ### 4. Publish the plan
       
      @@ -100,8 +100,8 @@ Only for a `Strong` top recommendation whose design is settled.
       
       Decide the plan's size. If one session, a single fresh context window, is enough for the whole change, the plan is a single **Ticket**. Otherwise it is a **Spec** with **Tickets**. Don't force a small refactor into a Spec.
       
      -- **A Spec with Tickets**: call the Skill tool with "thirdshift:to-spec" to publish the Spec, then with "thirdshift:to-tickets" to publish its Tickets as the Spec's sub-issues.
      -- **A single Ticket**: call the Skill tool with "thirdshift:to-tickets" to publish one standalone Ticket, with no parent.
      +- **A Spec with Tickets**: use the `thirdshift-to-spec` skill to publish the Spec, then the `thirdshift-to-tickets` skill to publish its Tickets as the Spec's sub-issues.
      +- **A single Ticket**: use the `thirdshift-to-tickets` skill to publish one standalone Ticket, with no parent.
       
       The plan's top issue, the Spec or the single Ticket, is labelled `needs-triage` and nothing else. thirdshift marks it ready once the session has ended and the plan passes its checks; you never do. Tickets under a Spec are labelled `ready-for-agent`.
       
      diff --git a/skills/improve-codebase-architecture/agents/openai.yaml b/skills/thirdshift-improve-codebase-architecture/agents/openai.yaml
      similarity index 100%
      rename from skills/improve-codebase-architecture/agents/openai.yaml
      rename to skills/thirdshift-improve-codebase-architecture/agents/openai.yaml
      diff --git a/skills/pr/CREDITS.md b/skills/thirdshift-pr/CREDITS.md
      similarity index 100%
      rename from skills/pr/CREDITS.md
      rename to skills/thirdshift-pr/CREDITS.md
      diff --git a/skills/pr/SKILL.md b/skills/thirdshift-pr/SKILL.md
      similarity index 99%
      rename from skills/pr/SKILL.md
      rename to skills/thirdshift-pr/SKILL.md
      index 8d71b376a..381e587eb 100644
      --- a/skills/pr/SKILL.md
      +++ b/skills/thirdshift-pr/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: pr
      +name: thirdshift-pr
       description: "Use when writing a PR body."
       metadata:
         credits:
      diff --git a/skills/pr/agents/openai.yaml b/skills/thirdshift-pr/agents/openai.yaml
      similarity index 100%
      rename from skills/pr/agents/openai.yaml
      rename to skills/thirdshift-pr/agents/openai.yaml
      diff --git a/skills/thirdshift-resolving-merge-conflicts/SKILL.md b/skills/thirdshift-resolving-merge-conflicts/SKILL.md
      new file mode 100644
      index 000000000..14a23d097
      --- /dev/null
      +++ b/skills/thirdshift-resolving-merge-conflicts/SKILL.md
      @@ -0,0 +1,14 @@
      +---
      +name: thirdshift-resolving-merge-conflicts
      +description: "Use when you need to resolve an in-progress git merge/rebase conflict."
      +---
      +
      +1. **See the current state** of the merge/rebase. Check git history, and the conflicting files.
      +
      +2. **Find the primary sources** for each conflict. Understand deeply why each change was made, and what the original intent was. Read the commit messages, check the PRs, check original issues/tickets.
      +
      +3. **Resolve each hunk.** Preserve both intents where possible. Where incompatible, pick the one matching the merge's stated goal and note the trade-off. Do **not** invent new behaviour. Always resolve; never `--abort`.
      +
      +4. Discover the project's **automated checks** and run them, typically typecheck, then tests, then format. Fix anything the merge broke.
      +
      +5. **Finish the merge/rebase.** Stage everything, commit, and push. If rebasing, continue the rebase process until all commits are rebased.
      \ No newline at end of file
      diff --git a/skills/thirdshift-resolving-merge-conflicts/agents/openai.yaml b/skills/thirdshift-resolving-merge-conflicts/agents/openai.yaml
      new file mode 100644
      index 000000000..331ffb9d3
      --- /dev/null
      +++ b/skills/thirdshift-resolving-merge-conflicts/agents/openai.yaml
      @@ -0,0 +1,3 @@
      +interface:
      +  display_name: "Resolving Merge Conflicts"
      +  short_description: "Resolve merge and rebase conflicts"
      diff --git a/skills/tdd/SKILL.md b/skills/thirdshift-tdd/SKILL.md
      similarity index 94%
      rename from skills/tdd/SKILL.md
      rename to skills/thirdshift-tdd/SKILL.md
      index df8b44d13..e5cc55c2f 100644
      --- a/skills/tdd/SKILL.md
      +++ b/skills/thirdshift-tdd/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: tdd
      +name: thirdshift-tdd
       description: Test-driven development. Use when the user wants to build features or fix bugs test-first, mentions "red-green-refactor", or wants integration tests.
       ---
       
      @@ -23,7 +23,7 @@ A **seam** is the public boundary you test at: the interface where you observe b
       
       Decide: "What's the public interface, and which seams should we test?"
       
      -When the shape of that interface is itself in question (how deep the module is, where the seam belongs, what the interface should expose), call the Skill tool with "codebase-design" for the vocabulary. It is the shared source of the module, interface, depth, seam, adapter, leverage and locality terms, and it is a reference to consult, not a session to run.
      +When the shape of that interface is itself in question (how deep the module is, where the seam belongs, what the interface should expose), use the `thirdshift-codebase-design` skill for the vocabulary. It is the shared source of the module, interface, depth, seam, adapter, leverage and locality terms, and it is a reference to consult, not a session to run.
       
       ## Anti-patterns
       
      @@ -35,4 +35,4 @@ When the shape of that interface is itself in question (how deep the module is,
       
       - **Red before green.** Write the failing test first, then only enough code to pass it. Don't anticipate future tests or add speculative features.
       - **One slice at a time.** One seam, one test, one minimal implementation per cycle.
      -- **Refactoring is not part of the loop.** It belongs to the review stage (see the `code-review` skill), not the red → green implementation cycle.
      +- **Refactoring is not part of the loop.** It belongs to the review stage (see the `thirdshift-code-review` skill), not the red → green implementation cycle.
      diff --git a/skills/tdd/agents/openai.yaml b/skills/thirdshift-tdd/agents/openai.yaml
      similarity index 100%
      rename from skills/tdd/agents/openai.yaml
      rename to skills/thirdshift-tdd/agents/openai.yaml
      diff --git a/skills/tdd/mocking.md b/skills/thirdshift-tdd/mocking.md
      similarity index 100%
      rename from skills/tdd/mocking.md
      rename to skills/thirdshift-tdd/mocking.md
      diff --git a/skills/tdd/tests.md b/skills/thirdshift-tdd/tests.md
      similarity index 100%
      rename from skills/tdd/tests.md
      rename to skills/thirdshift-tdd/tests.md
      diff --git a/skills/to-spec/SKILL.md b/skills/thirdshift-to-spec/SKILL.md
      similarity index 99%
      rename from skills/to-spec/SKILL.md
      rename to skills/thirdshift-to-spec/SKILL.md
      index 3fc5ec829..56dc42ed1 100644
      --- a/skills/to-spec/SKILL.md
      +++ b/skills/thirdshift-to-spec/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: to-spec
      +name: thirdshift-to-spec
       description: "Turn the current session into a spec and publish it to the project issue tracker: no interview, just synthesis of what you've already settled. Only for an Architecture review."
       disable-model-invocation: false
       ---
      diff --git a/skills/to-spec/agents/openai.yaml b/skills/thirdshift-to-spec/agents/openai.yaml
      similarity index 100%
      rename from skills/to-spec/agents/openai.yaml
      rename to skills/thirdshift-to-spec/agents/openai.yaml
      diff --git a/skills/to-tickets/SKILL.md b/skills/thirdshift-to-tickets/SKILL.md
      similarity index 99%
      rename from skills/to-tickets/SKILL.md
      rename to skills/thirdshift-to-tickets/SKILL.md
      index a15acbe9e..2ed473f33 100644
      --- a/skills/to-tickets/SKILL.md
      +++ b/skills/thirdshift-to-tickets/SKILL.md
      @@ -1,5 +1,5 @@
       ---
      -name: to-tickets
      +name: thirdshift-to-tickets
       description: Break a plan, spec, or the current session into a set of tracer-bullet tickets, each declaring its blocking edges, published to the issue tracker with native sub-issue and blocking links. Only for an Architecture review.
       disable-model-invocation: false
       ---
      diff --git a/skills/to-tickets/agents/openai.yaml b/skills/thirdshift-to-tickets/agents/openai.yaml
      similarity index 100%
      rename from skills/to-tickets/agents/openai.yaml
      rename to skills/thirdshift-to-tickets/agents/openai.yaml
      diff --git a/src/architect.rs b/src/architect.rs
      index b679b3db1..f5ed92e2a 100644
      --- a/src/architect.rs
      +++ b/src/architect.rs
      @@ -198,7 +198,7 @@ impl fmt::Display for Reviewed {
       /// idea. With `launch_pull`, the Launch directory's checkout of the Base
       /// branch, if that is the branch checked out, is first brought up to date
       /// with origin. The review's
      -/// worktree and the plugin directory are gone when this returns. A failure
      +/// worktree is gone when this returns. A failure
       /// after the plan is published leaves its labels as the review left them.
       ///
       /// Once the preflight checks pass, and before anything else, the Architect
      diff --git a/src/command.rs b/src/command.rs
      index 95c97e5c4..9f1380682 100644
      --- a/src/command.rs
      +++ b/src/command.rs
      @@ -6,8 +6,9 @@
       //! A command [`start`]s: it begins its record in the logs, loads the User
       //! config, offering Setup, installs the interrupt handler and checks its Run
       //! notification, in that order. It does its work, then [`Started::finish`]es
      -//! with its [`Ending`]: the Activity log's end line, the terminal display and
      -//! the Run notification, in that order, none of the first and last for a
      +//! with its [`Ending`]: the Factory skills it wrote out for its sessions
      +//! removed, then the Activity log's end line, the terminal display and the
      +//! Run notification, in that order, only the display of those three for a
       //! skipped pass.
       
       use std::process::ExitCode;
      @@ -23,6 +24,7 @@ use crate::pickup;
       use crate::progress;
       use crate::run::Ended;
       use crate::run_ending;
      +use crate::skills;
       
       /// How a command that did factory work, or was skipped before any, ended.
       pub enum Ending {
      @@ -114,12 +116,14 @@ impl Started {
               }
           }
       
      -    /// Finish the command that ended as `ending`: write the Activity log's
      +    /// Finish the command that ended as `ending`: remove the Factory skills
      +    /// it wrote out for its sessions, if it did, write the Activity log's
           /// end line, show how it ended, then send its Run notification, if it
           /// asked for one. A skipped pass only shows how it ended: its skip is
           /// recorded where it was decided, and it sends no notification. Returns
           /// its exit code.
           pub fn finish(self, ending: Ending) -> ExitCode {
      +        skills::remove();
               let account = match run_ending::read(&ending) {
                   Ok(account) => account,
                   Err(skip) => return run_ending::show_skip(skip),
      diff --git a/src/main.rs b/src/main.rs
      index f45022f69..507cd1b97 100644
      --- a/src/main.rs
      +++ b/src/main.rs
      @@ -22,7 +22,6 @@ mod logs;
       mod notification;
       mod pass;
       mod pickup;
      -mod plugin;
       mod poll;
       mod progress;
       mod prompt;
      @@ -34,6 +33,7 @@ mod resend_key;
       mod run;
       mod run_ending;
       mod session;
      +mod skills;
       mod spec_run;
       mod update;
       mod worktree;
      diff --git a/src/plugin.rs b/src/plugin.rs
      deleted file mode 100644
      index b08544508..000000000
      --- a/src/plugin.rs
      +++ /dev/null
      @@ -1,49 +0,0 @@
      -//! The Factory skills, embedded at compile time and written out as the
      -//! `thirdshift` plugin (ADR-0001).
      -
      -use std::fs;
      -use std::path::Path;
      -
      -use anyhow::{Context, Result};
      -use include_dir::{Dir, include_dir};
      -use tempfile::TempDir;
      -
      -/// The Factory skills, as the plugin writes them out and the Prompts and skills
      -/// page shows them.
      -pub static SKILLS: Dir = include_dir!("$CARGO_MANIFEST_DIR/skills");
      -
      -/// A temp directory holding the Factory skills and a generated plugin
      -/// manifest, removed when dropped.
      -pub struct Plugin {
      -    dir: TempDir,
      -}
      -
      -impl Plugin {
      -    pub fn write() -> Result {
      -        let dir = tempfile::Builder::new()
      -            .prefix("thirdshift-plugin-")
      -            .tempdir()?;
      -        let skills = dir.path().join("skills");
      -        fs::create_dir_all(&skills)?;
      -        SKILLS
      -            .extract(&skills)
      -            .context("could not write the Factory skills")?;
      -
      -        let manifest_dir = dir.path().join(".claude-plugin");
      -        fs::create_dir_all(&manifest_dir)?;
      -        let manifest = serde_json::json!({
      -            "name": "thirdshift",
      -            "version": env!("CARGO_PKG_VERSION"),
      -            "description": "Factory skills for thirdshift's headless sessions",
      -        });
      -        fs::write(
      -            manifest_dir.join("plugin.json"),
      -            serde_json::to_string_pretty(&manifest)?,
      -        )?;
      -        Ok(Plugin { dir })
      -    }
      -
      -    pub fn path(&self) -> &Path {
      -        self.dir.path()
      -    }
      -}
      diff --git a/src/progress.rs b/src/progress.rs
      index b055a9701..8433d0fa2 100644
      --- a/src/progress.rs
      +++ b/src/progress.rs
      @@ -354,9 +354,9 @@ mod tests {
               assert_eq!(
                   line_for(tool_use(
                       "Skill",
      -                json!({ "skill": "thirdshift:tdd", "args": "x" })
      +                json!({ "skill": "thirdshift-tdd", "args": "x" })
                   )),
      -            Some("skill thirdshift:tdd".to_string())
      +            Some("skill thirdshift-tdd".to_string())
               );
           }
       
      diff --git a/src/prompt.rs b/src/prompt.rs
      index db382ea13..493e86e09 100644
      --- a/src/prompt.rs
      +++ b/src/prompt.rs
      @@ -23,10 +23,10 @@ const HEADLESS: &str = "You run headless: nobody is watching, and ending your tu
       /// The fresh prompt, for a run that starts a new Issue branch.
       pub fn fresh(issue: &IssueUrl, base: &str, branch: &str) -> String {
           format!(
      -        "/thirdshift:implement {url}\n\
      -         The base branch is {base}. Review with /thirdshift:code-review using {base} as the fixed point.\n\
      +        "/thirdshift-implement {url}\n\
      +         The base branch is {base}. Review with the `thirdshift-code-review` skill using {base} as the fixed point.\n\
                Address the Standards and Spec findings you agree with.\n\
      -         Push branch {branch} and create a pull request against {base} using /thirdshift:pr, marked ready for review.\n\
      +         Push branch {branch} and create a pull request against {base} using the `thirdshift-pr` skill, marked ready for review.\n\
                In the PR body, add an \"Unaddressed findings\" section listing each skipped finding under Standards or Spec, with at least a one-line reason.\n\
                Include \"Closes #{number}\" in the PR body.\n\
                {HEADLESS}",
      @@ -40,18 +40,18 @@ pub fn fresh(issue: &IssueUrl, base: &str, branch: &str) -> String {
       pub fn continuation(issue: &IssueUrl, base: &str, branch: &str, pr_url: Option<&str>) -> String {
           let pr = match pr_url {
               Some(url) => format!(
      -            "Update PR {url} using /thirdshift:pr, rewriting its body to cover the whole branch, marked ready for review."
      +            "Update PR {url} using the `thirdshift-pr` skill, rewriting its body to cover the whole branch, marked ready for review."
               ),
               None => format!(
      -            "Create a pull request against {base} using /thirdshift:pr, marked ready for review."
      +            "Create a pull request against {base} using the `thirdshift-pr` skill, marked ready for review."
               ),
           };
           format!(
      -        "/thirdshift:implement {url}\n\
      +        "/thirdshift-implement {url}\n\
                \n\
                You are continuing work on branch {branch}, which already has commits (see git log {base}..HEAD). Build on them; don't start over.\n\
                \n\
      -         The base branch is {base}. Review with /thirdshift:code-review using {base} as the fixed point.\n\
      +         The base branch is {base}. Review with the `thirdshift-code-review` skill using {base} as the fixed point.\n\
                \n\
                Address the Standards and Spec findings you agree with.\n\
                \n\
      @@ -73,17 +73,17 @@ pub fn continuation(issue: &IssueUrl, base: &str, branch: &str, pr_url: Option<&
       /// Ticket has landed on it, with the draft Spec PR `pr_url` into `base`.
       pub fn spec_review(spec: &IssueUrl, base: &str, branch: &str, pr_url: &str) -> String {
           format!(
      -        "/thirdshift:code-review {base}, with the Spec {url} as the spec\n\
      +        "/thirdshift-code-review {base}, with the Spec {url} as the spec\n\
                \n\
                Every Ticket of the Spec {url} has landed on branch {branch}, its Spec branch (see git log {base}..HEAD). Review the Spec as a whole, including how the Tickets' work fits together.\n\
                \n\
      -         Review with /thirdshift:code-review using {base} as the fixed point and {url} as the spec.\n\
      +         Review with the `thirdshift-code-review` skill using {base} as the fixed point and {url} as the spec.\n\
                \n\
      -         Address the Standards and Spec findings you agree with, using /thirdshift:tdd where it fits, and commit.\n\
      +         Address the Standards and Spec findings you agree with, using the `thirdshift-tdd` skill where it fits, and commit.\n\
                \n\
                Push branch {branch}. Do not rebase or force-push.\n\
                \n\
      -         Update PR {pr_url} using /thirdshift:pr, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done.\n\
      +         Update PR {pr_url} using the `thirdshift-pr` skill, rewriting its body to cover the whole Spec. Leave out its Tickets checklist, or keep it between its markers as it is: thirdshift puts it back. Leave the PR a draft: thirdshift marks it ready once you are done.\n\
                \n\
                In the PR body, add an \"Unaddressed findings\" section listing each skipped finding under Standards or Spec, with at least a one-line reason.\n\
                \n\
      @@ -113,14 +113,14 @@ pub fn architecture_review(base: &str, focus: Option<&str>) -> String {
               None => String::new(),
           };
           format!(
      -        "/thirdshift:improve-codebase-architecture\n\
      +        "/thirdshift-improve-codebase-architecture\n\
                \n\
                This is an Architecture review of the base branch {base}: this worktree is checked out at its head on origin, on no branch.\n\
                \n\
                {focus}\
      -         Find the deepening opportunities with /thirdshift:improve-codebase-architecture, using /thirdshift:codebase-design for the vocabulary. Skip any that an open issue already covers, and take the top recommendation.\n\
      +         Find the deepening opportunities with the `thirdshift-improve-codebase-architecture` skill, using the `thirdshift-codebase-design` skill for the vocabulary. Skip any that an open issue already covers, and take the top recommendation.\n\
                \n\
      -         If it is Strong, settle its design yourself and publish it as the plan with /thirdshift:to-spec and /thirdshift:to-tickets: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `{NEEDS_TRIAGE}`, not `{READY_FOR_AGENT}`: thirdshift marks it ready once you are done.\n\
      +         If it is Strong, settle its design yourself and publish it as the plan with the `thirdshift-to-spec` and `thirdshift-to-tickets` skills: a Spec with Tickets, or a single Ticket when one session is enough. Label the plan's top issue, the Spec or the single Ticket, `{NEEDS_TRIAGE}`, not `{READY_FOR_AGENT}`: thirdshift marks it ready once you are done.\n\
                \n\
                If it is not Strong, file it as one issue labelled `{NEEDS_TRIAGE}`, unless an open issue already covers it.\n\
                \n\
      @@ -140,7 +140,7 @@ pub fn architecture_review(base: &str, focus: Option<&str>) -> String {
       /// branch with Foreign commits on it.
       pub fn conflict_repair(issue: &IssueUrl, merging: &str, branch: &str, pr_url: &str) -> String {
           format!(
      -        "/thirdshift:resolving-merge-conflicts\n\
      +        "/thirdshift-resolving-merge-conflicts\n\
                \n\
                A merge of origin/{merging} into {branch} is in progress in this worktree and has conflicts.\n\
                {branch} implements {url}; its pull request is {pr_url}.\n\
      @@ -192,11 +192,11 @@ pub fn ci_fix_repair(
       /// the Issue branch on top of `own_head`, the head it last knew as its own.
       pub fn review_repair(issue: &IssueUrl, branch: &str, pr_url: &str, own_head: &str) -> String {
           format!(
      -        "/thirdshift:code-review {own_head}\n\
      +        "/thirdshift-code-review {own_head}\n\
                \n\
                Someone else pushed commits to {branch} while it was being worked on, and they have been merged into {branch} in this worktree. {branch} implements {url}; its pull request is {pr_url}. They are merged into the base branch only once you have reviewed them.\n\
                \n\
      -         Review with /thirdshift:code-review using {own_head} as the fixed point: {branch}'s head before their commits were merged in.\n\
      +         Review with the `thirdshift-code-review` skill using {own_head} as the fixed point: {branch}'s head before their commits were merged in.\n\
                \n\
                Address the Standards and Spec findings you agree with.\n\
                \n\
      diff --git a/src/prompts_page.rs b/src/prompts_page.rs
      index 6ebedce31..6f082cb86 100644
      --- a/src/prompts_page.rs
      +++ b/src/prompts_page.rs
      @@ -14,9 +14,9 @@ use include_dir::{Dir, File};
       use crate::ci::FailedChecks;
       use crate::github::{Check, CheckState};
       use crate::issue::IssueUrl;
      -use crate::plugin::SKILLS;
       use crate::prompt;
       use crate::session::claude_args;
      +use crate::skills::SKILLS;
       
       use Sender::{ArchitectRun, Units};
       
      @@ -55,10 +55,9 @@ const CHECK_URL: &str = "";
       const INHERITED_CHECK: &str = "";
       const BACKGROUND_WORK: &str = "";
       const FOCUS: &str = "";
      -const PLUGIN_DIR: &str = "";
       const SESSION_ID: &str = "";
       const PROMPT: &str = "";
      -const PLACEHOLDERS: [&str; 16] = [
      +const PLACEHOLDERS: [&str; 15] = [
           ISSUE_URL,
           SPEC_URL,
           NUMBER,
      @@ -72,7 +71,6 @@ const PLACEHOLDERS: [&str; 16] = [
           INHERITED_CHECK,
           BACKGROUND_WORK,
           FOCUS,
      -    PLUGIN_DIR,
           SESSION_ID,
           PROMPT,
       ];
      @@ -266,7 +264,7 @@ fn readme(prompts: &[Prompt]) -> String {
       
       Every Session prompt thirdshift can start or resume an agent session with, one file each, with placeholders such as `{BRANCH}` where each Run fills in its own values. The [Prompts and skills page](https://thirdshift.app/prompts/) shows the same prompts.
       
      -Every session runs `claude` headless in the Run's worktree, in auto permission mode, with the Factory skills loaded as a plugin from a temporary directory:
      +Every session runs `claude` headless in the Run's worktree, in auto permission mode. The Factory skills are linked into the worktree's `.claude/skills/`, where Claude Code finds them, and kept out of git:
       
       ```sh
       {new}
      @@ -295,7 +293,7 @@ A [Resume]({resume_file}) continues the session that ended:
               );
           }
           readme.push_str(
      -        "\nThe Factory skills the prompts name are in [`skills/`](../skills/), as the plugin writes them out.\n",
      +        "\nThe Factory skills the prompts name are in [`skills/`](../skills/), as thirdshift writes them out.\n",
           );
           readme
       }
      @@ -322,10 +320,15 @@ fn fence(text: &str) -> String {
       /// when, if not always; or the Architecture review, which is no unit of a Run.
       fn skill_used_by(skill: &str) -> String {
           match skill {
      -        "implement" | "tdd" => unit_links(&[IMPLEMENT]),
      -        "code-review" | "pr" => unit_links(&[REVIEW]),
      -        "resolving-merge-conflicts" => format!("{}, in a Repair", unit_links(&[FINISH])),
      -        "improve-codebase-architecture" | "to-spec" | "to-tickets" | "codebase-design" => {
      +        "thirdshift-implement" | "thirdshift-tdd" => unit_links(&[IMPLEMENT]),
      +        "thirdshift-code-review" | "thirdshift-pr" => unit_links(&[REVIEW]),
      +        "thirdshift-resolving-merge-conflicts" => {
      +            format!("{}, in a Repair", unit_links(&[FINISH]))
      +        }
      +        "thirdshift-improve-codebase-architecture"
      +        | "thirdshift-to-spec"
      +        | "thirdshift-to-tickets"
      +        | "thirdshift-codebase-design" => {
                   "the Architecture review, in an Architect run".to_string()
               }
               _ => panic!("the Factory skill {skill} has no user: add it to skill_used_by"),
      @@ -346,9 +349,7 @@ fn render() -> String {
       /// The `claude` command line a session starts with, or, given a session id,
       /// resumes with, with the page's placeholders.
       fn command(resume: Option<&str>) -> String {
      -    let args = claude_args(OsStr::new(PLUGIN_DIR), resume, PROMPT);
      -    let args: Vec<_> = args.iter().map(|arg| arg.to_string_lossy()).collect();
      -    format!("claude {}", args.join(" "))
      +    format!("claude {}", claude_args(resume, PROMPT).join(" "))
       }
       
       fn command_line(html: &mut String) {
      @@ -357,7 +358,7 @@ fn command_line(html: &mut String) {
               r##"

      The command line

      How each session starts

      -

      Every session runs claude headless in the Run's worktree, in auto permission mode, with the Factory skills loaded as a plugin from a temporary directory. The prompt is one of the templates below.

      +

      Every session runs claude headless in the Run's worktree, in auto permission mode. The Factory skills are linked into the worktree's .claude/skills/, where Claude Code finds them, and kept out of git. The prompt is one of the templates below.

      {new}

      A Resume continues the session that ended:

      {resume}
      @@ -403,7 +404,7 @@ fn skills_section(html: &mut String) { r##"

      Factory skills

      The skills, as the agent reads them

      -

      Every Factory skill and supporting file, raw, exactly as the plugin writes it out.

      +

      Every Factory skill and supporting file, raw, exactly as thirdshift writes it out.

      "##, ); @@ -441,7 +442,7 @@ fn licence_section(html: &mut String) { r##"

      Licence and credits

      Whose skills these are

      -

      The Factory skills are adapted from Matt Pocock's skills, under this licence. The pr skill credits Dex Horthy's show-me skill in its CREDITS.md.

      +

      The Factory skills are adapted from Matt Pocock's skills, under this licence. The thirdshift-pr skill credits Dex Horthy's show-me skill in its CREDITS.md.

      "##, ); let mut files: Vec<&File> = SKILLS.files().collect(); diff --git a/src/run.rs b/src/run.rs index 73bca72f6..a2729e277 100644 --- a/src/run.rs +++ b/src/run.rs @@ -124,8 +124,8 @@ pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended /// Take `issue` to a ready PR, or in a Merge run a merged one. Any failure /// after the worktree exists, including a merge that fails, goes through the -/// Failed run path. The worktree, the local -/// Issue branch and the plugin directory are gone when this returns, except +/// Failed run path. The worktree and the local +/// Issue branch are gone when this returns, except /// that a Failed run whose work did not reach origin keeps the worktree and /// branch. With `asks.launch_pull`, the Launch directory's checkout of the /// Base branch, if that is the branch checked out, is first brought up to diff --git a/src/session.rs b/src/session.rs index da5abe292..97cdf5480 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,7 +1,6 @@ //! Headless Claude Code sessions and their logs. use std::cell::RefCell; -use std::ffi::OsStr; use std::fs::{self, File}; use std::io::{BufRead, BufReader, Read, Write}; use std::os::unix::process::CommandExt; @@ -15,9 +14,9 @@ use anyhow::{Context, Result, anyhow, bail}; use crate::interrupt; use crate::issue::{IssueUrl, Repo}; use crate::logs; -use crate::plugin::Plugin; use crate::progress::{self, Progress}; use crate::prompt; +use crate::skills; /// Where a Run's or an Architect run's Session logs go: in `sessions/` under /// the root of its repository's logs, under the User config's `logs.dir` or @@ -57,12 +56,11 @@ impl Logs { } /// Where a Run's or an Architect run's sessions run: in `worktree`, with the -/// Factory skills plugin loaded, each logged in its `logs`. It lasts for the -/// steps given to [`Sessions::within`], and so does the plugin. +/// Factory skills linked into it, each logged in its `logs`. It lasts for the +/// steps given to [`Sessions::within`]. pub struct Sessions<'a> { logs: &'a Logs, worktree: &'a Path, - plugin: Plugin, /// The log of the session started last, a Resume included. last_log: RefCell>, /// How each session ended whose last ending, its Resume's if it got one, @@ -73,26 +71,24 @@ pub struct Sessions<'a> { } impl<'a> Sessions<'a> { - /// Write the Factory skills plugin, then take `steps`, which run their - /// sessions through the `Sessions` they are given. Returns what `steps` - /// came to, with the most recent session's log, if a session created it. - /// If a session's last ending, its Resume's if it got one, left - /// background work to be killed and `steps` then fail, the failure names - /// that work ahead of its own cause: the session may have stopped short - /// of its job. The plugin directory is gone when this returns. + /// Link the Factory skills into `worktree`, then take `steps`, which run + /// their sessions through the `Sessions` they are given. Returns what + /// `steps` came to, with the most recent session's log, if a session + /// created it. If a session's last ending, its Resume's if it got one, + /// left background work to be killed and `steps` then fail, the failure + /// names that work ahead of its own cause: the session may have stopped + /// short of its job. pub fn within( logs: &'a Logs, worktree: &'a Path, steps: impl FnOnce(&Self) -> Result, ) -> (Result, Option) { - let plugin = match Plugin::write() { - Ok(plugin) => plugin, - Err(error) => return (Err(error), None), - }; + if let Err(error) = skills::link_into(worktree) { + return (Err(error), None); + } let sessions = Sessions { logs, worktree, - plugin, last_log: RefCell::default(), endings_with_killed_work: RefCell::default(), }; @@ -153,14 +149,7 @@ impl<'a> Sessions<'a> { let log = self.logs.path(kind); progress::step(format_args!("logging the session to {}", log.display())); *self.last_log.borrow_mut() = Some(log.clone()); - run( - kind, - self.worktree, - self.plugin.path(), - resume, - prompt, - &log, - ) + run(kind, self.worktree, resume, prompt, &log) } } @@ -176,8 +165,8 @@ fn ending_with(killed: &[&str]) -> String { } } -/// Run `claude` headless in auto mode in `worktree`, with the Factory skills -/// plugin at `plugin_dir` loaded, streaming its output to `log` and condensing +/// Run `claude` headless in auto mode in `worktree`, where it finds the +/// Factory skills, streaming its output to `log` and condensing /// it to progress lines on stderr, each labelled `kind`. With `resume`, the /// session with that id continues, given `prompt`. Returns what the stream /// showed once `claude` has exited cleanly. An interrupt stops the session and @@ -185,7 +174,6 @@ fn ending_with(killed: &[&str]) -> String { fn run( kind: &str, worktree: &Path, - plugin_dir: &Path, resume: Option<&str>, prompt: &str, log: &Path, @@ -197,7 +185,7 @@ fn run( File::create(log).with_context(|| format!("could not create {}", log.display()))?; let started = Instant::now(); let mut child = Command::new("claude") - .args(claude_args(plugin_dir.as_os_str(), resume, prompt)) + .args(claude_args(resume, prompt)) .current_dir(worktree) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -262,23 +250,21 @@ fn run( } /// The arguments every session runs `claude` with: headless in auto mode, -/// with the Factory skills plugin at `plugin_dir` loaded, streaming JSON. With -/// `resume`, the session with that id continues. The prompt comes last. -pub fn claude_args<'a>( - plugin_dir: &'a OsStr, - resume: Option<&'a str>, - prompt: &'a str, -) -> Vec<&'a OsStr> { - let mut args: Vec<&OsStr> = ["-p", "--permission-mode", "auto", "--plugin-dir"] - .into_iter() - .map(OsStr::new) - .collect(); - args.push(plugin_dir); - args.extend(["--output-format", "stream-json", "--verbose"].map(OsStr::new)); +/// streaming JSON. With `resume`, the session with that id continues. The +/// prompt comes last. +pub fn claude_args<'a>(resume: Option<&'a str>, prompt: &'a str) -> Vec<&'a str> { + let mut args = vec![ + "-p", + "--permission-mode", + "auto", + "--output-format", + "stream-json", + "--verbose", + ]; if let Some(session_id) = resume { - args.extend(["--resume", session_id].map(OsStr::new)); + args.extend(["--resume", session_id]); } - args.push(OsStr::new(prompt)); + args.push(prompt); args } diff --git a/src/skills.rs b/src/skills.rs new file mode 100644 index 000000000..a9acccd89 --- /dev/null +++ b/src/skills.rs @@ -0,0 +1,113 @@ +//! The Factory skills, embedded at compile time (ADR-0001), written out once +//! per Command and linked into each worktree its sessions run in, where +//! Claude Code finds project skills (ADR-0012). + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::symlink; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, PoisonError}; + +use anyhow::{Context, Result}; +use include_dir::{Dir, include_dir}; +use tempfile::TempDir; + +use crate::git::Git; + +/// The Factory skills, as they are written out and the Prompts and skills +/// page shows them, each in its `thirdshift-` directory. +pub static SKILLS: Dir = include_dir!("$CARGO_MANIFEST_DIR/skills"); + +/// Where in a worktree Claude Code finds its project skills. +const PROJECT_SKILLS: &str = ".claude/skills"; + +/// The `.git/info/exclude` pattern that keeps the linked skills out of git. +const EXCLUDE: &str = "/.claude/skills/thirdshift-*"; + +/// The temp directory the Command wrote the Factory skills out to, once it +/// has. +static WRITTEN: Mutex> = Mutex::new(None); + +/// Link every Factory skill into `worktree`'s `.claude/skills/`, writing them +/// out first if the Command has not yet, and make sure the repository's +/// `.git/info/exclude` keeps them out of git. The links are left in place: +/// they go with the worktree. +pub fn link_into(worktree: &Path) -> Result<()> { + let written = written()?; + exclude(&Git::new(worktree))?; + let project_skills = worktree.join(PROJECT_SKILLS); + fs::create_dir_all(&project_skills) + .with_context(|| format!("could not create {}", project_skills.display()))?; + for skill in SKILLS.dirs() { + let name = skill.path(); + let link = project_skills.join(name); + if fs::symlink_metadata(&link).is_ok() { + fs::remove_file(&link) + .with_context(|| format!("could not replace {}", link.display()))?; + } + symlink(written.join(name), &link) + .with_context(|| format!("could not link {}", link.display()))?; + } + Ok(()) +} + +/// Remove the Factory skills the Command wrote out, if it did, once its +/// sessions are over. +pub fn remove() { + drop( + WRITTEN + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(), + ); +} + +/// Where the Command wrote the Factory skills out, writing them now if it +/// has not yet. +fn written() -> Result { + let mut written = WRITTEN.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(dir) = &*written { + return Ok(dir.path().to_owned()); + } + let dir = tempfile::Builder::new() + .prefix("thirdshift-skills-") + .tempdir()?; + SKILLS + .extract(dir.path()) + .context("could not write the Factory skills")?; + let path = dir.path().to_owned(); + *written = Some(dir); + Ok(path) +} + +/// Add [`EXCLUDE`] to the `.git/info/exclude` of the repository `git` works +/// in, unless it is there already. The file is shared by every worktree, so +/// the entry is left there, under a lock other Runs from the same Launch +/// directory take too. +fn exclude(git: &Git) -> Result<()> { + let _lock = git.lock("thirdshift-exclude.lock")?; + let info = git.common_dir()?.join("info"); + let path = info.join("exclude"); + let existing = match fs::read_to_string(&path) { + Ok(existing) => existing, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => { + return Err(error).with_context(|| format!("could not read {}", path.display())); + } + }; + if existing.lines().any(|line| line == EXCLUDE) { + return Ok(()); + } + fs::create_dir_all(&info).with_context(|| format!("could not create {}", info.display()))?; + let separator = if existing.is_empty() || existing.ends_with('\n') { + "" + } else { + "\n" + }; + OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .and_then(|mut file| writeln!(file, "{separator}{EXCLUDE}")) + .with_context(|| format!("could not add the Factory skills to {}", path.display())) +} diff --git a/tests/architect_run.rs b/tests/architect_run.rs index e4670fc70..86231c3b1 100644 --- a/tests/architect_run.rs +++ b/tests/architect_run.rs @@ -335,10 +335,9 @@ fn the_session_prompt_names_the_factory_skills_the_base_branch_and_the_final_lin assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); let prompt = scenario.first_prompt(); for expected in [ - "/thirdshift:improve-codebase-architecture", - "/thirdshift:codebase-design", - "/thirdshift:to-spec", - "/thirdshift:to-tickets", + "/thirdshift-improve-codebase-architecture\n", + "the `thirdshift-codebase-design` skill", + "the `thirdshift-to-spec` and `thirdshift-to-tickets` skills", "the base branch develop", "\nArchitecture review plan: <", "\nArchitecture review idea: <", @@ -816,7 +815,7 @@ fn a_single_ticket_plan_starts_a_run_on_it_and_ends_as_that_run_does() { assert_eq!(calls.len(), 2, "sessions: {calls:?}"); let prompt = calls[1]["prompt"].as_str().unwrap(); assert!( - prompt.starts_with(&format!("/thirdshift:implement {PLAN_URL}\n")), + prompt.starts_with(&format!("/thirdshift-implement {PLAN_URL}\n")), "{prompt}" ); assert_eq!(calls[1]["branch"], "issue-8"); @@ -1084,6 +1083,9 @@ fn base_fix_reaches_each_tickets_run_of_the_dispatched_spec_run() { for file in ["issue-9.txt", "issue-10.txt", "ci-fix.txt"] { assert!(scenario.origin_file("issue-8", file).is_some(), "{file}"); } + // The Architecture review, each Ticket's Run, the Base fix and the Spec + // review. + scenario.assert_every_session_found_the_factory_skills(); } #[test] @@ -1825,7 +1827,7 @@ fn a_second_architect_run_is_skipped_while_the_firsts_dispatched_run_or_spec_run assert_ended_with_pr(&first, &pr_from(&scenario, "issue-8"), "ready for review"); let reviews = scenario.claude_calls().into_iter().filter(|call| { let prompt = call["prompt"].as_str().unwrap(); - prompt.contains("/thirdshift:improve-codebase-architecture") + prompt.contains("/thirdshift-improve-codebase-architecture") }); assert_eq!(reviews.count(), 1, "a second review was started"); } diff --git a/tests/background_work.rs b/tests/background_work.rs index 2bc9c6467..770e3c06a 100644 --- a/tests/background_work.rs +++ b/tests/background_work.rs @@ -51,6 +51,7 @@ fn a_session_whose_background_work_was_killed_is_resumed_once_and_the_run_goes_o "argv: {argv:?}" ); assert_eq!(calls[1]["cwd"], calls[0]["cwd"]); + scenario.assert_every_session_found_the_factory_skills(); let prompt = calls[1]["prompt"].as_str().unwrap(); assert!( prompt.contains("was killed when your turn ended") && prompt.contains("in the foreground"), diff --git a/tests/continuation.rs b/tests/continuation.rs index 1941fcd2d..0fee77730 100644 --- a/tests/continuation.rs +++ b/tests/continuation.rs @@ -80,7 +80,7 @@ fn the_open_prs_base_is_the_base_branch_whatever_is_checked_out() { let prompt = scenario.first_prompt(); assert!( prompt.contains("(see git log main..HEAD)") - && prompt.contains("The base branch is main. Review with /thirdshift:code-review using main as the fixed point.") + && prompt.contains("The base branch is main. Review with the `thirdshift-code-review` skill using main as the fixed point.") && !prompt.contains("develop"), "prompt: {prompt}" ); diff --git a/tests/fakes/claude.rs b/tests/fakes/claude.rs index 13b9696ba..684741aa5 100644 --- a/tests/fakes/claude.rs +++ b/tests/fakes/claude.rs @@ -2,8 +2,10 @@ //! //! Each call appends a record to the JSON list in $FAKE_CLAUDE_RECORD: its //! argv, prompt, working directory, the branch checked out there, whether a -//! merge is in progress there, and a snapshot of the --plugin-dir contents -//! (the directory is gone by the time a test looks). +//! merge is in progress there, a snapshot of the `thirdshift-*` skills in its +//! `.claude/skills/`, through their links, and of the files written out +//! beside the skills those links point to (the links' targets are gone by +//! the time a test looks). //! //! It then runs a per-test bash script in the working directory, emits a //! couple of stream-json lines, and exits with the script's exit code. A @@ -42,25 +44,72 @@ use std::process::{Command, ExitStatus}; use crate::json::{Array, Bool, Json, Null, object, string}; -/// Every file under `plugin_dir`, by its path relative to it, with its -/// contents. -fn plugin_snapshot(plugin_dir: &Path) -> Json { - let mut files = object([]); - let mut dirs = vec![plugin_dir.to_owned()]; +/// Where Claude Code finds a worktree's project skills. +const SKILLS: &str = ".claude/skills"; + +/// Every file under `dir`, by its path relative to `root`, with its +/// contents, into `files`. Like Python's os.walk, which the fake was first +/// written with, it doesn't follow a symlink to a directory below `dir`. +fn snapshot_into(files: &mut Json, root: &Path, dir: &Path) { + let mut dirs = vec![dir.to_owned()]; while let Some(dir) = dirs.pop() { for entry in fs::read_dir(&dir).unwrap() { let path = entry.unwrap().path(); - // Like Python's os.walk, which the fake was first written with, - // this doesn't follow a symlink to a directory. if fs::symlink_metadata(&path).unwrap().is_dir() { dirs.push(path); } else if !path.is_dir() { - let relative = path.strip_prefix(plugin_dir).unwrap(); + let relative = path.strip_prefix(root).unwrap(); let contents = String::from_utf8_lossy(&fs::read(&path).unwrap()).into_owned(); files.set(relative.to_str().unwrap(), string(contents)); } } } +} + +/// The `thirdshift-*` entries of the working directory's `.claude/skills/`. +fn linked_skills() -> Vec { + let Ok(entries) = fs::read_dir(SKILLS) else { + return Vec::new(); + }; + entries + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("thirdshift-")) + }) + .collect() +} + +/// Every file of the `thirdshift-*` skills in the working directory's +/// `.claude/skills/`, through their links, by its path relative to +/// `.claude/skills/`, with its contents. +fn skills_snapshot() -> Json { + let mut files = object([]); + for skill in linked_skills() { + snapshot_into(&mut files, Path::new(SKILLS), &skill); + } + files +} + +/// The files written out beside the skills the `thirdshift-*` links in the +/// working directory's `.claude/skills/` point to, such as their licence, by +/// name, with their contents. +fn beside_skills_snapshot() -> Json { + let mut files = object([]); + let written = linked_skills() + .first() + .and_then(|link| fs::read_link(link).ok()) + .and_then(|target| target.parent().map(Path::to_owned)); + if let Some(written) = written { + for entry in fs::read_dir(&written).unwrap() { + let path = entry.unwrap().path(); + if path.is_file() { + let name = path.file_name().unwrap().to_str().unwrap(); + files.set(name, string(fs::read_to_string(&path).unwrap())); + } + } + } files } @@ -122,10 +171,6 @@ fn exit_code(status: ExitStatus) -> i32 { } pub fn main(argv: Vec) { - let plugin_dir = argv - .iter() - .position(|arg| arg == "--plugin-dir") - .map(|at| PathBuf::from(&argv[at + 1])); let (branch, _) = git_here(&["branch", "--show-current"]); let (_, merging) = git_here(&["rev-parse", "-q", "--verify", "MERGE_HEAD"]); let text_mode = !argv.iter().any(|arg| arg == "--output-format"); @@ -150,20 +195,8 @@ pub fn main(argv: Vec) { ("cwd", string(cwd)), ("branch", string(branch)), ("merging", Bool(merging)), - ( - "plugin_dir", - plugin_dir - .as_ref() - .map(|dir| string(dir.to_str().unwrap())) - .unwrap_or(Null), - ), - ( - "plugin_files", - plugin_dir - .as_deref() - .map(plugin_snapshot) - .unwrap_or(object([])), - ), + ("skill_files", skills_snapshot()), + ("beside_skills", beside_skills_snapshot()), ("stdin", stdin), ]), ); diff --git a/tests/fresh_run.rs b/tests/fresh_run.rs index 9d64fb97f..65660c1f0 100644 --- a/tests/fresh_run.rs +++ b/tests/fresh_run.rs @@ -68,7 +68,7 @@ fn runs_claude_headless_in_auto_mode_in_a_sibling_worktree_on_the_issue_branch() "argv: {argv:?}" ); assert!(argv.contains(&"--verbose"), "argv: {argv:?}"); - assert!(argv.contains(&"--plugin-dir"), "argv: {argv:?}"); + assert!(!argv.contains(&"--plugin-dir"), "argv: {argv:?}"); assert_eq!( call["cwd"], scenario.path("work/widgets-issue-7").to_str().unwrap() @@ -85,10 +85,10 @@ fn gives_the_agent_the_fresh_prompt() { assert_eq!( scenario.claude_calls()[0]["prompt"], - "/thirdshift:implement https://github.com/acme/widgets/issues/7\n\ - The base branch is main. Review with /thirdshift:code-review using main as the fixed point.\n\ + "/thirdshift-implement https://github.com/acme/widgets/issues/7\n\ + The base branch is main. Review with the `thirdshift-code-review` skill using main as the fixed point.\n\ Address the Standards and Spec findings you agree with.\n\ - Push branch issue-7 and create a pull request against main using /thirdshift:pr, marked ready for review.\n\ + Push branch issue-7 and create a pull request against main using the `thirdshift-pr` skill, marked ready for review.\n\ In the PR body, add an \"Unaddressed findings\" section listing each skipped finding under Standards or Spec, with at least a one-line reason.\n\ Include \"Closes #7\" in the PR body.\n\ You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.\n" @@ -96,46 +96,23 @@ fn gives_the_agent_the_fresh_prompt() { } #[test] -fn loads_every_factory_skill_as_the_thirdshift_plugin() { +fn finds_every_factory_skill_in_the_worktree_as_thirdshift_skill() { let scenario = Scenario::new(); scenario.agent_does(AGENT_COMMITS_AND_OPENS_PR); scenario.run(&[&scenario.issue_url(7)]); - let files = &scenario.claude_calls()[0]["plugin_files"]; - let manifest: serde_json::Value = serde_json::from_str( - files[".claude-plugin/plugin.json"] - .as_str() - .expect("no plugin manifest"), - ) - .unwrap(); - assert_eq!(manifest["name"], "thirdshift"); - for skill in [ - "implement", - "code-review", - "pr", - "tdd", - "resolving-merge-conflicts", - "improve-codebase-architecture", - "to-spec", - "to-tickets", - "codebase-design", - ] { - let skill_md = files[format!("skills/{skill}/SKILL.md")].as_str(); - assert!( - skill_md.is_some_and(|text| text.contains(&format!("name: {skill}"))), - "skill {skill} missing from the plugin" - ); - } + scenario.assert_every_session_found_the_factory_skills(); + let files = &scenario.claude_calls()[0]["skill_files"]; for supporting in [ - "skills/tdd/tests.md", - "skills/improve-codebase-architecture/REPORT.md", - "skills/codebase-design/DEEPENING.md", - "skills/codebase-design/DESIGN-IT-TWICE.md", + "thirdshift-tdd/tests.md", + "thirdshift-improve-codebase-architecture/REPORT.md", + "thirdshift-codebase-design/DEEPENING.md", + "thirdshift-codebase-design/DESIGN-IT-TWICE.md", ] { assert!( files[supporting].is_string(), - "the supporting file {supporting} is missing from the plugin" + "the supporting file {supporting} is missing from the worktree's .claude/skills/" ); } } @@ -148,7 +125,7 @@ fn ships_the_mattpocock_skills_mit_notice_with_the_factory_skills() { scenario.run(&[&scenario.issue_url(7)]); let calls = scenario.claude_calls(); - let license = calls[0]["plugin_files"]["skills/LICENSE"].as_str(); + let license = calls[0]["beside_skills"]["LICENSE"].as_str(); assert!( license .is_some_and(|text| text.contains("MIT License") @@ -157,6 +134,56 @@ fn ships_the_mattpocock_skills_mit_notice_with_the_factory_skills() { ); } +#[test] +fn the_linked_skills_are_kept_out_of_git() { + let scenario = Scenario::new(); + let status = scenario.path("status.txt"); + scenario.agent_does(&format!( + "git status --porcelain --untracked-files=all > '{}'\n\ + echo feature > feature.txt\n\ + git add -A\n\ + git commit -q -m 'Add everything'\n\ + gh pr create --base main --head issue-7 --title 'Add feature' --body 'Closes #7'\n", + status.display() + )); + + let result = scenario.run(&[&scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_eq!(std::fs::read_to_string(&status).unwrap(), ""); + let committed = scenario.origin_git(&["ls-tree", "-r", "--name-only", "issue-7"]); + assert!(committed.contains("feature.txt"), "committed: {committed}"); + assert!(!committed.contains(".claude"), "committed: {committed}"); +} + +#[test] +fn a_second_run_adds_no_second_exclude_entry() { + let scenario = Scenario::new(); + scenario.issue_is(8, "OPEN"); + scenario.agent_does_for(7, AGENT_COMMITS_AND_OPENS_PR); + scenario.agent_does_for( + 8, + "echo other > other.txt\ngit add other.txt\ngit commit -q -m 'Add other'\n\ + gh pr create --base main --head issue-8 --title 'Add other' --body 'Closes #8'\n", + ); + + let first = scenario.run(&[&scenario.issue_url(7)]); + let second = scenario.run(&[&scenario.issue_url(8)]); + + assert_eq!(first.code, Some(0), "stderr: {}", first.stderr); + assert_eq!(second.code, Some(0), "stderr: {}", second.stderr); + let exclude = std::fs::read_to_string(scenario.launch_dir().join(".git/info/exclude")).unwrap(); + let entries: Vec<_> = exclude + .lines() + .filter(|line| line.contains("thirdshift-")) + .collect(); + assert_eq!( + entries, + ["/.claude/skills/thirdshift-*"], + "exclude: {exclude}" + ); +} + #[test] fn logs_the_session_stream_under_the_home_directory() { let scenario = Scenario::new(); diff --git a/tests/merge_run.rs b/tests/merge_run.rs index 6f359c22c..e8b6a6bc6 100644 --- a/tests/merge_run.rs +++ b/tests/merge_run.rs @@ -322,12 +322,12 @@ fn assert_review_repair_from(prompt: &serde_json::Value, fixed_point: &str) { let prompt = prompt.as_str().unwrap(); let fixed_point = fixed_point.trim(); assert!( - prompt.starts_with("/thirdshift:code-review"), + prompt.starts_with("/thirdshift-code-review"), "prompt: {prompt}" ); assert!( prompt.contains(&format!( - "Review with /thirdshift:code-review using {fixed_point} as the fixed point" + "Review with the `thirdshift-code-review` skill using {fixed_point} as the fixed point" )), "prompt: {prompt}" ); @@ -404,7 +404,7 @@ fn a_foreign_commit_that_conflicts_with_local_work_gets_a_conflict_repair_then_a assert_eq!(calls[2]["merging"], true); let conflict = calls[2]["prompt"].as_str().unwrap(); assert!( - conflict.starts_with("/thirdshift:resolving-merge-conflicts") + conflict.starts_with("/thirdshift-resolving-merge-conflicts") && conflict.contains("A merge of origin/issue-7 into issue-7 is in progress"), "prompt: {conflict}" ); diff --git a/tests/mergeable.rs b/tests/mergeable.rs index 5377b992e..2e1c5453f 100644 --- a/tests/mergeable.rs +++ b/tests/mergeable.rs @@ -107,7 +107,7 @@ fn hands_a_conflicting_merge_to_a_conflict_repair_and_pushes_its_resolution() { let repair = &calls[1]; assert_eq!( repair["prompt"], - "/thirdshift:resolving-merge-conflicts\n\ + "/thirdshift-resolving-merge-conflicts\n\ \n\ A merge of origin/main into issue-7 is in progress in this worktree and has conflicts.\n\ issue-7 implements https://github.com/acme/widgets/issues/7; its pull request is https://github.com/acme/widgets/pull/1.\n\ @@ -123,7 +123,8 @@ fn hands_a_conflicting_merge_to_a_conflict_repair_and_pushes_its_resolution() { let argv = call["argv"].as_array().unwrap().clone(); argv[..argv.len() - 1].to_vec() }; - assert_eq!(flags(repair), flags(&calls[0]), "same plugin and flags"); + assert_eq!(flags(repair), flags(&calls[0]), "same flags"); + scenario.assert_every_session_found_the_factory_skills(); assert_merged_main_into_issue_7(&scenario); assert_eq!( scenario.origin_git(&["show", "issue-7:feature.txt"]), diff --git a/tests/numbered_branches.rs b/tests/numbered_branches.rs index a776b8b28..dcdd64f26 100644 --- a/tests/numbered_branches.rs +++ b/tests/numbered_branches.rs @@ -30,7 +30,7 @@ fn the_next_numbered_branch_gets_the_fresh_prompt() { let prompt = scenario.first_prompt(); assert!( prompt.contains( - "\nPush branch issue-7-branch-2 and create a pull request against main using /thirdshift:pr, marked ready for review.\n" + "\nPush branch issue-7-branch-2 and create a pull request against main using the `thirdshift-pr` skill, marked ready for review.\n" ) && !prompt.contains("You are continuing work"), "prompt: {prompt}" ); diff --git a/tests/packaging.rs b/tests/packaging.rs index 318656704..1b9315e63 100644 --- a/tests/packaging.rs +++ b/tests/packaging.rs @@ -75,15 +75,15 @@ fn package_carries_the_factory_skills_with_their_license_and_credits() { let list = package_list(); for expected in [ "skills/LICENSE", - "skills/pr/CREDITS.md", - "skills/implement/SKILL.md", - "skills/improve-codebase-architecture/SKILL.md", - "skills/improve-codebase-architecture/REPORT.md", - "skills/to-spec/SKILL.md", - "skills/to-tickets/SKILL.md", - "skills/codebase-design/SKILL.md", - "skills/codebase-design/DEEPENING.md", - "skills/codebase-design/DESIGN-IT-TWICE.md", + "skills/thirdshift-pr/CREDITS.md", + "skills/thirdshift-implement/SKILL.md", + "skills/thirdshift-improve-codebase-architecture/SKILL.md", + "skills/thirdshift-improve-codebase-architecture/REPORT.md", + "skills/thirdshift-to-spec/SKILL.md", + "skills/thirdshift-to-tickets/SKILL.md", + "skills/thirdshift-codebase-design/SKILL.md", + "skills/thirdshift-codebase-design/DEEPENING.md", + "skills/thirdshift-codebase-design/DESIGN-IT-TWICE.md", "src/main.rs", "README.md", "LICENSE", diff --git a/tests/pickup_run.rs b/tests/pickup_run.rs index 2e2ea7f2d..1320fc729 100644 --- a/tests/pickup_run.rs +++ b/tests/pickup_run.rs @@ -100,7 +100,7 @@ fn the_lower_numbered_of_two_ready_issues_is_run_and_the_other_left_untouched() let prompt = calls[0]["prompt"].as_str().unwrap(); assert!( prompt.starts_with(&format!( - "/thirdshift:implement {}\n", + "/thirdshift-implement {}\n", scenario.issue_url(7) )), "{prompt}" diff --git a/tests/progress_lines.rs b/tests/progress_lines.rs index b499dd560..05879c5f1 100644 --- a/tests/progress_lines.rs +++ b/tests/progress_lines.rs @@ -43,7 +43,7 @@ fn prints_one_line_per_notable_session_event() { "{}{}{}{}{AGENT_COMMITS_AND_OPENS_PR}", emits(tool_use( "Skill", - json!({ "skill": "thirdshift:implement" }) + json!({ "skill": "thirdshift-implement" }) )), emits(tool_use( "Read", @@ -65,7 +65,7 @@ fn prints_one_line_per_notable_session_event() { let lines = stderr_lines(&result.stderr); for expected in [ "thirdshift: implement: session started", - "thirdshift: implement: skill thirdshift:implement", + "thirdshift: implement: skill thirdshift-implement", "thirdshift: implement: Read src/lib.rs", "thirdshift: implement: commit", "thirdshift: implement: push", diff --git a/tests/spec_run.rs b/tests/spec_run.rs index ee50017ee..330376e53 100644 --- a/tests/spec_run.rs +++ b/tests/spec_run.rs @@ -509,7 +509,7 @@ fn a_base_branch_that_moved_into_a_conflict_gets_a_conflict_repair_and_the_spec_ assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); let prompts = spec_prompts(&scenario); assert_eq!(prompts.len(), 2, "{prompts:?}"); - assert_contains(&prompts[1], "/thirdshift:resolving-merge-conflicts"); + assert_contains(&prompts[1], "/thirdshift-resolving-merge-conflicts"); assert_contains(&prompts[1], "A merge of origin/main into issue-20"); assert_contains(&prompts[1], &spec_url(&scenario)); let spec = spec_pr(&scenario); @@ -970,11 +970,11 @@ fn once_the_last_ticket_lands_the_spec_review_reviews_the_spec_branch_against_th assert_eq!(review["branch"], "issue-20"); let prompt = review["prompt"].as_str().unwrap(); for part in [ - "/thirdshift:code-review main, with the Spec https://github.com/acme/widgets/issues/20 as the spec\n", + "/thirdshift-code-review main, with the Spec https://github.com/acme/widgets/issues/20 as the spec\n", &spec_url(&scenario), "using main as the fixed point", - "/thirdshift:tdd", - "Update PR https://github.com/acme/widgets/pull/2 using /thirdshift:pr", + "the `thirdshift-tdd` skill", + "Update PR https://github.com/acme/widgets/pull/2 using the `thirdshift-pr` skill", "\"Unaddressed findings\"", "Include \"Closes #20\"", "You run headless", diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 44df8a240..d3afd916b 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -47,6 +47,19 @@ use tempfile::TempDir; pub const OWNER: &str = "acme"; pub const REPO: &str = "widgets"; +/// Every Factory skill, by the name a session finds it under. +pub const FACTORY_SKILLS: [&str; 9] = [ + "thirdshift-implement", + "thirdshift-code-review", + "thirdshift-pr", + "thirdshift-tdd", + "thirdshift-resolving-merge-conflicts", + "thirdshift-improve-codebase-architecture", + "thirdshift-to-spec", + "thirdshift-to-tickets", + "thirdshift-codebase-design", +]; + /// How long a test waits for a Run to reach a point, such as starting the /// agent or showing a prompt, before it gives the Run up as hung. A Run that /// exits without reaching the point fails its test at once, so only a hung @@ -788,6 +801,24 @@ test -f {root}/{COPY_REPLACED} } } + /// Assert every `claude` call found every Factory skill, by its + /// `thirdshift-` name, in its worktree's `.claude/skills/`. + pub fn assert_every_session_found_the_factory_skills(&self) { + let calls = self.claude_calls(); + assert!(!calls.is_empty(), "no session ran"); + for call in calls { + for skill in FACTORY_SKILLS { + let skill_md = call["skill_files"][format!("{skill}/SKILL.md")].as_str(); + assert!( + skill_md.is_some_and(|text| text.contains(&format!("name: {skill}\n"))), + "the session in {} with the prompt {} found no {skill}", + call["cwd"], + call["prompt"] + ); + } + } + } + /// The prompt of the first `claude` call. pub fn first_prompt(&self) -> String { self.claude_calls()[0]["prompt"] From ed9a63e1befabf74b459e9ec99faa891238fe365 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 5 Oct 2026 00:12:24 -0400 Subject: [PATCH 02/14] Review clean-ups: clearer names for writing out and removing the Factory skills (#391) Co-Authored-By: Claude Opus 5.5 --- src/command.rs | 2 +- src/skills.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/command.rs b/src/command.rs index 9f1380682..2179a678a 100644 --- a/src/command.rs +++ b/src/command.rs @@ -123,7 +123,7 @@ impl Started { /// recorded where it was decided, and it sends no notification. Returns /// its exit code. pub fn finish(self, ending: Ending) -> ExitCode { - skills::remove(); + skills::remove_written(); let account = match run_ending::read(&ending) { Ok(account) => account, Err(skip) => return run_ending::show_skip(skip), diff --git a/src/skills.rs b/src/skills.rs index a9acccd89..3043887ec 100644 --- a/src/skills.rs +++ b/src/skills.rs @@ -3,7 +3,7 @@ //! Claude Code finds project skills (ADR-0012). use std::fs::{self, OpenOptions}; -use std::io::Write; +use std::io::{ErrorKind, Write}; use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; @@ -33,7 +33,7 @@ static WRITTEN: Mutex> = Mutex::new(None); /// `.git/info/exclude` keeps them out of git. The links are left in place: /// they go with the worktree. pub fn link_into(worktree: &Path) -> Result<()> { - let written = written()?; + let written = written_out()?; exclude(&Git::new(worktree))?; let project_skills = worktree.join(PROJECT_SKILLS); fs::create_dir_all(&project_skills) @@ -53,7 +53,7 @@ pub fn link_into(worktree: &Path) -> Result<()> { /// Remove the Factory skills the Command wrote out, if it did, once its /// sessions are over. -pub fn remove() { +pub fn remove_written() { drop( WRITTEN .lock() @@ -64,7 +64,7 @@ pub fn remove() { /// Where the Command wrote the Factory skills out, writing them now if it /// has not yet. -fn written() -> Result { +fn written_out() -> Result { let mut written = WRITTEN.lock().unwrap_or_else(PoisonError::into_inner); if let Some(dir) = &*written { return Ok(dir.path().to_owned()); @@ -90,7 +90,7 @@ fn exclude(git: &Git) -> Result<()> { let path = info.join("exclude"); let existing = match fs::read_to_string(&path) { Ok(existing) => existing, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) if error.kind() == ErrorKind::NotFound => String::new(), Err(error) => { return Err(error).with_context(|| format!("could not read {}", path.display())); } From d62a5ac2ed6535969910c42b689ddfd9a91a3cbd Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 5 Oct 2026 00:19:59 -0400 Subject: [PATCH 03/14] Choose the Harness, Model and Effort on the command line and in the User config (#392) harness , model and effort on a Run, an Architect run (not with --plan-only) and a Pickup run, falling back to the User config's [harness] section, then to claude with its own Model and Effort. The choice reaches every session and child Run, is checked before any work (CLI on PATH, a test call for a named Claude Model, codex not supported yet), and is recorded in the Command log, the Activity log, the PR body and the Run notification. Co-Authored-By: Claude Opus 5.5 --- README.md | 31 ++- src/architect.rs | 65 +++--- src/args.rs | 184 ++++++++++++++-- src/asks.rs | 131 ++++++++++- src/base_fix.rs | 34 ++- src/child_run.rs | 123 +++++++++-- src/command.rs | 9 + src/config.rs | 121 ++++++++++- src/delivery.rs | 63 +++++- src/harness.rs | 457 +++++++++++++++++++++++++++++++++++++++ src/logs.rs | 20 +- src/logs/activity.rs | 9 +- src/main.rs | 47 +++- src/notification.rs | 36 ++- src/pickup.rs | 14 +- src/prompts_page.rs | 4 +- src/run.rs | 25 ++- src/session.rs | 50 ++--- src/spec_run.rs | 13 +- tests/activity_log.rs | 17 +- tests/architect_run.rs | 50 +++++ tests/base_fix.rs | 36 +++ tests/command_surface.rs | 8 +- tests/fresh_run.rs | 11 +- tests/harness_choice.rs | 374 ++++++++++++++++++++++++++++++++ tests/pickup_run.rs | 71 ++++++ tests/spec_run.rs | 49 ++++- 27 files changed, 1876 insertions(+), 176 deletions(-) create mode 100644 src/harness.rs create mode 100644 tests/harness_choice.rs diff --git a/README.md b/README.md index d7fc5da02..490c7283c 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,17 @@ parallel = 2 # how many Tickets a Spec run runs at once, instead of 3 [pickup] limit = 5 # how many open issues labelled in-progress stop a Pickup run taking another, instead of 3 + +[harness] +default = "claude" # the Harness every session runs on; claude unless set + +[harness.claude] +model = "opus" # the Model Claude Code's sessions run on; blank for its own default +effort = "high" # how hard that Model reasons; blank for Claude Code's own default + +[harness.codex] +model = "" +effort = "" ``` `thirdshift setup` writes this file for you, listing every setting at its default so the file itself shows what can be changed: @@ -259,8 +270,24 @@ With `email.always = true`, every Run sends a [Run notification](#run-notificati `logs.dir` sets the root of the [logs](#logs), with each repository's in its `//` folder, created if missing. It must be an absolute path, `~` or a path starting with `~/`, where `~` stands for `$HOME`. A relative path stops the Run before any work, since the directory a Run is launched from is no base for a setting that holds for every Run. +`[harness]` chooses the **Harness**, the agent CLI every session of a command runs on, and for each Harness the **Model** its sessions run on and their **Effort**, how hard that Model reasons. See [Harness, Model and Effort](#harness-model-and-effort). + A Run reads the file before any work. One that isn't valid TOML, or that has a key or section thirdshift doesn't know, such as `alway` for `always`, or a value of the wrong type, such as anything but `true` or `false` for `always`, or `0` for `spec.parallel` or `pickup.limit`, stops the Run with an error naming the file and the offending key, so a typo can't silently leave a setting off. `email-test` and `setup` read it the same way. `update`, `version` and `help` never read it, so a broken User config can't block them, and they never offer Setup; nor does `email-test`. +### Harness, Model and Effort + +Every session of a command, its implement session, each Repair and Resume, a Spec review, an Architecture review, each Ticket's Run and a Base fix, runs on one **Harness**, with one **Model** and one **Effort**. `harness `, `model ` and `effort ` (or `--harness`, `--model` and `--effort`) choose them for one command, before or after the Issue URL, and on `architect` (but not with `--plan-only`) and `pickup` too: + +```sh +thirdshift https://github.com///issues/ model claude-opus-5-5 effort high +``` + +For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Only `claude` works for now: `harness codex` fails with `not supported yet`. + +Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and a named Model gets a minimal test call, `claude -p --model ` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. + +The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. + ### Email thirdshift sends email itself, with one HTTPS request to [Resend](https://resend.com)'s API, so it needs no mail server on the machine and works where SMTP ports are blocked ([ADR 0005](docs/adr/0005-run-notifications-through-resend.md)). It needs a Resend account and an API key: @@ -501,7 +528,7 @@ thirdshift architect --email you@example.com # email how the Architect run The focus is optional free text, given as one argument, anywhere among the flags; it goes into the Session prompt. `architect` is a command only as the first argument. -`merge`, `no-merge`, `base-fix`, `no-base-fix` and `parallel ` (or `--merge`, `--no-merge`, `--base-fix`, `--no-base-fix` and `--parallel `) are for the run the plan is dispatched as, and mean what they do for `thirdshift `, so none of them is ever read as the focus. `--plan-only` stops the Architect run once the plan is marked ready, and dispatches nothing. `email`, optionally followed by an address, and `no-email` (or `--email` and `--no-email`) are for the Architect run's own [Run notification](#one-run-notification), with or without `--plan-only`. The word after `email` is the address only if it contains `@`, so a focus without one is never taken for it. +`merge`, `no-merge`, `base-fix`, `no-base-fix`, `parallel `, `harness `, `model ` and `effort ` (or the same with dashes) are for the run the plan is dispatched as, and mean what they do for `thirdshift `, so none of them is ever read as the focus. The [Harness, Model and Effort](#harness-model-and-effort) are the Architecture review's too. `--plan-only` stops the Architect run once the plan is marked ready, and dispatches nothing. `email`, optionally followed by an address, and `no-email` (or `--email` and `--no-email`) are for the Architect run's own [Run notification](#one-run-notification), with or without `--plan-only`. The word after `email` is the address only if it contains `@`, so a focus without one is never taken for it. `base ` (or `--base `) names the Architect run's **Base branch**, so the branch your clone has checked out stops mattering: start it from a clone on another branch, on a detached HEAD, or with uncommitted changes. The Architecture review's worktree starts at the head of `` on `origin`, and the run the plan is dispatched as takes `` as its Base branch too: a Run's **Issue branch**, or a Spec run's **Spec branch**, is branched off it, and the pull request or **Spec PR** targets it. It goes before or after the focus and the other flags, `--plan-only` included, and the word after it is always the branch, never the focus. Without `base`, the Base branch is the branch checked out. `base` is a flag of `architect` and [`pickup`](#pickup-runs) only: `thirdshift ` doesn't take it. It is what lets an Architect run be started [on a schedule](#on-a-schedule). @@ -671,7 +698,7 @@ A Pickup run: 5. Lists the repository's open issues labelled `ready-for-agent`, lowest number first, and takes the first that is a Ready issue, saying so on stderr: `taking Ready issue # "", as thirdshift <Issue URL> would`. Each one it passes over on the way gets [a line saying why](#why-an-issue-was-passed-over). One issue a pass: the rest wait for the next. 6. Dispatches it exactly as `thirdshift <Issue URL>` would from the same clone on the Pickup run's Base branch: a [Spec run](#spec-runs) when the issue has sub-issues, a Run otherwise. That run makes the Claim, so the issue's `ready-for-agent` is swapped for `in-progress` and no later pass takes it again. -`merge`, `no-merge`, `base-fix`, `no-base-fix` and `parallel <n>`, with or without dashes, are for the dispatched run, and mean what they do for `thirdshift <Issue URL>`, as do `email`, optionally followed by an address, and `no-email` for the Pickup run's [Run notification](#one-run-notification-for-the-issue-taken). The [User config](#user-config) sets what they leave unsaid: `merge.always`, `base.fix`, `spec.parallel`, `email.always`, `launch.pull` and `logs.dir`. A User config a Run would refuse stops the pass the same way, before any check. `parallel <n>` applies when the Ready issue is a Spec and is ignored, with no error, when it isn't, unlike on an Issue URL: the command can't know which it will take. `base <branch>` (or `--base <branch>`) names the Base branch as it does for [`architect`](#architect-runs), and the dispatched run takes it: its Issue branch or Spec branch is branched off `<branch>`, and its pull request targets it. With `base <branch>`, a Pickup run started [on a schedule](#a-pickup-run-on-a-schedule) runs from the clone you work in, whatever branch you left it on. +`merge`, `no-merge`, `base-fix`, `no-base-fix`, `parallel <n>`, `harness <name>`, `model <name>` and `effort <level>`, with or without dashes, are for the dispatched run, and mean what they do for `thirdshift <Issue URL>`, as do `email`, optionally followed by an address, and `no-email` for the Pickup run's [Run notification](#one-run-notification-for-the-issue-taken). The [User config](#user-config) sets what they leave unsaid: `merge.always`, `base.fix`, `spec.parallel`, `email.always`, `launch.pull` and `logs.dir`. A User config a Run would refuse stops the pass the same way, before any check. `parallel <n>` applies when the Ready issue is a Spec and is ignored, with no error, when it isn't, unlike on an Issue URL: the command can't know which it will take. `base <branch>` (or `--base <branch>`) names the Base branch as it does for [`architect`](#architect-runs), and the dispatched run takes it: its Issue branch or Spec branch is branched off `<branch>`, and its pull request targets it. With `base <branch>`, a Pickup run started [on a schedule](#a-pickup-run-on-a-schedule) runs from the clone you work in, whatever branch you left it on. `pickup` is a command only as the first argument, and takes nothing but those flags, each at most once: a focus, `--plan-only`, an Issue URL, a repeated or contradictory flag, or any other argument is an argument error (exit `2`). diff --git a/src/architect.rs b/src/architect.rs index b679b3db1..9b3bf8537 100644 --- a/src/architect.rs +++ b/src/architect.rs @@ -17,13 +17,13 @@ //! work a human shaped goes first. use std::fmt; -use std::path::PathBuf; use anyhow::{Context, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use crate::failed_run::FailedRun; use crate::github::ListedIssue; +use crate::harness::Choice; use crate::interrupt; use crate::issue::{IssueUrl, Repo}; use crate::labels::{Edit, Label, Labels, NEEDS_TRIAGE, READY_FOR_AGENT}; @@ -211,12 +211,16 @@ impl fmt::Display for Reviewed { /// repository has a Ready issue, by the very search a Pickup run makes, with /// its lines on the issues passed over, but with no Sweep and no Claim limit, /// which are the Pickup run's. A skip is recorded in the repository's -/// Activity log. Past all four, it records that it started work, which keeps -/// its Command log with its repository's logs, where its Session logs go too. +/// Activity log. Past all four, it checks `harness`, the Harness, Model and +/// Effort its review, and the run it dispatches, run their sessions on, +/// failing before any work if they can't run. Then it records that it +/// started work, which keeps its Command log with its repository's logs, +/// where its Session logs go too. pub fn run( focus: Option<&str>, base: Option<&str>, launch_pull: bool, + harness: &Choice, ) -> Result<Outcome, FailedRun> { let started = Utc::now(); let Launch { @@ -237,7 +241,8 @@ pub fn run( if let Decision::Skip(skipped) = gates(&mut on_github)? { return Ok(skip(&repo, skipped)); } - logs::started(Work::ArchitectRun(&repo)); + harness.check()?; + logs::started(Work::ArchitectRun(&repo), harness); if launch_pull { directory.pull(&base); } @@ -247,15 +252,18 @@ pub fn run( } let worktree = ReviewWorktree::create(directory.git(), &repo.name, &base)?; let logs = Logs::of_architect_run(&repo); - let (reviewed, log) = review( - &mut on_github, - worktree, - &base, - focus, - directory.origin(), - started, - &logs, - ); + let (reviewed, log) = Sessions::within(&logs, worktree.path(), harness, |sessions| { + review( + &mut on_github, + sessions, + &base, + focus, + directory.origin(), + started, + ) + }); + // Removed once the review is concluded. + drop(worktree); reviewed.map(Outcome::Reviewed).map_err(|error| FailedRun { log, ..FailedRun::from(error) @@ -300,30 +308,25 @@ fn skip(repo: &Repo, skipped: Skipped) -> Outcome { Outcome::Skipped(skipped) } -/// The Architecture review session in `worktree`, of the Base branch `base`, -/// then [`conclude`] on its final message, through `outside`. The worktree is -/// removed once the review is concluded. Returns how it ended with the most -/// recent session's log, if a session created it. +/// The Architecture review session, of the Base branch `base`, run through +/// `sessions`, then [`conclude`] on its final message, through `outside`. fn review( outside: &mut impl Outside, - worktree: ReviewWorktree, + sessions: &Sessions, base: &str, focus: Option<&str>, origin: &str, started: DateTime<Utc>, - logs: &Logs, -) -> (Result<Reviewed>, Option<PathBuf>) { - Sessions::within(logs, worktree.path(), |sessions| { - match focus { - Some(focus) => progress::step(format_args!( - "starting the Architecture review of {base}, focused on: {focus}" - )), - None => progress::step(format_args!("starting the Architecture review of {base}")), - } - let prompt = prompt::architecture_review(base, focus); - let final_message = sessions.run_to_final_message(REVIEW, &prompt)?; - conclude(outside, final_message.as_deref(), origin, started, base) - }) +) -> Result<Reviewed> { + match focus { + Some(focus) => progress::step(format_args!( + "starting the Architecture review of {base}, focused on: {focus}" + )), + None => progress::step(format_args!("starting the Architecture review of {base}")), + } + let prompt = prompt::architecture_review(base, focus); + let final_message = sessions.run_to_final_message(REVIEW, &prompt)?; + conclude(outside, final_message.as_deref(), origin, started, base) } /// What an Architecture review reported in the last line of its final diff --git a/src/args.rs b/src/args.rs index 031d33aa5..4b8a94ca6 100644 --- a/src/args.rs +++ b/src/args.rs @@ -10,6 +10,7 @@ use anyhow::{Result, bail}; use crate::asks::Flags; use crate::base_fix::BaseFixAsk; use crate::child_run::{self, Given}; +use crate::harness::{self, Harness}; use crate::issue::IssueUrl; use crate::notification::NotificationAsk; use crate::run::Goal; @@ -134,8 +135,8 @@ pub fn parse(args: &[String]) -> Result<Command> { /// Parse the arguments after `architect`: at most one focus, and its flags, /// each at most once, in any order. `base` must be followed by the Base /// branch. [`PLAN_ONLY`] dispatches nothing, so the -/// flags for the dispatched run, `merge`, `no-merge`, `parallel`, `base-fix` -/// and `no-base-fix`, can't go with it. `email` and `no-email` are for the +/// flags for the dispatched run, `merge`, `no-merge`, `parallel`, `base-fix`, +/// `no-base-fix`, `harness`, `model` and `effort`, can't go with it. `email` and `no-email` are for the /// Architect run's own Run notification, and `base` is for the Architecture /// review too, so they can. None of a Run's flags, nor `base`, /// is ever the focus, and any other argument that starts with a dash is @@ -157,7 +158,7 @@ fn parse_architect(args: &[String]) -> Result<ArchitectArgs> { } plan_only = true; } - "base" | "--base" => ask_base(&mut base, arg, args.next())?, + "base" | "--base" => ask_word(&mut base, arg, args.next(), "a branch")?, _ if arg.starts_with('-') => bail!("unexpected argument after architect: {arg}"), _ if focus.is_some() => bail!("unexpected argument after the focus: {arg}"), _ if arg.trim().is_empty() => bail!("the focus is empty"), @@ -166,8 +167,8 @@ fn parse_architect(args: &[String]) -> Result<ArchitectArgs> { } if plan_only && flags.any_for_dispatched_run() { bail!( - "merge, no-merge, parallel, base-fix and no-base-fix can't be used with \ - {PLAN_ONLY}: it dispatches no run for them to apply to" + "merge, no-merge, parallel, base-fix, no-base-fix, harness, model and effort \ + can't be used with {PLAN_ONLY}: it dispatches no run for them to apply to" ); } Ok(ArchitectArgs { @@ -189,7 +190,7 @@ fn parse_pickup(args: &[String]) -> Result<PickupArgs> { continue; } match arg.as_str() { - "base" | "--base" => ask_base(&mut base, arg, args.next())?, + "base" | "--base" => ask_word(&mut base, arg, args.next(), "a branch")?, _ => bail!("unexpected argument after pickup: {arg}"), } } @@ -198,8 +199,9 @@ fn parse_pickup(args: &[String]) -> Result<PickupArgs> { /// Record in `flags` what `arg` asks for, if it is one of the flags a Run, /// an Architect run and a Pickup run all take, with or without its dashes, -/// taking from `rest` the address after `email`, if one is there, and the -/// number after `parallel`. False, taking nothing, if `arg` is none of them. +/// taking from `rest` the address after `email`, if one is there, the +/// number after `parallel`, and the name or level after `harness`, `model` +/// and `effort`. False, taking nothing, if `arg` is none of them. fn take_flag<'a>( flags: &mut Flags, arg: &str, @@ -229,6 +231,15 @@ fn take_flag<'a>( "no-base-fix" | "--no-base-fix" => { ask_once(&mut flags.base_fix, BaseFixAsk::Forbid, arg, BASE_FIX_FLAGS)? } + "harness" | "--harness" => ask_harness(&mut flags.harness.harness, arg, rest.next())?, + "model" | "--model" => { + let model = &mut flags.harness.model_and_effort.model; + ask_word(model, arg, rest.next(), "a model")? + } + "effort" | "--effort" => { + let effort = &mut flags.harness.model_and_effort.effort; + ask_word(effort, arg, rest.next(), "an effort level")? + } _ => return Ok(false), } Ok(true) @@ -266,22 +277,42 @@ fn ask_parallel(parallel: &mut Option<NonZeroUsize>, arg: &str, n: Option<&Strin Ok(()) } -/// Record in `base` the `branch` that follows the flag `arg`, given once. No -/// branch's name starts with a dash, so a flag there is not taken for one. -fn ask_base(base: &mut Option<String>, arg: &str, branch: Option<&String>) -> Result<()> { - if base.is_some() { +/// Record in `given` the `word` that follows the flag `arg`, given once: a +/// branch, a model or an effort level, as `what` says. None of those starts +/// with a dash, so a flag there is not taken for one. +fn ask_word( + given: &mut Option<String>, + arg: &str, + word: Option<&String>, + what: &str, +) -> Result<()> { + if given.is_some() { bail!("repeated argument: {arg}"); } - match branch { - Some(branch) if branch.starts_with('-') => { - bail!("{arg} must be followed by a branch, not {branch}") + match word { + Some(word) if word.starts_with('-') => { + bail!("{arg} must be followed by {what}, not {word}") } - Some(branch) if !branch.trim().is_empty() => *base = Some(branch.clone()), - _ => bail!("{arg} must be followed by a branch"), + Some(word) if !word.trim().is_empty() => *given = Some(word.clone()), + _ => bail!("{arg} must be followed by {what}"), } Ok(()) } +/// Record in `harness` the Harness `name` that follows the flag `arg`, given +/// once. +fn ask_harness(harness: &mut Option<Harness>, arg: &str, name: Option<&String>) -> Result<()> { + if harness.is_some() { + bail!("repeated argument: {arg}"); + } + let Some(named) = name.and_then(|name| Harness::named(name)) else { + let given = name.map(|name| format!(", not {name}")).unwrap_or_default(); + bail!("{arg} must be followed by {}{given}", harness::NAMES); + }; + *harness = Some(named); + Ok(()) +} + /// Is `arg`, after `email`, the address to send to? Only if it looks like /// one, so the Issue URL is never taken for it, nor is an Architect run's /// focus unless it has an `@`. @@ -292,6 +323,7 @@ fn is_address(arg: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::harness::ModelAndEffort; const URL: &str = "https://github.com/acme/widgets/issues/7"; @@ -521,8 +553,8 @@ mod tests { ] { assert_eq!( rejection(&args), - "merge, no-merge, parallel, base-fix and no-base-fix can't be used with \ - --plan-only: it dispatches no run for them to apply to", + "merge, no-merge, parallel, base-fix, no-base-fix, harness, model and effort \ + can't be used with --plan-only: it dispatches no run for them to apply to", "{args:?}" ); } @@ -681,6 +713,13 @@ mod tests { email: Some(NotificationAsk::Send(Some("me@example.com".to_string()))), parallel: NonZeroUsize::new(2), base_fix: Some(BaseFixAsk::Allow), + harness: harness::Asked { + harness: Some(Harness::Codex), + model_and_effort: ModelAndEffort { + model: Some("gpt-6.1-sol".to_string()), + effort: Some("max".to_string()), + }, + }, }, }; for args in [ @@ -689,19 +728,31 @@ mod tests { "merge", "parallel", "2", + "harness", + "codex", "base-fix", "email", "me@example.com", + "model", + "gpt-6.1-sol", "base", "develop", + "effort", + "max", ], vec![ "pickup", + "--effort", + "max", "--base", "develop", + "--model", + "gpt-6.1-sol", "--email", "me@example.com", "--base-fix", + "--harness", + "codex", "--parallel", "2", "--merge", @@ -716,6 +767,7 @@ mod tests { email: Some(NotificationAsk::Skip), parallel: None, base_fix: Some(BaseFixAsk::Forbid), + harness: harness::Asked::default(), }, }; for args in [ @@ -889,4 +941,98 @@ mod tests { } } } + + #[test] + fn harness_model_and_effort_take_the_word_after_them_on_a_run_an_architect_run_and_a_pickup_run() + { + let asked = |harness, model: Option<&str>, effort: Option<&str>| harness::Asked { + harness, + model_and_effort: ModelAndEffort { + model: model.map(str::to_string), + effort: effort.map(str::to_string), + }, + }; + for (args, expected) in [ + ( + vec!["harness", "claude", URL], + asked(Some(Harness::Claude), None, None), + ), + ( + vec![URL, "--model", "claude-opus-5-5", "merge", "effort", "high"], + asked(None, Some("claude-opus-5-5"), Some("high")), + ), + ( + vec!["effort", "max", URL, "--harness", "codex"], + asked(Some(Harness::Codex), None, Some("max")), + ), + ] { + assert_eq!(run_args(&args).flags.harness, expected, "{args:?}"); + } + let architect = architect_args(&[ + "architect", + "the Spec run", + "model", + "opus", + "harness", + "claude", + ]); + assert_eq!( + architect.flags.harness, + asked(Some(Harness::Claude), Some("opus"), None) + ); + assert_eq!(architect.focus.as_deref(), Some("the Spec run")); + assert_eq!( + pickup_args(&["pickup", "--effort", "low"]).flags.harness, + asked(None, None, Some("low")) + ); + } + + #[test] + fn harness_model_and_effort_are_rejected_twice_without_a_value_or_with_plan_only() { + let plan_only = "merge, no-merge, parallel, base-fix, no-base-fix, harness, model and \ + effort can't be used with --plan-only: it dispatches no run for them \ + to apply to"; + for (args, error) in [ + ( + vec![URL, "harness"], + "harness must be followed by claude or codex", + ), + ( + vec![URL, "harness", "gemini"], + "harness must be followed by claude or codex, not gemini", + ), + ( + vec!["--harness", "claude", URL, "harness", "codex"], + "repeated argument: harness", + ), + (vec![URL, "model"], "model must be followed by a model"), + ( + vec![URL, "--model", "--merge"], + "--model must be followed by a model, not --merge", + ), + ( + vec!["model", "opus", URL, "model", "sonnet"], + "repeated argument: model", + ), + ( + vec![URL, "effort"], + "effort must be followed by an effort level", + ), + ( + vec!["pickup", "effort", "max", "--effort", "low"], + "repeated argument: --effort", + ), + ( + vec!["architect", "--plan-only", "harness", "claude"], + plan_only, + ), + (vec!["architect", "model", "opus", "--plan-only"], plan_only), + ( + vec!["architect", "--plan-only", "--effort", "max"], + plan_only, + ), + ] { + assert_eq!(rejection(&args), error, "{args:?}"); + } + } } diff --git a/src/asks.rs b/src/asks.rs index c98bfeb3a..74f5f68af 100644 --- a/src/asks.rs +++ b/src/asks.rs @@ -9,6 +9,7 @@ use crate::args; use crate::base_fix::BaseFixAsk; use crate::child_run::Given; use crate::config::UserConfig; +use crate::harness::{self, Choice}; use crate::issue::IssueUrl; use crate::notification::NotificationAsk; use crate::run::Goal; @@ -25,6 +26,8 @@ pub struct Flags { pub parallel: Option<NonZeroUsize>, /// What `base-fix` or `no-base-fix` asked for. pub base_fix: Option<BaseFixAsk>, + /// What `harness <name>`, `model <name>` and `effort <level>` asked for. + pub harness: harness::Asked, } /// Everything a Run, or a Spec run, is asked, resolved. @@ -44,6 +47,9 @@ pub struct Asks { /// Whether it first brings the Launch directory's checkout of the Base /// branch up to date with origin. pub launch_pull: bool, + /// The Harness, Model and Effort its sessions, and its child Runs', run + /// on. + pub harness: Choice, } impl Asks { @@ -72,13 +78,15 @@ impl Asks { parallel_asked: flags.parallel.is_some(), base_fix, launch_pull: config.launch_pull, + harness: flags.harness(config), } } /// What a child Run, a Ticket's Run in a Spec run or a Base fix, is /// asked, from what it was `given`: it is always a Merge run, sends no /// Run notification, leaves the Launch directory alone, and asks about a - /// Base fix what it was given. Neither its flags nor `config` decide + /// Base fix what it was given, running its sessions on the Harness, + /// Model and Effort it was given. Neither its flags nor `config` decide /// any of that: those are for what started it. As it runs no Tickets, /// `config` says only how many it would run at once. pub fn of_child_run(given: &Given, config: &UserConfig) -> Asks { @@ -89,6 +97,7 @@ impl Asks { parallel_asked: false, base_fix: given.base_fix.clone(), launch_pull: false, + harness: given.harness.clone(), } } @@ -157,6 +166,14 @@ impl Flags { NotificationAsk::Skip }) } + + /// The Harness, Model and Effort a command with these flags runs its + /// sessions on: each as its flag asked, else as `config` sets it, else + /// the default. It is a Run's, and an Architect run's or a Pickup run's + /// for its own sessions as for the run it dispatches. + pub fn harness(&self, config: &UserConfig) -> Choice { + Choice::of(&self.harness, &config.harness) + } } /// The command that starts the Run on `issue` again as `flags` asked for it, @@ -177,6 +194,16 @@ fn retry_with_base_fix(issue: &IssueUrl, flags: &Flags) -> String { if let Some(parallel) = flags.parallel { command += &format!(" parallel {parallel}"); } + let asked = &flags.harness; + if let Some(harness) = asked.harness { + command += &format!(" harness {}", harness.name()); + } + if let Some(model) = &asked.model_and_effort.model { + command += &format!(" model {model}"); + } + if let Some(effort) = &asked.model_and_effort.effort { + command += &format!(" effort {effort}"); + } command + " " + args::BASE_FIX } @@ -187,6 +214,7 @@ mod tests { use super::*; use crate::child_run::Kind; use crate::config::EmailSettings; + use crate::harness::{ChosenBy, Harness, ModelAndEffort}; const URL: &str = "https://github.com/acme/widgets/issues/7"; @@ -214,6 +242,7 @@ mod tests { email: EmailSettings::default(), spec_parallel: n(3), pickup_limit: n(3), + harness: harness::Settings::default(), } } @@ -229,10 +258,53 @@ mod tests { from: None, }, spec_parallel: n(5), + harness: harness::Settings { + default: Some(Harness::Codex), + claude: model_and_effort("opus", "high"), + codex: model_and_effort("gpt-6.1-sol", "max"), + }, ..no_settings() } } + fn model_and_effort(model: &str, effort: &str) -> ModelAndEffort { + ModelAndEffort { + model: Some(model.to_string()), + effort: Some(effort.to_string()), + } + } + + /// The sessions [`every_setting`] chooses with no flag. + fn configured_codex() -> Choice { + Choice { + harness: Harness::Codex, + model: Some("gpt-6.1-sol".to_string()), + effort: Some("max".to_string()), + chosen_by: ChosenBy::UserConfig, + } + } + + /// Claude on the Model `sonnet`, as the command chose, with `effort`. + fn sonnet(effort: Option<&str>) -> Choice { + Choice { + harness: Harness::Claude, + model: Some("sonnet".to_string()), + effort: effort.map(str::to_string), + chosen_by: ChosenBy::Command, + } + } + + /// The flags that choose Claude, on the Model `sonnet`. + fn claude_on_sonnet() -> harness::Asked { + harness::Asked { + harness: Some(Harness::Claude), + model_and_effort: ModelAndEffort { + model: Some("sonnet".to_string()), + effort: None, + }, + } + } + /// The command that starts the Run on [`URL`] again with a Base fix /// allowed, `flags` being its other flags as the command would spell /// them. @@ -255,6 +327,7 @@ mod tests { parallel_asked: false, base_fix: undecided(""), launch_pull: false, + harness: Choice::default(), } ); } @@ -272,6 +345,7 @@ mod tests { parallel_asked: false, base_fix: BaseFixAsk::Allow, launch_pull: true, + harness: configured_codex(), } ); } @@ -283,6 +357,7 @@ mod tests { email: Some(NotificationAsk::Skip), parallel: Some(n(2)), base_fix: Some(BaseFixAsk::Forbid), + harness: claude_on_sonnet(), }; let asks = Asks::of_run(&issue(), &against, &every_setting()); assert_eq!( @@ -295,6 +370,7 @@ mod tests { base_fix: BaseFixAsk::Forbid, // No flag sets it. launch_pull: true, + harness: sonnet(Some("high")), } ); @@ -303,6 +379,7 @@ mod tests { email: Some(to("flag@example.com")), parallel: Some(n(2)), base_fix: Some(BaseFixAsk::Allow), + harness: claude_on_sonnet(), }; let asks = Asks::of_run(&issue(), &for_it, &no_settings()); assert_eq!( @@ -314,16 +391,19 @@ mod tests { parallel_asked: true, base_fix: BaseFixAsk::Allow, launch_pull: false, + harness: sonnet(None), } ); } - /// What a child Run of `kind` is given, asked `base_fix` about a Base fix. + /// What a child Run of `kind` is given, asked `base_fix` about a Base + /// fix, its sessions on Claude's `sonnet`. fn given(kind: Kind, base_fix: BaseFixAsk) -> Given { Given { kind, stamp: "20261003T120000-0400".to_string(), base_fix, + harness: sonnet(None), } } @@ -356,6 +436,15 @@ mod tests { } } + #[test] + fn a_child_run_runs_its_sessions_on_what_it_was_given_whatever_the_user_config_says() { + for config in [no_settings(), every_setting()] { + let asks = Asks::of_child_run(&given(ticket(), BaseFixAsk::Forbid), &config); + + assert_eq!(asks.harness, sonnet(None)); + } + } + #[test] fn a_child_run_asks_about_a_base_fix_what_it_was_given_whatever_the_user_config_says() { let offer = BaseFixAsk::Undecided { @@ -378,6 +467,7 @@ mod tests { email: Some(to("flag@example.com")), parallel: Some(n(2)), base_fix: None, + harness: claude_on_sonnet(), } } @@ -391,8 +481,11 @@ mod tests { notification: NotificationAsk::Skip, tickets_at_once: n(2), parallel_asked: true, - base_fix: undecided(" merge --email flag@example.com parallel 2"), + base_fix: undecided( + " merge --email flag@example.com parallel 2 harness claude model sonnet" + ), launch_pull: false, + harness: sonnet(None), } ); @@ -407,6 +500,7 @@ mod tests { parallel_asked: false, base_fix: BaseFixAsk::Allow, launch_pull: true, + harness: configured_codex(), } ); } @@ -421,8 +515,11 @@ mod tests { notification: NotificationAsk::Skip, tickets_at_once: n(2), parallel_asked: true, - base_fix: undecided(" merge --email flag@example.com parallel 2"), + base_fix: undecided( + " merge --email flag@example.com parallel 2 harness claude model sonnet" + ), launch_pull: false, + harness: sonnet(None), } ); @@ -437,6 +534,7 @@ mod tests { parallel_asked: false, base_fix: BaseFixAsk::Allow, launch_pull: true, + harness: configured_codex(), } ); } @@ -462,6 +560,10 @@ mod tests { base_fix: Some(BaseFixAsk::Forbid), ..Flags::default() }, + Flags { + harness: claude_on_sonnet(), + ..Flags::default() + }, ] { assert!(flags.any_for_dispatched_run(), "{flags:?}"); } @@ -477,11 +579,14 @@ mod tests { // Nor is it in the command that retries the issue, which would fail // a Run on an issue that is not a Spec. let asks = Asks::of_ready_issue(&issue(), false, &dispatch_flags(), &no_settings()); - assert_eq!(asks.base_fix, undecided(" merge --email flag@example.com")); + assert_eq!( + asks.base_fix, + undecided(" merge --email flag@example.com harness claude model sonnet") + ); let asks = Asks::of_ready_issue(&issue(), true, &dispatch_flags(), &no_settings()); assert_eq!( asks.base_fix, - undecided(" merge --email flag@example.com parallel 2") + undecided(" merge --email flag@example.com parallel 2 harness claude model sonnet") ); } @@ -510,8 +615,22 @@ mod tests { email, parallel, base_fix: None, + harness: harness::Asked::default(), + }; + let on_codex = Flags { + harness: harness::Asked { + harness: Some(Harness::Codex), + model_and_effort: model_and_effort("gpt-6.1-sol", "max"), + }, + ..flags(Some(Goal::Merged), None, None) }; for (flags, retry) in [ + ( + on_codex, + format!( + "thirdshift {URL} merge harness codex model gpt-6.1-sol effort max base-fix" + ), + ), (Flags::default(), format!("thirdshift {URL} base-fix")), ( flags(Some(Goal::Merged), Some(NotificationAsk::Send(None)), None), diff --git a/src/base_fix.rs b/src/base_fix.rs index 8007672ee..1ab90a3f1 100644 --- a/src/base_fix.rs +++ b/src/base_fix.rs @@ -18,6 +18,7 @@ use crate::child_run::{self, Ended, Kind}; use crate::ci::{self, FailedChecks}; use crate::git::Git; use crate::github::{self, Check, ListedIssue}; +use crate::harness::Choice; use crate::issue::IssueUrl; use crate::labels::{Label, Labels, READY_FOR_AGENT}; use crate::poll; @@ -112,6 +113,9 @@ pub struct BaseFix { /// What the Run says after its cause, if Inherited failures failed it /// with no Base fix taken. advice: Vec<Advice>, + /// The Harness, Model and Effort a Base fix it starts runs its sessions + /// on: the Run's. + harness: Choice, } /// The Base fix a Run took as its one: one it started, or one it found @@ -134,9 +138,10 @@ impl Taken { impl BaseFix { /// The Base fix of a Run that is the child Run `child`, if it is one, - /// and that asked `ask` about a Base fix. A Base fix starts none of its - /// own, whatever it asked. - pub fn new(child: Option<&Kind>, ask: BaseFixAsk) -> Self { + /// that asked `ask` about a Base fix, and whose sessions run on + /// `harness`, as a Base fix it starts does. A Base fix starts none of + /// its own, whatever it asked. + pub fn new(child: Option<&Kind>, ask: BaseFixAsk, harness: Choice) -> Self { let (on_inherited_failures, retry) = match (child, ask) { (Some(Kind::BaseFix { .. }), _) => (OnInheritedFailures::IsBaseFix, None), (_, BaseFixAsk::Allow) => (OnInheritedFailures::StartBaseFix, None), @@ -148,6 +153,7 @@ impl BaseFix { retry, taken: None, advice: Vec::new(), + harness, } } @@ -201,7 +207,12 @@ impl BaseFix { base_commit: &str, failed: &FailedChecks, ) -> Result<()> { - let mut outside = LaunchAndGitHub { launch, issue }; + let harness = self.harness.clone(); + let mut outside = LaunchAndGitHub { + launch, + issue, + harness: &harness, + }; self.fix_through(&mut outside, issue, pr_url, base, base_commit, failed) } @@ -447,10 +458,12 @@ impl<W> Mark<W> { } /// The outside world of a Run on `issue` started from `launch`: its git -/// directory's locks and marks, GitHub, the child Run and the poll interval. +/// directory's locks and marks, GitHub, the child Run, its sessions on +/// `harness`, and the poll interval. struct LaunchAndGitHub<'a> { launch: &'a Git, issue: &'a IssueUrl, + harness: &'a Choice, } impl Outside for LaunchAndGitHub<'_> { @@ -508,7 +521,7 @@ impl Outside for LaunchAndGitHub<'_> { let kind = Kind::BaseFix { base: base.to_string(), }; - child_run::start(fix, kind, BaseFixAsk::Forbid)?.wait() + child_run::start(fix, kind, BaseFixAsk::Forbid, self.harness)?.wait() } fn pause(&mut self) -> Result<()> { @@ -856,7 +869,7 @@ mod tests { /// A Run's Base fix as asked `ask`, not itself a Base fix. fn asked(ask: BaseFixAsk) -> BaseFix { - BaseFix::new(None, ask) + BaseFix::new(None, ask, Choice::default()) } fn undecided() -> BaseFixAsk { @@ -1303,14 +1316,17 @@ mod tests { base: "main".to_string(), }; for ask in [BaseFixAsk::Allow, BaseFixAsk::Forbid, undecided()] { - let base_fix = BaseFix::new(Some(&child), ask); + let base_fix = BaseFix::new(Some(&child), ask, Choice::default()); assert!(!base_fix.sees_inherited_failures()); assert_eq!(base_fix.ask_of_tickets(), BaseFixAsk::Forbid); } let ticket = Kind::Ticket { spec_branch: "spec-3".to_string(), }; - assert!(BaseFix::new(Some(&ticket), BaseFixAsk::Allow).sees_inherited_failures()); + assert!( + BaseFix::new(Some(&ticket), BaseFixAsk::Allow, Choice::default()) + .sees_inherited_failures() + ); assert!(asked(BaseFixAsk::Forbid).sees_inherited_failures()); } } diff --git a/src/child_run.rs b/src/child_run.rs index a13a94797..109e621d0 100644 --- a/src/child_run.rs +++ b/src/child_run.rs @@ -13,6 +13,7 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use crate::base_fix::BaseFixAsk; +use crate::harness::{Choice, ChosenBy, Harness}; use crate::interrupt; use crate::issue::IssueUrl; use crate::logs; @@ -62,6 +63,9 @@ pub struct Given { pub stamp: String, /// What it is asked about a Base fix. pub base_fix: BaseFixAsk, + /// The Harness, Model and Effort its sessions run on: the command's + /// that started it. + pub harness: Choice, } /// The hidden argument that makes a child Run a [`Kind::Ticket`], followed by @@ -86,6 +90,18 @@ const ALLOW_BASE_FIX: &str = "--allow-base-fix"; /// child Run is asked [`BaseFixAsk::Forbid`]. const OFFER_BASE_FIX: &str = "--offer-base-fix"; +/// The hidden argument followed by the Harness the child Run's sessions run +/// on. Not `--harness`, which is the `harness` flag with its dashes. +const SESSIONS_HARNESS: &str = "--sessions-harness"; + +/// The hidden argument followed by the Model the child Run's sessions run +/// on. Without it, the Model is left to the Harness. +const SESSIONS_MODEL: &str = "--sessions-model"; + +/// The hidden argument followed by the Effort the child Run's sessions run +/// on. Without it, the Effort is left to the Harness. +const SESSIONS_EFFORT: &str = "--sessions-effort"; + impl Given { /// The hidden arguments that give a child Run this, as [`Reader`] reads /// them back. @@ -101,6 +117,13 @@ impl Given { BaseFixAsk::Forbid => {} BaseFixAsk::Undecided { retry } => args.extend([OFFER_BASE_FIX, retry]), } + args.extend([SESSIONS_HARNESS, self.harness.harness.name()]); + if let Some(model) = &self.harness.model { + args.extend([SESSIONS_MODEL, model]); + } + if let Some(effort) = &self.harness.effort { + args.extend([SESSIONS_EFFORT, effort]); + } args } } @@ -113,6 +136,9 @@ pub struct Reader { kind: Option<Kind>, stamp: Option<String>, base_fix: Option<BaseFixAsk>, + harness: Option<Harness>, + model: Option<String>, + effort: Option<String>, } impl Reader { @@ -157,26 +183,60 @@ impl Reader { } }); } + SESSIONS_HARNESS => { + not_yet_given(&self.harness, arg)?; + let name = value("Harness")?; + let harness = Harness::named(&name) + .with_context(|| format!("{arg} must be followed by a Harness, not {name}"))?; + self.harness = Some(harness); + } + SESSIONS_MODEL => { + not_yet_given(&self.model, arg)?; + self.model = Some(value("Model")?); + } + SESSIONS_EFFORT => { + not_yet_given(&self.effort, arg)?; + self.effort = Some(value("Effort")?); + } _ => return Ok(false), } Ok(true) } /// What the child Run was given, once every argument has been taken, if - /// it is one. A kind needs a stamp, and the other hidden arguments a - /// kind. + /// it is one. A kind needs a stamp and a Harness, and the other hidden + /// arguments a kind. pub fn finish(self) -> Result<Option<Given>> { let Some(kind) = self.kind else { - if self.stamp.is_some() || self.base_fix.is_some() { - bail!("only a child Run is given {STAMP}, {ALLOW_BASE_FIX} or {OFFER_BASE_FIX}"); + if self.stamp.is_some() + || self.base_fix.is_some() + || self.harness.is_some() + || self.model.is_some() + || self.effort.is_some() + { + bail!( + "only a child Run is given {STAMP}, {ALLOW_BASE_FIX}, {OFFER_BASE_FIX}, \ + {SESSIONS_HARNESS}, {SESSIONS_MODEL} or {SESSIONS_EFFORT}" + ); } return Ok(None); }; let stamp = self.stamp.with_context(|| format!("missing {STAMP}"))?; + let harness = self + .harness + .with_context(|| format!("missing {SESSIONS_HARNESS}"))?; Ok(Some(Given { kind, stamp, base_fix: self.base_fix.unwrap_or(BaseFixAsk::Forbid), + harness: Choice { + harness, + model: self.model, + effort: self.effort, + // Chosen by the command that started it, whose checks it + // passed. + chosen_by: ChosenBy::Command, + }, })) } } @@ -212,12 +272,18 @@ pub struct Handle { /// Start a Run of `kind` on `issue` in a child `thirdshift`, from the same /// Launch directory, given this command's start stamp for its Session logs, -/// and asked `base_fix` about a Base fix. -pub fn start(issue: &IssueUrl, kind: Kind, base_fix: BaseFixAsk) -> Result<Handle> { +/// asked `base_fix` about a Base fix, and running its sessions on `harness`. +pub fn start( + issue: &IssueUrl, + kind: Kind, + base_fix: BaseFixAsk, + harness: &Choice, +) -> Result<Handle> { let given = Given { kind, stamp: logs::stamp().to_string(), base_fix, + harness: harness.clone(), }; start_from(&own_executable()?, issue, &given) } @@ -371,6 +437,7 @@ mod tests { kind: ticket(), stamp: STAMPED.to_string(), base_fix: BaseFixAsk::Forbid, + harness: Choice::default(), }; let Err(error) = start_from(executable, &issue, &given) else { panic!("started from {}", executable.display()); @@ -408,12 +475,26 @@ mod tests { let offer = BaseFixAsk::Undecided { retry: format!("thirdshift {URL} --no-merge base-fix"), }; - for kind in [ticket(), base_fix()] { + let on_claude = Choice { + chosen_by: ChosenBy::Command, + ..Choice::default() + }; + let on_opus = Choice { + model: Some("opus".to_string()), + effort: Some("max".to_string()), + ..on_claude.clone() + }; + let harnesses = [on_claude, on_opus]; + for (kind, harness) in [ticket(), base_fix()] + .into_iter() + .zip(harnesses.iter().cycle()) + { for base_fix in [BaseFixAsk::Allow, BaseFixAsk::Forbid, offer.clone()] { let given = Given { kind: kind.clone(), stamp: STAMPED.to_string(), base_fix, + harness: harness.clone(), }; let written = given.to_args(); // Before, between and after the hidden arguments, each of @@ -434,6 +515,11 @@ mod tests { assert_eq!(read(&[URL, "base-fix"]).unwrap(), None); } + /// What a Run given a hidden argument only a child Run is given, without + /// a kind, is rejected with. + const ONLY_A_CHILD_RUN: &str = "only a child Run is given --stamp, --allow-base-fix, \ + --offer-base-fix, --sessions-harness, --sessions-model or --sessions-effort"; + #[test] fn hidden_arguments_that_are_repeated_have_no_value_or_lack_a_kind_or_stamp_are_rejected() { let stamp = ["--stamp", STAMPED]; @@ -484,19 +570,24 @@ mod tests { "missing command to offer", ), (vec![&ticket, &[URL]], "missing --stamp"), - (vec![&["--base-fix-into", "main", URL]], "missing --stamp"), - ( - vec![&stamp, &[URL]], - "only a child Run is given --stamp, --allow-base-fix or --offer-base-fix", - ), + (vec![&ticket, &stamp], "missing --sessions-harness"), ( - vec![&[URL, "--allow-base-fix"]], - "only a child Run is given --stamp, --allow-base-fix or --offer-base-fix", + vec![&ticket, &stamp, &["--sessions-harness", "gemini"]], + "--sessions-harness must be followed by a Harness, not gemini", ), ( - vec![&[URL, "--offer-base-fix", "x"]], - "only a child Run is given --stamp, --allow-base-fix or --offer-base-fix", + vec![ + &ticket, + &stamp, + &["--sessions-model", "opus", "--sessions-model", "opus"], + ], + "repeated argument: --sessions-model", ), + (vec![&[URL, "--sessions-effort", "max"]], ONLY_A_CHILD_RUN), + (vec![&["--base-fix-into", "main", URL]], "missing --stamp"), + (vec![&stamp, &[URL]], ONLY_A_CHILD_RUN), + (vec![&[URL, "--allow-base-fix"]], ONLY_A_CHILD_RUN), + (vec![&[URL, "--offer-base-fix", "x"]], ONLY_A_CHILD_RUN), ] { let mut args: Vec<&str> = args.concat(); if !args.contains(&URL) { diff --git a/src/command.rs b/src/command.rs index 95c97e5c4..6d199ff9e 100644 --- a/src/command.rs +++ b/src/command.rs @@ -15,6 +15,7 @@ use std::process::ExitCode; use crate::architect::{self, Reviewed}; use crate::config::{self, UserConfig}; use crate::failed_run::FailedRun; +use crate::harness::Choice; use crate::interrupt; use crate::issue::IssueUrl; use crate::logs::{self, Begin}; @@ -106,6 +107,14 @@ pub fn start( } impl Started { + /// Take note that the command's sessions run on `harness`, which its Run + /// notification names. + pub fn built_with(&mut self, harness: &Choice) { + if let Some(notification) = &mut self.notification { + notification.built_with(harness); + } + } + /// Take note that the Pickup run took the Ready issue `issue`, titled /// `title`, which its Run notification is then about. pub fn took(&mut self, issue: &IssueUrl, title: String) { diff --git a/src/config.rs b/src/config.rs index bd16f83e3..570effded 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,7 @@ use toml::{Table, Value}; use toml_edit::{DocumentMut, Item}; use crate::git::Git; +use crate::harness::{self, Harness, ModelAndEffort}; use crate::questions::{self, Answers}; use crate::resend_key::Credentials; use crate::{email, github, progress}; @@ -42,6 +43,9 @@ pub struct UserConfig { /// `pickup.limit`: the Claim limit, how many open issues carrying a /// Claim stop a Pickup run from taking another, by default 3. pub pickup_limit: NonZeroUsize, + /// The `[harness]` section: the Harness every Command runs its sessions + /// on, and a Model and Effort for each Harness. + pub harness: harness::Settings, } /// The `[email]` section: where email goes and who it comes from. The Resend @@ -82,6 +86,7 @@ impl UserConfig { email: EmailSettings::default(), spec_parallel: NonZeroUsize::new(3).unwrap(), pickup_limit: NonZeroUsize::new(3).unwrap(), + harness: harness::Settings::default(), } } @@ -98,7 +103,15 @@ impl UserConfig { for (section, value) in &table { let known = matches!( section.as_str(), - "merge" | "base" | "launch" | "logs" | "activity" | "email" | "spec" | "pickup" + "merge" + | "base" + | "launch" + | "logs" + | "activity" + | "email" + | "spec" + | "pickup" + | "harness" ); let settings = match value { Value::Table(settings) if known => settings, @@ -142,6 +155,26 @@ impl UserConfig { ("pickup", "limit", value) => { config.pickup_limit = whole_number_from_1(value, "pickup.limit", &file)? } + ("harness", "default", Value::String(name)) => match Harness::named(name) { + Some(harness) => config.harness.default = Some(harness), + None => bail!( + "harness.default must be {}, not {name:?}, in {file}", + harness::NAMES + ), + }, + ("harness", "default", _) => { + bail!("harness.default must be a quoted name in {file}") + } + ("harness", "claude" | "codex", Value::Table(settings)) => { + let set = match key.as_str() { + "claude" => &mut config.harness.claude, + _ => &mut config.harness.codex, + }; + *set = model_and_effort(settings, key, &file)?; + } + ("harness", "claude" | "codex", _) => { + bail!("harness.{key} must be the section [harness.{key}] in {file}") + } _ => bail!("unknown key {section}.{key} in {file}"), } } @@ -632,6 +665,27 @@ fn whole_number_from_1(value: &Value, key: &str, file: &str) -> Result<NonZeroUs .with_context(|| format!("{key} must be a whole number from 1 up in {file}")) } +/// `settings`, the section `[harness.<harness>]` of `file`, as the Model and +/// Effort set for that Harness, each blank to leave it to the Harness. Any +/// other key, or a value that isn't a string, is an error naming it. +fn model_and_effort(settings: &Table, harness: &str, file: &str) -> Result<ModelAndEffort> { + let mut set = ModelAndEffort::default(); + for (key, value) in settings { + let setting = match key.as_str() { + "model" => &mut set.model, + "effort" => &mut set.effort, + _ => bail!("unknown key harness.{harness}.{key} in {file}"), + }; + let Value::String(value) = value else { + bail!( + "harness.{harness}.{key} must be a quoted string, blank for {harness}'s own default, in {file}" + ); + }; + *setting = Some(value.trim().to_string()).filter(|value| !value.is_empty()); + } + Ok(set) +} + /// `$HOME`, and the User config's path under it. fn home_and_path() -> Result<(PathBuf, PathBuf)> { let home = home()?; @@ -701,6 +755,71 @@ mod tests { ); } + #[test] + fn the_harness_section_sets_the_default_harness_and_a_model_and_effort_for_each() { + let config = parse( + "[harness]\ndefault = \"codex\"\n\n\ + [harness.claude]\nmodel = \"opus\"\neffort = \"\"\n\n\ + [harness.codex]\nmodel = \"gpt-6.1-sol\"\neffort = \"max\"\n", + ) + .unwrap(); + + assert_eq!( + config.harness, + harness::Settings { + default: Some(Harness::Codex), + claude: ModelAndEffort { + model: Some("opus".to_string()), + effort: None, + }, + codex: ModelAndEffort { + model: Some("gpt-6.1-sol".to_string()), + effort: Some("max".to_string()), + }, + } + ); + assert_eq!(parse("").unwrap().harness, harness::Settings::default()); + } + + #[test] + fn a_harness_setting_thirdshift_cant_use_is_an_error_naming_it() { + for (text, error) in [ + ( + "[harness]\ndefault = \"gemini\"\n", + "harness.default must be claude or codex, not \"gemini\"", + ), + ( + "[harness]\ndefault = true\n", + "harness.default must be a quoted name", + ), + ("[harness]\nmodel = \"opus\"\n", "unknown key harness.model"), + ( + "[harness.gemini]\nmodel = \"x\"\n", + "unknown key harness.gemini", + ), + ( + "[harness]\nclaude = \"opus\"\n", + "harness.claude must be the section [harness.claude]", + ), + ( + "[harness.claude]\nmodels = \"opus\"\n", + "unknown key harness.claude.models", + ), + ( + "[harness.codex]\neffort = 3\n", + "harness.codex.effort must be a quoted string", + ), + ( + "harness = \"claude\"\n", + "harness must be the section [harness]", + ), + ] { + let error_text = format!("{:#}", parse(text).unwrap_err()); + + assert!(error_text.contains(error), "{text:?}: {error_text}"); + } + } + #[test] fn merge_always_is_read() { assert!(parse("[merge]\nalways = true\n").unwrap().merge_always); diff --git a/src/delivery.rs b/src/delivery.rs index 323b12fab..173fe6d74 100644 --- a/src/delivery.rs +++ b/src/delivery.rs @@ -21,6 +21,7 @@ use crate::base_fix::BaseFix; use crate::ci::{self, Ci, FailedChecks}; use crate::failed_run::{FailedRun, PolicyRefusal, interrupted_or}; use crate::github::{self, Mergeable, PullRequest}; +use crate::harness::{self, Choice}; use crate::host; use crate::interrupt; use crate::issue::IssueUrl; @@ -44,6 +45,9 @@ pub struct Delivery<'a> { pub base_fix: &'a mut BaseFix, /// Where its Session logs go. pub logs: &'a Logs, + /// The Harness, Model and Effort its sessions run on, and its child + /// Runs' too. + pub harness: &'a Choice, } /// The session a Delivery opens with: the implement session, or the Spec @@ -61,7 +65,9 @@ impl Delivery<'_> { /// Take the pull request from the branch checked out in `worktree` to the /// goal. With `opening.catch_up_from_origin`, first fast-forward the /// branch to origin. Then run the opening session, push the branch, for - /// any commit the session left unpushed, run `before_ready`, and mark the + /// any commit the session left unpushed, write the line that says what + /// it was built with in the pull request's body, only warning if that + /// fails, run `before_ready`, and mark the /// pull request ready, failing unless it exists, is open and targets the /// Base branch. Then keep it mergeable and its CI green through the /// Repair loop, and for [`Goal::Merged`], Self-merge it. A merge that @@ -82,16 +88,30 @@ impl Delivery<'_> { branch: worktree.branch(), goal: self.goal, }; - let (delivered, log) = Sessions::within(self.logs, worktree.path(), |sessions| { - let mut outside = InWorktree { - issue: self.issue, - worktree: &worktree, - base: self.base, - base_fix: self.base_fix, - sessions, - }; - route.steps(&mut outside, &opening, |_| before_ready()) - }); + let sessions_on = self.harness; + let (delivered, log) = + Sessions::within(self.logs, worktree.path(), sessions_on, |sessions| { + let mut outside = InWorktree { + issue: self.issue, + worktree: &worktree, + base: self.base, + base_fix: self.base_fix, + sessions, + }; + route.steps(&mut outside, &opening, |_| { + if let Err(error) = write_built_with(self.issue, worktree.branch(), sessions_on) + { + progress::warn( + &error, + format_args!( + "could not write \"{}\" in the pull request's body", + sessions_on.built_with() + ), + ); + } + before_ready() + }) + }); match delivered { Ok(pr_url) => Ok(Reached { pr_url, @@ -111,6 +131,27 @@ impl Delivery<'_> { } } +/// Write the line that says the pull request from `branch`, for `issue`, was +/// built with `harness` in its body, in place of any thirdshift wrote there +/// before. Nothing if there is no such pull request: marking it ready says +/// so. A failure is for the Delivery to warn of, as the work is done +/// without it. +fn write_built_with(issue: &IssueUrl, branch: &str, harness: &Choice) -> Result<()> { + let Some(pr) = github::pull_request_for(issue, branch)? else { + return Ok(()); + }; + let body = github::pr_body(issue, pr.number)?; + let written = harness::with_built_with(&body, harness); + if written != body { + progress::step(format_args!( + "writing \"{}\" in the pull request's body", + harness.built_with() + )); + github::set_pr_body(issue, pr.number, &written)?; + } + Ok(()) +} + /// The steps of a Delivery of the pull request for `issue`, from `branch` /// into the Base branch `base`, to `goal`. struct Route<'a> { diff --git a/src/harness.rs b/src/harness.rs new file mode 100644 index 000000000..cbccd93fe --- /dev/null +++ b/src/harness.rs @@ -0,0 +1,457 @@ +//! The Harness a Command's agent sessions run on, with its Model and Effort: +//! one [`Choice`] per Command, made by the command, else the User config, +//! else the default, checked before any work, passed on to every session and +//! child Run, and recorded with what the Command did. + +use std::fmt; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; + +use crate::progress; + +/// The headless agent CLI a Command's sessions run on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Harness { + Claude, + Codex, +} + +impl Harness { + /// The Harness named `name`, as the command line and the User config + /// write it, if it is one. + pub fn named(name: &str) -> Option<Harness> { + match name { + "claude" => Some(Harness::Claude), + "codex" => Some(Harness::Codex), + _ => None, + } + } + + /// Its name, which is also its CLI's. + pub fn name(self) -> &'static str { + match self { + Harness::Claude => "claude", + Harness::Codex => "codex", + } + } +} + +/// The names a Harness is chosen by, for the messages that list them. +pub const NAMES: &str = "claude or codex"; + +/// A Model and an Effort, each none where it is left to the Harness. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ModelAndEffort { + pub model: Option<String>, + pub effort: Option<String>, +} + +/// The User config's `[harness]` section: the default Harness, if it names +/// one, and a Model and Effort for each Harness. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Settings { + /// `harness.default`. + pub default: Option<Harness>, + /// `[harness.claude]`. + pub claude: ModelAndEffort, + /// `[harness.codex]`. + pub codex: ModelAndEffort, +} + +impl Settings { + /// The Model and Effort set for `harness`. + fn of(&self, harness: Harness) -> &ModelAndEffort { + match harness { + Harness::Claude => &self.claude, + Harness::Codex => &self.codex, + } + } +} + +/// What a command asked for, each none if it said nothing about it. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Asked { + /// What `harness <name>` asked for. + pub harness: Option<Harness>, + /// What `model <name>` and `effort <level>` asked for. + pub model_and_effort: ModelAndEffort, +} + +/// What chose the Harness, for a failure to name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChosenBy { + /// The command's `harness` flag, or for a child Run, the command that + /// started it. + Command, + /// The User config's `harness.default`. + UserConfig, + /// Neither: Claude Code, the default. + Default, +} + +/// The Harness, Model and Effort every session of a Command runs on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Choice { + pub harness: Harness, + /// The Model, or none to leave it to the Harness. + pub model: Option<String>, + /// The Effort, or none to leave it to the Harness. + pub effort: Option<String>, + pub chosen_by: ChosenBy, +} + +impl Default for Choice { + /// Claude Code, with its own Model and Effort: a Command's sessions with + /// nothing asked and nothing set. + fn default() -> Self { + Choice { + harness: Harness::Claude, + model: None, + effort: None, + chosen_by: ChosenBy::Default, + } + } +} + +impl Choice { + /// What a command that `asked` for it runs on, given the User config's + /// `settings`: for each of the Harness, Model and Effort, what the + /// command asked, else what the User config sets, else the default. The + /// Model and Effort it sets are those of the Harness chosen. + pub fn of(asked: &Asked, settings: &Settings) -> Choice { + let (harness, chosen_by) = match (asked.harness, settings.default) { + (Some(harness), _) => (harness, ChosenBy::Command), + (None, Some(harness)) => (harness, ChosenBy::UserConfig), + (None, None) => (Harness::Claude, ChosenBy::Default), + }; + let set = settings.of(harness); + let asked = &asked.model_and_effort; + Choice { + harness, + model: asked.model.clone().or_else(|| set.model.clone()), + effort: asked.effort.clone().or_else(|| set.effort.clone()), + chosen_by, + } + } + + /// The line thirdshift writes in a pull request's body and a Run + /// notification, as in `Built with claude · claude-opus-5-5 · high`. + pub fn built_with(&self) -> String { + format!("Built with {self}") + } + + /// The arguments that run a Claude session on its Model and Effort, + /// where they are set. + pub fn claude_args(&self) -> Vec<&str> { + let mut args = Vec::new(); + if let Some(model) = &self.model { + args.extend(["--model", model]); + } + if let Some(effort) = &self.effort { + args.extend(["--effort", effort]); + } + args + } + + /// Check, before any work, that sessions can run on this: Codex is not + /// supported yet; the Harness's CLI must be on `PATH`; and a Model named + /// for Claude must take a minimal test call, with the Effort, if any. + /// Each failure says what to change. + pub fn check(&self) -> Result<()> { + let cli = self.harness.name(); + if self.harness == Harness::Codex { + bail!( + "harness codex, chosen by {}, is not supported yet: Codex sessions come in a \ + later thirdshift; choose harness claude", + self.chosen_by + ); + } + if !on_path(cli) { + bail!( + "{cli} is not on PATH, and the Harness {cli} is chosen by {}: install it, \ + or choose another Harness", + self.chosen_by + ); + } + let Some(model) = &self.model else { + return Ok(()); + }; + progress::step(format_args!( + "checking the Model {model} with a test call to {cli}" + )); + let mut child = Command::new(cli) + .arg("-p") + .args(self.claude_args()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("could not run {cli} to check the Model {model}"))?; + // Dropped once written, closing stdin, so the call can end. + if let Some(mut stdin) = child.stdin.take() { + let _ = stdin.write_all(TEST_PROMPT.as_bytes()); + } + let output = child + .wait_with_output() + .with_context(|| format!("could not run {cli} to check the Model {model}"))?; + if output.status.success() { + return Ok(()); + } + let said = [&output.stderr, &output.stdout] + .map(|said| String::from_utf8_lossy(said).trim().to_string()) + .into_iter() + .filter(|said| !said.is_empty()) + .collect::<Vec<_>>() + .join("\n"); + let effort = match &self.effort { + Some(effort) => format!(" with the Effort {effort}"), + None => String::new(), + }; + bail!("{cli} refused a test call on the Model {model}{effort}: {said}") + } +} + +/// What the test call that checks a Model asks. +const TEST_PROMPT: &str = "Reply with OK."; + +impl fmt::Display for Choice { + /// `<harness> · <model> · <effort>`, as in `claude · claude-opus-5-5 · + /// high`, saying `default model` or `default effort` for one left to the + /// Harness. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{} · {} · {}", + self.harness.name(), + self.model.as_deref().unwrap_or("default model"), + self.effort.as_deref().unwrap_or("default effort") + ) + } +} + +impl fmt::Display for ChosenBy { + /// The setting that chose the Harness, as a failure names it. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(match self { + ChosenBy::Command => "the harness flag", + ChosenBy::UserConfig => "harness.default in the User config", + ChosenBy::Default => { + "the default, as neither the harness flag nor harness.default in the User \ + config names one" + } + }) + } +} + +/// Whether an executable file named `cli` is in a directory on `PATH`. +fn on_path(cli: &str) -> bool { + use std::os::unix::fs::PermissionsExt; + let Some(path) = std::env::var_os("PATH") else { + return false; + }; + std::env::split_paths(&path).any(|dir| { + Path::new(&dir) + .join(cli) + .metadata() + .is_ok_and(|file| file.is_file() && file.permissions().mode() & 0o111 != 0) + }) +} + +/// The marker that ends the line thirdshift writes in a pull request's body, +/// so it can be found and replaced. +const BUILT_WITH_MARKER: &str = "<!-- thirdshift:built-with -->"; + +/// `body` with `choice`'s line in it: in place of the one thirdshift wrote +/// before, or else added at the end. +pub fn with_built_with(body: &str, choice: &Choice) -> String { + let line = format!("{} {BUILT_WITH_MARKER}", choice.built_with()); + if body.contains(BUILT_WITH_MARKER) { + return body + .split_inclusive('\n') + .map(|old| { + if old.trim_end().ends_with(BUILT_WITH_MARKER) { + let end = &old[old.trim_end_matches(['\r', '\n']).len()..]; + format!("{line}{end}") + } else { + old.to_string() + } + }) + .collect(); + } + if body.trim().is_empty() { + return format!("{line}\n"); + } + format!("{}\n\n{line}\n", body.trim_end()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(model: &str, effort: &str) -> ModelAndEffort { + let named = |value: &str| (!value.is_empty()).then(|| value.to_string()); + ModelAndEffort { + model: named(model), + effort: named(effort), + } + } + + /// A User config whose default is Codex, with a Model and Effort for + /// each Harness. + fn codex_default() -> Settings { + Settings { + default: Some(Harness::Codex), + claude: set("opus", "high"), + codex: set("gpt-6.1-sol", "max"), + } + } + + fn choice(harness: Harness, model: &str, effort: &str, chosen_by: ChosenBy) -> Choice { + let set = set(model, effort); + Choice { + harness, + model: set.model, + effort: set.effort, + chosen_by, + } + } + + #[test] + fn with_nothing_asked_and_nothing_set_sessions_run_on_claude_with_its_own_model_and_effort() { + let chosen = Choice::of(&Asked::default(), &Settings::default()); + + assert_eq!(chosen, Choice::default()); + assert!(chosen.claude_args().is_empty()); + } + + #[test] + fn the_user_config_chooses_the_harness_and_its_own_model_and_effort() { + let chosen = Choice::of(&Asked::default(), &codex_default()); + + assert_eq!( + chosen, + choice(Harness::Codex, "gpt-6.1-sol", "max", ChosenBy::UserConfig) + ); + } + + #[test] + fn a_harness_on_the_command_takes_that_harnesss_model_and_effort_from_the_user_config() { + let asked = Asked { + harness: Some(Harness::Claude), + ..Asked::default() + }; + + let chosen = Choice::of(&asked, &codex_default()); + + assert_eq!( + chosen, + choice(Harness::Claude, "opus", "high", ChosenBy::Command) + ); + } + + #[test] + fn a_model_or_effort_on_the_command_applies_to_whichever_harness_is_chosen() { + for (asked, model, effort) in [ + (set("sonnet", ""), "sonnet", "max"), + (set("", "low"), "gpt-6.1-sol", "low"), + (set("sonnet", "low"), "sonnet", "low"), + ] { + let asked = Asked { + harness: None, + model_and_effort: asked, + }; + + let chosen = Choice::of(&asked, &codex_default()); + + assert_eq!( + chosen, + choice(Harness::Codex, model, effort, ChosenBy::UserConfig), + "{asked:?}" + ); + } + } + + #[test] + fn a_harness_with_nothing_set_leaves_its_model_and_effort_to_it() { + let settings = Settings { + default: Some(Harness::Claude), + ..codex_default() + }; + let asked = Asked { + harness: Some(Harness::Codex), + ..Asked::default() + }; + let chosen = Choice::of( + &asked, + &Settings { + codex: ModelAndEffort::default(), + ..settings + }, + ); + + assert_eq!(chosen, choice(Harness::Codex, "", "", ChosenBy::Command)); + } + + #[test] + fn claude_is_given_the_model_and_effort_that_are_set() { + for (model, effort, args) in [ + ("opus", "", vec!["--model", "opus"]), + ("", "max", vec!["--effort", "max"]), + ( + "claude-opus-5-5", + "high", + vec!["--model", "claude-opus-5-5", "--effort", "high"], + ), + ] { + let chosen = choice(Harness::Claude, model, effort, ChosenBy::Command); + + assert_eq!(chosen.claude_args(), args); + } + } + + #[test] + fn the_built_with_line_names_the_harness_model_and_effort_or_the_harnesss_default() { + for (chosen, line) in [ + ( + choice( + Harness::Claude, + "claude-opus-5-5", + "high", + ChosenBy::Command, + ), + "Built with claude · claude-opus-5-5 · high", + ), + ( + Choice::default(), + "Built with claude · default model · default effort", + ), + ( + choice(Harness::Codex, "", "max", ChosenBy::UserConfig), + "Built with codex · default model · max", + ), + ] { + assert_eq!(chosen.built_with(), line); + } + } + + #[test] + fn the_built_with_line_is_added_to_a_body_and_replaces_the_one_written_before() { + let opus = choice(Harness::Claude, "opus", "high", ChosenBy::Command); + let line = "Built with claude · opus · high <!-- thirdshift:built-with -->"; + + let added = with_built_with("Adds a button.\n\nCloses #7\n", &opus); + assert_eq!(added, format!("Adds a button.\n\nCloses #7\n\n{line}\n")); + assert_eq!(with_built_with("", &opus), format!("{line}\n")); + + let old = "Adds a button.\n\nBuilt with claude · default model · default effort \ + <!-- thirdshift:built-with -->\n\nCloses #7\n"; + assert_eq!( + with_built_with(old, &opus), + format!("Adds a button.\n\n{line}\n\nCloses #7\n") + ); + assert_eq!(with_built_with(&added, &opus), added); + } +} diff --git a/src/logs.rs b/src/logs.rs index c38ee4576..b4150d19e 100644 --- a/src/logs.rs +++ b/src/logs.rs @@ -19,7 +19,9 @@ use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard, PoisonError}; use crate::config::UserConfig; +use crate::harness::Choice; use crate::issue::{IssueUrl, Repo}; +use crate::progress; use activity::Kind; @@ -104,19 +106,21 @@ pub fn configured(config: &UserConfig) { } } -/// Record that the command starts `work`: keep its Command log, showing any -/// lines held from the terminal, then write the Activity log's line that it -/// started, naming that log. Only the outermost command's first call does -/// anything: the work a Pickup run or an Architect run started covers the -/// Run it dispatches, and a child Run records nothing, as the command that -/// started it does. -pub fn started(work: Work) { +/// Record that the command starts `work`, its sessions on `harness`: keep its +/// Command log, showing any lines held from the terminal, with a line naming +/// the Harness, Model and Effort, then write the Activity log's line that it +/// started, naming that log and those. Only the outermost command's first +/// call does anything: the work a Pickup run or an Architect run started +/// covers the Run it dispatches, and a child Run records nothing, as the +/// command that started it does. +pub fn started(work: Work, harness: &Choice) { if !record().take_start() { return; } let root = work.root(); command_log::keep(work.command_log(&root, command_log::stamp())); - activity::start(&root, work.kind(), work.issue()); + progress::step(format_args!("sessions run on {harness}")); + activity::start(&root, work.kind(), work.issue(), harness); } /// Record that `pass` was skipped on `repo` for `reason`, unless the last diff --git a/src/logs/activity.rs b/src/logs/activity.rs index 4f35f06ea..a6847d70a 100644 --- a/src/logs/activity.rs +++ b/src/logs/activity.rs @@ -23,6 +23,7 @@ use anyhow::{Context, Result}; use chrono::Local; use super::command_log; +use crate::harness::Choice; use crate::progress; /// The file's name, at the root of a repository's logs. @@ -73,8 +74,10 @@ static WARNED: AtomicBool = AtomicBool::new(false); /// Record that this command, a `kind` on `issue`, if it has one, starts work /// on the repository whose logs are at `root`, naming its Command log, if it -/// keeps one. Only once: [`end`] then writes its end line. -pub(super) fn start(root: &Path, kind: Kind, issue: Option<u64>) { +/// keeps one, and the Harness, Model and Effort of `harness`, as in +/// `Run #7 started: commands/issue/7-<stamp>.log, on claude · opus · high`. +/// Only once: [`end`] then writes its end line. +pub(super) fn start(root: &Path, kind: Kind, issue: Option<u64>, harness: &Choice) { let command_log = match command_log::path() { Some(path) => path .strip_prefix(root) @@ -88,7 +91,7 @@ pub(super) fn start(root: &Path, kind: Kind, issue: Option<u64>) { kind, issue, }; - let line = format!("{} started: {command_log}", started.subject()); + let line = format!("{} started: {command_log}, on {harness}", started.subject()); if STARTED.set(started).is_ok() { write(root, &line, None); } diff --git a/src/main.rs b/src/main.rs index f45022f69..6c070f485 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ mod email; mod failed_run; mod git; mod github; +mod harness; mod host; mod interrupt; mod issue; @@ -67,8 +68,28 @@ usage: thirdshift <Issue URL> Run the factory on the iss thirdshift version Print thirdshift's version thirdshift help Print this help -merge, --no-merge, --email, --no-email, base-fix, --no-base-fix and parallel <n> (or ---parallel <n>) go before or after the Issue URL, in any order. +merge, --no-merge, --email, --no-email, base-fix, --no-base-fix, parallel <n> (or +--parallel <n>), harness <name>, model <name> and effort <level> (or --harness, --model and +--effort) go before or after the Issue URL, in any order. + +harness, model and effort choose the Harness every session of the command runs on, and its +Model and Effort, for each Ticket's Run and a Base fix too. For each, the command wins, then +the User config, then the default: claude, with its own Model and Effort. A Model and Effort +in the User config come from the chosen Harness's own section, so harness claude over a codex +default takes [harness.claude]: + + [harness] + default = \"claude\" + + [harness.claude] + model = \"opus\" + effort = \"high\" + +Before any work, the Harness's CLI must be on PATH, and a named Model gets a minimal test call +with its Effort, which must succeed: a failure stops the command naming what to fix, before +the Claim, the worktree and any Command log. Only claude works for now: harness codex fails as +not supported yet. The Command log, the Activity log's start line, the pull request's body, +as in Built with claude · opus · high, and the Run notification each name all three. --email sends one Run notification when the Run ends, whatever the outcome: ready for review, merged, failed or interrupted. --email <address> sends it to <address>; a word @@ -133,8 +154,9 @@ labels it architect-plan, creating the label if the repository lacks it, and dis as thirdshift <Issue URL> would: a Spec run on a Spec, a Run on a single Ticket. A Spec's Tickets are never labelled architect-plan. That run makes the Claim on the plan, which keeps architect-plan. The Architect run ends as that run does, with its exit code and its PR's URL. -merge, --no-merge, base-fix, --no-base-fix and parallel <n> apply to that run, as do the -User config's defaults; parallel <n> fails it if the plan is a single Ticket. The review +merge, --no-merge, base-fix, --no-base-fix, parallel <n>, harness, model and effort apply to +that run, as do the User config's defaults; parallel <n> fails it if the plan is a single +Ticket. harness, model and effort apply to the review too. The review itself watches no CI, so only that run can start a Base fix. With --plan-only, the Architect run prints the plan's URL and stops instead, for you to read, edit and run with thirdshift <Issue URL>, and takes none of those flags. <focus> is free text, one argument, @@ -210,8 +232,8 @@ Each ready-for-agent issue a pass looks at and does not take gets one line on st the first reason that applies, such as #21 is a Ticket of #20, which is not ready or #30 blocked by #29, before the line that says what the pass did. A line names the issue taken, and the run it is dispatched as makes the Claim on it. The Pickup run ends as that run does, -with its exit code and its PR's URL. merge, --no-merge, base-fix, --no-base-fix and -parallel <n> apply to that run, as do the User config's defaults; parallel <n> is ignored +with its exit code and its PR's URL. merge, --no-merge, base-fix, --no-base-fix, +parallel <n>, harness, model and effort apply to that run, as do the User config's defaults; parallel <n> is ignored when the issue is not a Spec. base <branch> names the Pickup run's Base branch as it does an Architect run's, and the dispatched run branches off <branch> and targets it: @@ -357,11 +379,12 @@ fn main() -> ExitCode { None => Asks::of_run(&issue, &flags, config), }; let ask = |config: &UserConfig| asks_of(config).notification; - let (config, started) = match command::start(begin, ask, About::Issue(&issue)) { + let (config, mut started) = match command::start(begin, ask, About::Issue(&issue)) { Ok(started) => started, Err(failure) => return failure, }; let asks = asks_of(&config); + started.built_with(&asks.harness); let started_by = given .as_ref() .map_or(StartedBy::Command, |given| StartedBy::Child(&given.kind)); @@ -392,14 +415,18 @@ fn main() -> ExitCode { /// `activity.quiet_skips` set, a skipped run prints nothing at all. fn architect(args: ArchitectArgs) -> ExitCode { let ask = |config: &UserConfig| args.flags.notification(config); - let (config, started) = match command::start(Begin::ArchitectRun, ask, About::ArchitectRun) { + let (config, mut started) = match command::start(Begin::ArchitectRun, ask, About::ArchitectRun) + { Ok(started) => started, Err(failure) => return failure, }; + let harness = args.flags.harness(&config); + started.built_with(&harness); let review = match architect::run( args.focus.as_deref(), args.base.as_deref(), config.launch_pull, + &harness, ) { Ok(Outcome::Skipped(skipped)) => { return started.finish(Ending::Skipped(skipped.into())); @@ -443,7 +470,9 @@ fn pickup(args: PickupArgs) -> ExitCode { Ok(started) => started, Err(failure) => return failure, }; - let taken = match pickup::run(args.base.as_deref(), config.pickup_limit) { + let harness = args.flags.harness(&config); + started.built_with(&harness); + let taken = match pickup::run(args.base.as_deref(), config.pickup_limit, &harness) { Ok(pickup::Outcome::Taken(taken)) => taken, Ok(pickup::Outcome::Skipped(skipped)) => { return started.finish(Ending::Skipped(skipped.into())); diff --git a/src/notification.rs b/src/notification.rs index 439a382be..56b1f2246 100644 --- a/src/notification.rs +++ b/src/notification.rs @@ -10,6 +10,7 @@ use anyhow::Result; use crate::config::EmailSettings; use crate::email::Resend; use crate::github; +use crate::harness::Choice; use crate::host; use crate::issue::{IssueUrl, Repo}; use crate::launch; @@ -76,6 +77,8 @@ enum Subject { pub struct RunNotification { checked: Checked, subject: Subject, + /// The line that says what the command's sessions ran on, once known. + built_with: Option<String>, } impl RunNotification { @@ -100,7 +103,11 @@ impl RunNotification { About::ArchitectRun => Subject::ArchitectRun(launch::repo().ok()), About::PickupRun => Subject::Pending, }; - Ok(RunNotification { checked, subject }) + Ok(RunNotification { + checked, + subject, + built_with: None, + }) } /// Take note that the Pickup run took `issue`, titled `title`, as its @@ -114,6 +121,12 @@ impl RunNotification { }; } + /// Take note that the command's sessions run on `harness`, as the + /// notification then says, as its pull request's body does. + pub fn built_with(&mut self, harness: &Choice) { + self.built_with = Some(harness.built_with()); + } + /// Send the one notification for the command that ended as `account` /// tells. A Pickup run that took no issue has nothing to tell, and sends /// none, as a skipped pass, which has no account, does not either. A @@ -127,6 +140,7 @@ impl RunNotification { let command_log = logs::command_log_path(); let text = body( account, + self.built_with.as_deref(), command_log.as_deref(), host.as_deref().unwrap_or("unknown host"), self.checked.started.elapsed(), @@ -170,11 +184,18 @@ fn architect_subject(repo: Option<&Repo>, outcome: &str) -> String { } /// The plain-text body of the notification for a command that ended as -/// `account` tells, naming `command_log`, if the command keeps one, and -/// `host`, for a command that `took` so long. An Architect run's starts with +/// `account` tells, with the line that says what its sessions were +/// `built_with`, if known, naming `command_log`, if the command keeps one, +/// and `host`, for a command that `took` so long. An Architect run's starts with /// how its review ended, and the outcome of the run it dispatched, if any. /// The cause is left out of an interrupted command's, whose outcome says so. -fn body(account: &Account, command_log: Option<&Path>, host: &str, took: Duration) -> String { +fn body( + account: &Account, + built_with: Option<&str>, + command_log: Option<&Path>, + host: &str, + took: Duration, +) -> String { let mut text = String::new(); if let Some(review) = &account.review { text += &format!("Review: {}\n", review.line); @@ -202,6 +223,9 @@ fn body(account: &Account, command_log: Option<&Path>, host: &str, took: Duratio if let Some(log) = command_log { text += &format!("Command log: {}\n", log.display()); } + if let Some(built_with) = built_with { + text += &format!("{built_with}\n"); + } text += &format!("Host: {host}\n"); text += &format!("Took: {}\n", self::took(took)); if !account.ticket_lines.is_empty() { @@ -295,7 +319,7 @@ mod tests { /// The body for a command that ended as `account` tells, run on /// `droplet-1` for 4s with no Command log. fn body_of(account: &Account) -> String { - body(account, None, "droplet-1", Duration::from_secs(4)) + body(account, None, None, "droplet-1", Duration::from_secs(4)) } #[test] @@ -463,6 +487,7 @@ mod tests { assert_eq!( body( &account, + Some("Built with claude · opus · high"), Some(Path::new("/home/me/.thirdshift/logs/commands/issue/x.log")), "droplet-1", Duration::from_secs(4) @@ -470,6 +495,7 @@ mod tests { "Pull request: https://github.com/acme/widgets/pull/1\n\ Session log: /home/me/.thirdshift/logs/sessions/x.jsonl\n\ Command log: /home/me/.thirdshift/logs/commands/issue/x.log\n\ + Built with claude · opus · high\n\ Host: droplet-1\n\ Took: 4s\n" ); diff --git a/src/pickup.rs b/src/pickup.rs index 7380e8db1..e67bc7569 100644 --- a/src/pickup.rs +++ b/src/pickup.rs @@ -14,6 +14,7 @@ use anyhow::Result; use crate::claim; use crate::github::ListedIssue; +use crate::harness::Choice; use crate::issue::{IssueUrl, Repo}; use crate::labels::Edit; use crate::launch::{self, AlreadyRunning, Launch, Start}; @@ -106,10 +107,12 @@ impl fmt::Display for Skipped { /// It first makes the Sweep, taking the Claim off the repository's closed /// issues, and is then skipped, still before any search, if the repository /// is at its Claim limit: `limit` or more of its open issues carry a Claim. -/// A skip is recorded in the repository's Activity log. Once it takes an -/// issue, it records that it started work on it, which keeps its Command -/// log. -pub fn run(base: Option<&str>, limit: NonZeroUsize) -> Result<Outcome> { +/// A skip is recorded in the repository's Activity log. Once it has found an +/// issue to take, it checks `harness`, the Harness, Model and Effort the run +/// it dispatches will run its sessions on, failing before any work if they +/// can't run; then it records that it started work on the issue, which +/// keeps its Command log. +pub fn run(base: Option<&str>, limit: NonZeroUsize, harness: &Choice) -> Result<Outcome> { let Launch { directory, repo, @@ -132,7 +135,8 @@ pub fn run(base: Option<&str>, limit: NonZeroUsize) -> Result<Outcome> { "taking Ready issue #{} \"{}\", as thirdshift {} would", listed.issue.number, listed.title, listed.issue.url )); - logs::started(Work::PickupRun(&listed.issue)); + harness.check()?; + logs::started(Work::PickupRun(&listed.issue), harness); Ok(Outcome::Taken(Taken { issue: listed.issue, title: listed.title, diff --git a/src/prompts_page.rs b/src/prompts_page.rs index 6ebedce31..ba3fd7b73 100644 --- a/src/prompts_page.rs +++ b/src/prompts_page.rs @@ -13,6 +13,7 @@ use include_dir::{Dir, File}; use crate::ci::FailedChecks; use crate::github::{Check, CheckState}; +use crate::harness::Choice; use crate::issue::IssueUrl; use crate::plugin::SKILLS; use crate::prompt; @@ -346,7 +347,8 @@ fn render() -> String { /// The `claude` command line a session starts with, or, given a session id, /// resumes with, with the page's placeholders. fn command(resume: Option<&str>) -> String { - let args = claude_args(OsStr::new(PLUGIN_DIR), resume, PROMPT); + let harness = Choice::default(); + let args = claude_args(OsStr::new(PLUGIN_DIR), &harness, resume, PROMPT); let args: Vec<_> = args.iter().map(|arg| arg.to_string_lossy()).collect(); format!("claude {}", args.join(" ")) } diff --git a/src/run.rs b/src/run.rs index 73bca72f6..cef201c6d 100644 --- a/src/run.rs +++ b/src/run.rs @@ -14,6 +14,7 @@ use crate::claim::{self, Claim}; use crate::delivery::{Delivery, Opening}; use crate::failed_run::FailedRun; use crate::github::{self, Ticket}; +use crate::harness::Choice; use crate::interrupt; use crate::issue::IssueUrl; use crate::launch::LaunchDirectory; @@ -113,7 +114,11 @@ impl<'a> StartedBy<'a> { /// `asks`, to its end. Its Run notification, if `asks` has it send one, is /// for what started the Run to send, once it has ended. pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended { - let mut base_fix = BaseFix::new(started_by.child(), asks.base_fix.clone()); + let mut base_fix = BaseFix::new( + started_by.child(), + asks.base_fix.clone(), + asks.harness.clone(), + ); let outcome = run(issue, asks, started_by, &mut base_fix); Ended { outcome, @@ -132,8 +137,11 @@ pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended /// date with origin. /// /// The Launch directory is opened first, with its Origin match, the check -/// that `issue` is open and the git identity check; the rest is [`start`], -/// through the Launch directory, GitHub and the logs. +/// that `issue` is open and the git identity check. A Run started by its +/// command then checks the Harness, Model and Effort its sessions run on, as +/// `asks` say: one started by another thirdshift, or dispatched by a pass, +/// runs on what that command checked. The rest is [`start`], through the +/// Launch directory, GitHub and the logs. /// /// `base_fix` is the one Base fix the Run, or a Spec run for its Spec PR, may /// start, or wait on, when its only red checks are Inherited failures: what @@ -145,12 +153,16 @@ fn run( base_fix: &mut BaseFix, ) -> Result<Reached, FailedRun> { let directory = LaunchDirectory::open_for_run(issue)?; + if let StartedBy::Command = started_by { + asks.harness.check()?; + } let mut outside = LaunchAndGitHub { directory: &directory, issue, goal: asks.goal, base_fix, logs: Logs::of_run(issue), + harness: &asks.harness, claim: None, }; start( @@ -340,13 +352,14 @@ trait Outside { /// The outside world of a Run on `issue` from the opened Launch directory /// `directory`: GitHub, its git, the logs, the Claim it made, the worktree, /// the Spec run and the Delivery to `goal`, with `base_fix` and the Run's -/// Session `logs`. +/// Session `logs`, its sessions on `harness`. struct LaunchAndGitHub<'a> { directory: &'a LaunchDirectory, issue: &'a IssueUrl, goal: Goal, base_fix: &'a mut BaseFix, logs: Logs, + harness: &'a Choice, claim: Option<Claim<'a>>, } @@ -359,6 +372,7 @@ impl LaunchAndGitHub<'_> { goal: self.goal, base_fix: self.base_fix, logs: &self.logs, + harness: self.harness, } } } @@ -371,7 +385,7 @@ impl<'a> Outside for LaunchAndGitHub<'a> { } fn started(&mut self, work: Work) { - logs::started(work); + logs::started(work, self.harness); } fn select(&mut self) -> Result<Selection> { @@ -678,6 +692,7 @@ mod tests { parallel_asked: false, base_fix: BaseFixAsk::Forbid, launch_pull: false, + harness: Choice::default(), } } diff --git a/src/session.rs b/src/session.rs index da5abe292..65d5c7165 100644 --- a/src/session.rs +++ b/src/session.rs @@ -12,6 +12,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail}; +use crate::harness::Choice; use crate::interrupt; use crate::issue::{IssueUrl, Repo}; use crate::logs; @@ -57,11 +58,13 @@ impl Logs { } /// Where a Run's or an Architect run's sessions run: in `worktree`, with the -/// Factory skills plugin loaded, each logged in its `logs`. It lasts for the -/// steps given to [`Sessions::within`], and so does the plugin. +/// Factory skills plugin loaded, on the Command's `harness`, each logged in +/// its `logs`. It lasts for the steps given to [`Sessions::within`], and so +/// does the plugin. pub struct Sessions<'a> { logs: &'a Logs, worktree: &'a Path, + harness: &'a Choice, plugin: Plugin, /// The log of the session started last, a Resume included. last_log: RefCell<Option<PathBuf>>, @@ -74,7 +77,7 @@ pub struct Sessions<'a> { impl<'a> Sessions<'a> { /// Write the Factory skills plugin, then take `steps`, which run their - /// sessions through the `Sessions` they are given. Returns what `steps` + /// sessions on `harness` through the `Sessions` they are given. Returns what `steps` /// came to, with the most recent session's log, if a session created it. /// If a session's last ending, its Resume's if it got one, left /// background work to be killed and `steps` then fail, the failure names @@ -83,6 +86,7 @@ impl<'a> Sessions<'a> { pub fn within<T>( logs: &'a Logs, worktree: &'a Path, + harness: &'a Choice, steps: impl FnOnce(&Self) -> Result<T>, ) -> (Result<T>, Option<PathBuf>) { let plugin = match Plugin::write() { @@ -92,6 +96,7 @@ impl<'a> Sessions<'a> { let sessions = Sessions { logs, worktree, + harness, plugin, last_log: RefCell::default(), endings_with_killed_work: RefCell::default(), @@ -153,14 +158,8 @@ impl<'a> Sessions<'a> { let log = self.logs.path(kind); progress::step(format_args!("logging the session to {}", log.display())); *self.last_log.borrow_mut() = Some(log.clone()); - run( - kind, - self.worktree, - self.plugin.path(), - resume, - prompt, - &log, - ) + let args = claude_args(self.plugin.path().as_os_str(), self.harness, resume, prompt); + run(kind, self.worktree, &args, &log) } } @@ -176,20 +175,12 @@ fn ending_with(killed: &[&str]) -> String { } } -/// Run `claude` headless in auto mode in `worktree`, with the Factory skills -/// plugin at `plugin_dir` loaded, streaming its output to `log` and condensing -/// it to progress lines on stderr, each labelled `kind`. With `resume`, the -/// session with that id continues, given `prompt`. Returns what the stream -/// showed once `claude` has exited cleanly. An interrupt stops the session and -/// fails with `interrupted`. -fn run( - kind: &str, - worktree: &Path, - plugin_dir: &Path, - resume: Option<&str>, - prompt: &str, - log: &Path, -) -> Result<Progress> { +/// Run `claude` with `args`, as [`claude_args`] gives them, in `worktree`, +/// streaming its output to `log` and condensing it to progress lines on +/// stderr, each labelled `kind`. Returns what the stream showed once +/// `claude` has exited cleanly. An interrupt stops the session and fails +/// with `interrupted`. +fn run(kind: &str, worktree: &Path, args: &[&OsStr], log: &Path) -> Result<Progress> { if let Some(dir) = log.parent() { fs::create_dir_all(dir).with_context(|| format!("could not create {}", dir.display()))?; } @@ -197,7 +188,7 @@ fn run( File::create(log).with_context(|| format!("could not create {}", log.display()))?; let started = Instant::now(); let mut child = Command::new("claude") - .args(claude_args(plugin_dir.as_os_str(), resume, prompt)) + .args(args) .current_dir(worktree) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -262,10 +253,12 @@ fn run( } /// The arguments every session runs `claude` with: headless in auto mode, -/// with the Factory skills plugin at `plugin_dir` loaded, streaming JSON. With -/// `resume`, the session with that id continues. The prompt comes last. +/// with the Factory skills plugin at `plugin_dir` loaded, on the Model and +/// Effort `harness` sets, if any, streaming JSON. With `resume`, the session +/// with that id continues. The prompt comes last. pub fn claude_args<'a>( plugin_dir: &'a OsStr, + harness: &'a Choice, resume: Option<&'a str>, prompt: &'a str, ) -> Vec<&'a OsStr> { @@ -274,6 +267,7 @@ pub fn claude_args<'a>( .map(OsStr::new) .collect(); args.push(plugin_dir); + args.extend(harness.claude_args().into_iter().map(OsStr::new)); args.extend(["--output-format", "stream-json", "--verbose"].map(OsStr::new)); if let Some(session_id) = resume { args.extend(["--resume", session_id].map(OsStr::new)); diff --git a/src/spec_run.rs b/src/spec_run.rs index 9915374eb..ca12ddbdc 100644 --- a/src/spec_run.rs +++ b/src/spec_run.rs @@ -18,6 +18,7 @@ use crate::child_run::{self, Ended, Handle, Kind}; use crate::delivery::{Delivery, Opening}; use crate::failed_run::FailedRun; use crate::github::{self, Ticket}; +use crate::harness::Choice; use crate::interrupt; use crate::issue::IssueUrl; use crate::progress; @@ -66,7 +67,8 @@ pub fn run( let (spec, base) = (delivery.issue, delivery.base); let mut spec_pr = SpecPr::resume(spec, worktree.branch(), base)?; let base_fix = delivery.base_fix.ask_of_tickets(); - let mut outside = ChildRunsAndGitHub::new(spec, &worktree, &base_fix, &mut spec_pr); + let mut outside = + ChildRunsAndGitHub::new(spec, &worktree, &base_fix, delivery.harness, &mut spec_pr); let (ticket_lines, landed) = land_tickets(&mut outside, tickets, parallel); let ended = match landed { Ok(checklist) => review_and_deliver(worktree, spec_pr, &checklist, delivery), @@ -137,13 +139,15 @@ trait Outside { /// The outside world of the Ticket loop of a Spec run on `spec`, from its /// Spec branch checked out in `worktree`: child `thirdshift` Runs, each of -/// which may start a Base fix if `base_fix` allows one, whose threads send +/// which may start a Base fix if `base_fix` allows one, and runs its sessions +/// on `harness`, whose threads send /// how each ended on `ended`, received from `endings`, GitHub and the Spec /// PR `spec_pr`. struct ChildRunsAndGitHub<'a, 'pr> { spec: &'a IssueUrl, worktree: &'a Worktree, base_fix: &'a BaseFixAsk, + harness: &'a Choice, spec_pr: &'a mut SpecPr<'pr>, ended: Sender<(u64, Result<Ended>)>, endings: Receiver<(u64, Result<Ended>)>, @@ -156,6 +160,7 @@ impl<'a, 'pr> ChildRunsAndGitHub<'a, 'pr> { spec: &'a IssueUrl, worktree: &'a Worktree, base_fix: &'a BaseFixAsk, + harness: &'a Choice, spec_pr: &'a mut SpecPr<'pr>, ) -> Self { let (ended, endings) = mpsc::channel(); @@ -163,6 +168,7 @@ impl<'a, 'pr> ChildRunsAndGitHub<'a, 'pr> { spec, worktree, base_fix, + harness, spec_pr, ended, endings, @@ -181,7 +187,8 @@ impl Outside for ChildRunsAndGitHub<'_, '_> { let kind = Kind::Ticket { spec_branch: self.worktree.branch().to_string(), }; - let child = child_run::start(&self.spec.sibling(number), kind, self.base_fix.clone())?; + let ticket = self.spec.sibling(number); + let child = child_run::start(&ticket, kind, self.base_fix.clone(), self.harness)?; let ended = self.ended.clone(); thread::spawn(move || { let _ = ended.send((number, finish_ticket(number, child))); diff --git a/tests/activity_log.rs b/tests/activity_log.rs index 15e8d9a58..0831d2b6a 100644 --- a/tests/activity_log.rs +++ b/tests/activity_log.rs @@ -85,6 +85,10 @@ fn open_plan(scenario: &Scenario, number: u32) { scenario.issue_labelled(number, &["architect-plan"]); } +/// How a start line names the Harness, Model and Effort when nothing chose +/// them. +const ON_CLAUDE: &str = "on claude · default model · default effort"; + #[test] fn a_run_typed_by_hand_writes_its_start_naming_its_command_log_and_its_end_with_its_outcome() { let scenario = Scenario::new(); @@ -97,7 +101,7 @@ fn a_run_typed_by_hand_writes_its_start_naming_its_command_log_and_its_end_with_ assert_eq!( activity(&scenario), [ - format!("Run #7 started: {command_log}"), + format!("Run #7 started: {command_log}, {ON_CLAUDE}"), format!("Run #7 ended: PR {PR_URL} is ready for review"), ] ); @@ -131,7 +135,10 @@ fn a_spec_run_writes_its_lines_and_its_tickets_write_none() { let command_log = the_one_command_log(&scenario, "issue"); let lines = activity(&scenario); assert_eq!(lines.len(), 2, "{lines:?}"); - assert_eq!(lines[0], format!("Spec run #20 started: {command_log}")); + assert_eq!( + lines[0], + format!("Spec run #20 started: {command_log}, {ON_CLAUDE}") + ); assert!(lines[1].starts_with("Spec run #20 ended: PR "), "{lines:?}"); } @@ -148,7 +155,7 @@ fn a_pickup_run_that_takes_an_issue_names_it_and_the_run_it_dispatched_writes_no assert_eq!( activity(&scenario), [ - format!("Pickup run #7 started: {command_log}"), + format!("Pickup run #7 started: {command_log}, {ON_CLAUDE}"), format!("Pickup run #7 ended: PR {PR_URL} is ready for review"), ] ); @@ -174,7 +181,7 @@ printf 'Architecture review plan: %s\n' "$url" > "$FAKE_CLAUDE_FINAL_MESSAGE" assert_eq!( activity(&scenario), [ - format!("Architect run started: {command_log}"), + format!("Architect run started: {command_log}, {ON_CLAUDE}"), format!( "Architect run ended: plan {} dispatched: PR {PR_URL} is ready for review", scenario.issue_url(8) @@ -255,7 +262,7 @@ fn a_pass_that_works_resets_its_kinds_skips_and_the_other_kinds_lines_do_not() { [ "Pickup run skipped: no Ready issue on acme/widgets".to_string(), lines[1].clone(), - format!("Pickup run #7 started: {command_log}"), + format!("Pickup run #7 started: {command_log}, {ON_CLAUDE}"), format!("Pickup run #7 ended: PR {PR_URL} is ready for review"), "Pickup run skipped: no Ready issue on acme/widgets".to_string(), ] diff --git a/tests/architect_run.rs b/tests/architect_run.rs index e4670fc70..936f0557f 100644 --- a/tests/architect_run.rs +++ b/tests/architect_run.rs @@ -2406,3 +2406,53 @@ fn a_ready_for_agent_ticket_inside_a_spec_does_not_skip_the_architect_run() { ); assert_eq!(scenario.claude_calls().len(), 1); } + +/// The arguments of each call to `claude` that was an agent session, which +/// streams JSON, and of each that was the test call that checks a Model. +fn sessions_and_test_calls(scenario: &Scenario) -> (Vec<Vec<String>>, Vec<Vec<String>>) { + scenario + .claude_calls() + .iter() + .map(|call| { + let args = call["argv"].as_array().unwrap().iter(); + args.map(|arg| arg.as_str().unwrap().to_string()) + .collect::<Vec<_>>() + }) + .partition(|args| args.iter().any(|arg| arg == "--output-format")) +} + +#[test] +fn the_review_and_the_run_it_dispatches_run_on_the_model_checked_once() { + let scenario = scenario(); + // The first call to claude is the test call, the second the review. + scenario.agent_does_in_session(1, "true"); + scenario.agent_does_in_session(2, AGENT_PUBLISHES_A_TICKET); + scenario.agent_does_for(8, &agent_opens_pr(8, "main")); + + let result = scenario.run(&["architect", "effort", "max", "model", "opus"]); + + assert_ended_with_pr(&result, &pr_from(&scenario, "issue-8"), "ready for review"); + let (sessions, test_calls) = sessions_and_test_calls(&scenario); + assert_eq!(test_calls, [["-p", "--model", "opus", "--effort", "max"]]); + assert_eq!(sessions.len(), 2, "the review and #8's: {sessions:?}"); + for args in sessions { + assert!( + args.windows(4) + .any(|window| window == ["--model", "opus", "--effort", "max"]), + "{args:?}" + ); + } +} + +#[test] +fn a_skipped_architect_run_makes_no_check_of_its_harness_or_model() { + let scenario = scenario(); + scenario.issue_is(5, "OPEN"); + scenario.issue_labelled(5, &[ARCHITECT_PLAN]); + scenario.agent_does("echo 'no such model'\nexit 1"); + + let result = scenario.run(&["architect", "model", "Opus 5.5", "harness", "codex"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert!(scenario.claude_calls().is_empty()); +} diff --git a/tests/base_fix.rs b/tests/base_fix.rs index 1d2a08fb7..21ce2d1b3 100644 --- a/tests/base_fix.rs +++ b/tests/base_fix.rs @@ -479,3 +479,39 @@ fn help_names_base_fix_and_not_the_base_fixs_hidden_argument() { result.stdout ); } + +#[test] +fn a_base_fix_runs_its_sessions_on_the_runs_model_and_effort() { + let scenario = Scenario::new(); + // The first call to claude is the test call that checks the Model. + scenario.agent_does_in_session(1, "true"); + scenario.agent_does(RUN_OPENS_PR_WITH_INHERITED_FAILURE); + scenario.agent_does_for(8, &format!("{BASE_FIX_OPENS_PR}{GREEN_ON_HEAD}")); + + let result = scenario.run(&[ + &scenario.issue_url(7), + "base-fix", + "model", + "opus", + "effort", + "low", + ]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let calls = scenario.claude_calls(); + assert_eq!(calls.len(), 3, "the test call, then #7's and #8's sessions"); + for call in &calls[1..] { + let args: Vec<&str> = call["argv"] + .as_array() + .unwrap() + .iter() + .map(|arg| arg.as_str().unwrap()) + .collect(); + assert!( + args.windows(4) + .any(|window| window == ["--model", "opus", "--effort", "low"]), + "{args:?}" + ); + } + assert!(prompts(&scenario)[2].contains(BASE_FIX_URL)); +} diff --git a/tests/command_surface.rs b/tests/command_surface.rs index 8b0e02eb5..44239fc91 100644 --- a/tests/command_surface.rs +++ b/tests/command_surface.rs @@ -88,7 +88,7 @@ fn help_documents_architect_its_focus_the_dispatch_plan_only_and_when_an_archite for mention in [ "<focus> is free text", "dispatches it as thirdshift <Issue URL> would: a Spec run on a Spec, a Run on a single Ticket", - "merge, --no-merge, base-fix, --no-base-fix and parallel <n> apply to that run", + "merge, --no-merge, base-fix, --no-base-fix, parallel <n>, harness, model and effort apply to that run", "as do the User config's defaults", "With --plan-only, the Architect run prints the plan's URL and stops instead", "thirdshift architect \"the Spec run\"", @@ -116,8 +116,8 @@ fn help_documents_architect_its_focus_the_dispatch_plan_only_and_when_an_archite } } -const PLAN_ONLY_DISPATCHES_NOTHING: &str = "merge, no-merge, parallel, base-fix and no-base-fix can't be used with \ - --plan-only: it dispatches no run for them to apply to"; +const PLAN_ONLY_DISPATCHES_NOTHING: &str = "merge, no-merge, parallel, base-fix, no-base-fix, harness, model and \ + effort can't be used with --plan-only: it dispatches no run for them to apply to"; #[test] fn architect_with_arguments_it_cant_use_prints_an_error_and_the_help_to_stderr() { @@ -229,7 +229,7 @@ fn help_documents_pickup_the_ready_issue_the_dispatch_the_skips_and_the_run_noti "A sub-issue is reached through its Spec, when the Spec is itself a Ready issue", "Each ready-for-agent issue a pass looks at and does not take gets one line on stderr with the first reason that applies, such as #21 is a Ticket of #20, which is not ready or #30 blocked by #29, before the line that says what the pass did", "The Pickup run ends as that run does, with its exit code and its PR's URL", - "merge, --no-merge, base-fix, --no-base-fix and parallel <n> apply to that run, as do the User config's defaults", + "merge, --no-merge, base-fix, --no-base-fix, parallel <n>, harness, model and effort apply to that run, as do the User config's defaults", "parallel <n> is ignored when the issue is not a Spec", "base <branch> names the Pickup run's Base branch as it does an Architect run's", "pickup takes nothing else: no focus and no --plan-only", diff --git a/tests/fresh_run.rs b/tests/fresh_run.rs index 9d64fb97f..e1ac519cc 100644 --- a/tests/fresh_run.rs +++ b/tests/fresh_run.rs @@ -259,6 +259,13 @@ fn another_issues_branch_and_pr_do_not_count_as_this_issues() { ); } +/// Every request the Run made to change an issue's labels. +fn label_requests(scenario: &Scenario) -> Vec<Vec<String>> { + let mut requests = scenario.gh_calls_of("api", "--method"); + requests.retain(|request| request.iter().any(|arg| arg.ends_with("/labels"))); + requests +} + #[test] fn a_run_that_ends_with_its_pr_ready_for_review_keeps_its_claim() { let scenario = Scenario::new(); @@ -274,7 +281,7 @@ fn a_run_that_ends_with_its_pr_ready_for_review_keeps_its_claim() { result.stderr ); assert_eq!(scenario.issue_labels(7), ["in-progress"]); - assert_eq!(scenario.gh_calls_of("api", "--method").len(), 1); + assert_eq!(label_requests(&scenario).len(), 1); } #[test] @@ -295,7 +302,7 @@ fn the_claim_is_made_in_one_request_before_the_worktree_is_created() { "stderr: {}", result.stderr ); - let requests = scenario.gh_calls_of("api", "--method"); + let requests = label_requests(&scenario); assert_eq!(requests.len(), 1, "{requests:?}"); assert_eq!( requests[0][2..], diff --git a/tests/harness_choice.rs b/tests/harness_choice.rs new file mode 100644 index 000000000..3ddc4677b --- /dev/null +++ b/tests/harness_choice.rs @@ -0,0 +1,374 @@ +//! The Harness, Model and Effort a Command's sessions run on: chosen by +//! `harness`, `model` and `effort` on the command, else by the User config's +//! `[harness]` section, else Claude Code with its own Model and Effort; +//! checked before any work; and recorded in the Command log, the Activity +//! log, the pull request's body and the Run notification. How they reach a +//! Spec run's Tickets, a Base fix and the run a pass dispatches is covered +//! with each of those. + +mod support; + +use std::fs; + +use serde_json::Value; +use support::resend::ResendStandIn; +use support::{RunResult, Scenario}; + +/// The agent commits its work and opens a PR that closes issue #7. +const AGENT_OPENS_PR: &str = r#" +echo "feature" > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +/// What Claude says, and how it exits, when it refuses the Model it is +/// asked to run on. +const CLAUDE_REFUSES_THE_MODEL: &str = r#" +echo "There's an issue with the selected model (Opus 5.5). It may not exist or you may not have access to it." +exit 1 +"#; + +/// The scenario's repository's logs, relative to the scenario. +const LOGS: &str = "home/.thirdshift/logs/acme/widgets"; + +/// A scenario whose agent opens the PR for #7. The test call that checks a +/// named Model, the first call to `claude` when there is one, runs no script +/// of the agent's: it does nothing and succeeds. +fn scenario() -> Scenario { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + scenario +} + +/// The arguments of each call to `claude` that was an agent session. +fn sessions(scenario: &Scenario) -> Vec<Vec<String>> { + calls(scenario, true) +} + +/// Each call to `claude` that was the test call checking a Model: its +/// arguments and what it was given on stdin. +fn test_calls(scenario: &Scenario) -> Vec<(Vec<String>, String)> { + scenario + .claude_calls() + .into_iter() + .filter(|call| !is_session(call)) + .map(|call| (argv(&call), call["stdin"].as_str().unwrap().to_string())) + .collect() +} + +fn calls(scenario: &Scenario, sessions: bool) -> Vec<Vec<String>> { + scenario + .claude_calls() + .iter() + .filter(|call| is_session(call) == sessions) + .map(argv) + .collect() +} + +fn argv(call: &Value) -> Vec<String> { + call["argv"] + .as_array() + .unwrap() + .iter() + .map(|arg| arg.as_str().unwrap().to_string()) + .collect() +} + +/// Whether `call` was an agent session, which streams JSON, rather than the +/// test call. +fn is_session(call: &Value) -> bool { + argv(call).iter().any(|arg| arg == "--output-format") +} + +/// The value after `flag` in `args`, if `flag` is there. +fn value_of<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + let at = args.iter().position(|arg| arg == flag)?; + args.get(at + 1).map(String::as_str) +} + +/// Assert every agent session ran with `model` and `effort`, each `None` +/// where it should be left to Claude, and that there was at least one. +fn assert_sessions_on(scenario: &Scenario, model: Option<&str>, effort: Option<&str>) { + let sessions = sessions(scenario); + assert!(!sessions.is_empty(), "no session ran"); + for args in sessions { + assert_eq!(value_of(&args, "--model"), model, "{args:?}"); + assert_eq!(value_of(&args, "--effort"), effort, "{args:?}"); + } +} + +/// Assert the Run failed with `error` before any work: no Claim, no +/// worktree, no agent session and no Command log. +fn assert_failed_before_any_work(scenario: &Scenario, result: &RunResult, error: &str) { + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!( + result.stderr.contains(error), + "expected {error:?} in stderr: {}", + result.stderr + ); + assert_eq!(result.stdout, ""); + assert!(sessions(scenario).is_empty(), "a session ran"); + assert_eq!(scenario.entries("work"), vec!["widgets"]); + assert_eq!( + scenario + .launch_git(&["worktree", "list", "--porcelain"]) + .matches("worktree ") + .count(), + 1 + ); + assert!(scenario.issue_labels(7).is_empty(), "the Claim was made"); + assert!( + !scenario.path(&format!("{LOGS}/commands")).exists(), + "a Command log was kept" + ); +} + +/// The PR's body once the Run has ended. +fn pr_body(scenario: &Scenario) -> String { + scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .to_string() +} + +#[test] +fn with_no_flag_and_no_user_config_sessions_run_on_claude_with_no_model_and_no_test_call() { + let scenario = scenario(); + + let result = scenario.run(&[&scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, None, None); + assert!(test_calls(&scenario).is_empty()); + assert!( + pr_body(&scenario).ends_with( + "\n\nBuilt with claude · default model · default effort \ + <!-- thirdshift:built-with -->\n" + ), + "{}", + pr_body(&scenario) + ); +} + +#[test] +fn model_and_effort_on_the_command_reach_every_session_after_one_test_call_with_them() { + let scenario = scenario(); + + let result = scenario.run(&[ + "model", + "claude-opus-5-5", + &scenario.issue_url(7), + "--effort", + "high", + ]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, Some("claude-opus-5-5"), Some("high")); + let test_calls = test_calls(&scenario); + assert_eq!( + test_calls, + [( + ["-p", "--model", "claude-opus-5-5", "--effort", "high"] + .map(String::from) + .to_vec(), + "Reply with OK.".to_string() + )] + ); + assert!( + result.stderr.contains( + "thirdshift: checking the Model claude-opus-5-5 with a test call to claude\n" + ), + "stderr: {}", + result.stderr + ); +} + +#[test] +fn an_effort_alone_makes_no_test_call() { + let scenario = scenario(); + + let result = scenario.run(&[&scenario.issue_url(7), "effort", "max"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, None, Some("max")); + assert!(test_calls(&scenario).is_empty()); +} + +#[test] +fn the_user_config_sets_the_model_and_effort_of_the_harness_it_chooses() { + let scenario = scenario(); + scenario.user_config_is( + "[harness]\ndefault = \"claude\"\n\n[harness.claude]\nmodel = \"opus\"\neffort = \"low\"\n", + ); + + let result = scenario.run(&[&scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, Some("opus"), Some("low")); + assert_eq!(test_calls(&scenario).len(), 1); +} + +#[test] +fn harness_claude_on_the_command_overrides_a_codex_default_and_takes_claudes_settings() { + let scenario = scenario(); + scenario.user_config_is( + "[harness]\ndefault = \"codex\"\n\n\ + [harness.claude]\nmodel = \"opus\"\neffort = \"\"\n\n\ + [harness.codex]\nmodel = \"gpt-6.1-sol\"\neffort = \"max\"\n", + ); + + let result = scenario.run(&["harness", "claude", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, Some("opus"), None); +} + +#[test] +fn a_model_on_the_command_wins_over_the_user_configs() { + let scenario = scenario(); + scenario.user_config_is("[harness.claude]\nmodel = \"opus\"\neffort = \"low\"\n"); + + let result = scenario.run(&[&scenario.issue_url(7), "model", "sonnet"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on(&scenario, Some("sonnet"), Some("low")); +} + +#[test] +fn a_model_claude_refuses_fails_the_run_before_any_work_with_claudes_error() { + let scenario = scenario(); + scenario.agent_does_in_session(1, CLAUDE_REFUSES_THE_MODEL); + + let result = scenario.run(&[&scenario.issue_url(7), "model", "Opus 5.5"]); + + assert_failed_before_any_work( + &scenario, + &result, + "thirdshift: claude refused a test call on the Model Opus 5.5: There's an issue with \ + the selected model (Opus 5.5).", + ); + assert_eq!(test_calls(&scenario).len(), 1); +} + +#[test] +fn a_harness_missing_from_path_fails_the_run_before_any_work_naming_what_chose_it() { + for (config, chosen_by) in [ + (None, "chosen by the default"), + ( + Some("[harness]\ndefault = \"claude\"\n"), + "chosen by harness.default in the User config", + ), + ] { + let scenario = scenario(); + if let Some(config) = config { + scenario.user_config_is(config); + } + fs::remove_file(scenario.path("bin/claude")).unwrap(); + let path = format!("{}:/usr/bin:/bin", scenario.path("bin").display()); + + let result = scenario.run_with_env(&[&scenario.issue_url(7)], &[("PATH", &path)]); + + assert_failed_before_any_work(&scenario, &result, "claude is not on PATH"); + assert!( + result.stderr.contains(chosen_by), + "stderr: {}", + result.stderr + ); + } +} + +#[test] +fn harness_codex_fails_the_run_before_any_work_as_not_supported_yet() { + for (args, config, chosen_by) in [ + (vec!["harness", "codex"], None, "the harness flag"), + ( + vec![], + Some("[harness]\ndefault = \"codex\"\n"), + "harness.default in the User config", + ), + ] { + let scenario = scenario(); + if let Some(config) = config { + scenario.user_config_is(config); + } + let url = scenario.issue_url(7); + let mut args = args; + args.push(&url); + + let result = scenario.run(&args); + + assert_failed_before_any_work( + &scenario, + &result, + &format!("harness codex, chosen by {chosen_by}, is not supported yet"), + ); + assert!(scenario.claude_calls().is_empty()); + } +} + +#[test] +fn the_command_log_the_activity_log_and_the_pr_body_name_the_harness_model_and_effort() { + let scenario = scenario(); + + let result = scenario.run(&[&scenario.issue_url(7), "model", "opus", "effort", "high"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let commands = scenario.entries(&format!("{LOGS}/commands/issue")); + assert_eq!(commands.len(), 1, "{commands:?}"); + let command_log = + fs::read_to_string(scenario.path(&format!("{LOGS}/commands/issue/{}", commands[0]))) + .unwrap(); + let opening: Vec<&str> = command_log.lines().take(4).collect(); + assert!( + opening + .iter() + .any(|line| line.ends_with(" sessions run on claude · opus · high")), + "{command_log}" + ); + let activity = fs::read_to_string(scenario.path(&format!("{LOGS}/activity.log"))).unwrap(); + let started = activity.lines().next().unwrap(); + assert!( + started.ends_with(&format!( + " Run #7 started: commands/issue/{}, on claude · opus · high", + commands[0] + )), + "{activity}" + ); + assert!( + pr_body(&scenario).ends_with( + "Closes #7\n\nBuilt with claude · opus · high <!-- thirdshift:built-with -->\n" + ), + "{}", + pr_body(&scenario) + ); +} + +#[test] +fn the_run_notification_names_the_harness_model_and_effort() { + let scenario = scenario(); + let resend = ResendStandIn::replying(200, r#"{"id":"49a3999c"}"#); + + let result = scenario.run_with_env( + &[ + &scenario.issue_url(7), + "email", + "me@example.com", + "effort", + "max", + ], + &[ + ("THIRDSHIFT_RESEND_URL", resend.url()), + ("RESEND_API_KEY", "re_test_123"), + ], + ); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let requests = resend.requests(); + assert_eq!(requests.len(), 1, "{requests:?}"); + let text = requests[0].body["text"].as_str().unwrap(); + assert!( + text.contains("\nBuilt with claude · default model · max\nHost:"), + "{text}" + ); +} diff --git a/tests/pickup_run.rs b/tests/pickup_run.rs index 2e2ea7f2d..b9f7676fc 100644 --- a/tests/pickup_run.rs +++ b/tests/pickup_run.rs @@ -925,6 +925,7 @@ fn the_notification_of_a_run_left_ready_for_review_is_that_runs() { "Pull request: {}\n\ Session log: {}\n\ Command log: {}\n\ + Built with claude · default model · default effort\n\ Host: ", pr["url"].as_str().unwrap(), log.display(), @@ -1369,3 +1370,73 @@ fn the_sweep_takes_the_label_off_as_the_closed_issue_spells_it() { "repos/acme/widgets/issues/3/labels/In-Progress" ); } + +/// The arguments of each call to `claude` that was an agent session, which +/// streams JSON, and of each that was the test call that checks a Model. +fn sessions_and_test_calls(scenario: &Scenario) -> (Vec<Vec<String>>, Vec<Vec<String>>) { + scenario + .claude_calls() + .iter() + .map(|call| { + let args = call["argv"].as_array().unwrap().iter(); + args.map(|arg| arg.as_str().unwrap().to_string()) + .collect::<Vec<_>>() + }) + .partition(|args| args.iter().any(|arg| arg == "--output-format")) +} + +#[test] +fn the_model_reaches_the_dispatched_run_once_the_pass_has_checked_it() { + let scenario = ready_ticket(); + + let result = scenario.run(&["pickup", "--model", "opus"]); + + assert_ended_with_pr(&result, &pr_from(&scenario, "issue-7"), "ready for review"); + let (sessions, test_calls) = sessions_and_test_calls(&scenario); + assert_eq!(test_calls, [["-p", "--model", "opus"]]); + assert_eq!(sessions.len(), 1, "{sessions:?}"); + assert!( + sessions[0] + .windows(2) + .any(|window| window == ["--model", "opus"]), + "{sessions:?}" + ); +} + +#[test] +fn a_skipped_pass_makes_no_check_of_its_harness_or_model() { + let scenario = Scenario::new(); + scenario.agent_does("echo 'no such model'\nexit 1"); + + let result = scenario.run(&["pickup", "model", "Opus 5.5"]); + + assert_skipped(&scenario, &result, NO_READY_ISSUE); + let result = scenario.run(&["pickup", "harness", "codex"]); + + assert_skipped(&scenario, &result, NO_READY_ISSUE); +} + +#[test] +fn a_model_claude_refuses_fails_the_pass_once_it_has_taken_an_issue_before_any_work() { + let scenario = ready_ticket(); + scenario.agent_does_in_session(1, "echo 'no such model: Opus 5.5'\nexit 1"); + + let result = scenario.run(&["pickup", "model", "Opus 5.5"]); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!( + result.stderr.contains( + "thirdshift: claude refused a test call on the Model Opus 5.5: no such model: Opus 5.5\n" + ), + "stderr: {}", + result.stderr + ); + assert_eq!(scenario.issue_labels(7), [READY_FOR_AGENT]); + assert_eq!(scenario.claude_calls().len(), 1); + assert!( + !scenario + .path("home/.thirdshift/logs/acme/widgets/commands") + .exists(), + "a Command log was kept" + ); +} diff --git a/tests/spec_run.rs b/tests/spec_run.rs index ee50017ee..1096d0567 100644 --- a/tests/spec_run.rs +++ b/tests/spec_run.rs @@ -1066,11 +1066,17 @@ fn the_spec_review_writes_the_spec_pr_body_and_the_checklist_is_put_back_after_i assert_eq!( spec["body"], format!( - "The whole Spec.\n\nUnaddressed findings: none\n\nCloses #20\n\n{DONE_CHECKLIST}\n" + "The whole Spec.\n\nUnaddressed findings: none\n\nCloses #20\n\n{BUILT_WITH}\n\n\ + {DONE_CHECKLIST}\n" ) ); } +/// The line thirdshift writes in the Spec PR's body once the Spec review has +/// written it, when nothing chose the Harness, Model or Effort. +const BUILT_WITH: &str = + "Built with claude · default model · default effort <!-- thirdshift:built-with -->"; + /// The linear Spec's Tickets checklist once every Ticket is done. const DONE_CHECKLIST: &str = "<!-- thirdshift:tickets --> ## Tickets @@ -1093,7 +1099,7 @@ fn a_spec_review_that_keeps_the_markers_has_the_checklist_replaced_between_them( assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); assert_eq!( spec_pr(&scenario)["body"], - format!("Before.\n\n{DONE_CHECKLIST}\n\nAfter. Closes #20") + format!("Before.\n\n{DONE_CHECKLIST}\n\nAfter. Closes #20\n\n{BUILT_WITH}\n") ); } @@ -1126,7 +1132,7 @@ fn a_spec_review_whose_resume_ends_with_killed_background_work_still_delivers_th assert_eq!(spec["isDraft"], false); assert_eq!( spec["body"], - format!("Reviewed.\n\nCloses #20\n\n{DONE_CHECKLIST}\n") + format!("Reviewed.\n\nCloses #20\n\n{BUILT_WITH}\n\n{DONE_CHECKLIST}\n") ); } @@ -2060,3 +2066,40 @@ fn a_merge_run_of_a_spec_ends_with_the_spec_closed_and_its_claim_removed() { assert_eq!(scenario.issue_labels(ticket), ["ready-for-agent"]); } } + +/// The arguments of each call to `claude` that was an agent session, which +/// streams JSON, and of each that was the test call that checks a Model. +fn sessions_and_test_calls(scenario: &Scenario) -> (Vec<Vec<String>>, Vec<Vec<String>>) { + scenario + .claude_calls() + .iter() + .map(|call| { + let args = call["argv"].as_array().unwrap().iter(); + args.map(|arg| arg.as_str().unwrap().to_string()) + .collect::<Vec<_>>() + }) + .partition(|args| args.iter().any(|arg| arg == "--output-format")) +} + +#[test] +fn a_spec_run_checks_its_model_once_and_every_ticket_and_the_spec_review_runs_on_it() { + let scenario = linear_spec(); + + let result = scenario.run(&[&spec_url(&scenario), "model", "opus", "effort", "high"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let (sessions, test_calls) = sessions_and_test_calls(&scenario); + assert_eq!(test_calls.len(), 1, "{test_calls:?}"); + assert_eq!( + sessions.len(), + 3, + "#21, #22 and the Spec review: {sessions:?}" + ); + for args in sessions { + assert!( + args.windows(4) + .any(|window| window == ["--model", "opus", "--effort", "high"]), + "{args:?}" + ); + } +} From 731386a156e1f2ee6a49d9c2c26044f310efa2b2 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 00:23:20 -0400 Subject: [PATCH 04/14] Review clean-ups for the Harness choice (#392) Read [harness.<name>] through Harness::named and Settings::of_mut, keep a User config's Model and Effort as written, move the PR body's Built with line next to the Delivery that writes it, and rewrap doc comments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/args.rs | 10 ++-- src/config.rs | 15 +++--- src/delivery.rs | 120 ++++++++++++++++++++++++++++++-------------- src/harness.rs | 53 +++---------------- src/notification.rs | 7 +-- src/session.rs | 12 ++--- src/spec_run.rs | 5 +- 7 files changed, 114 insertions(+), 108 deletions(-) diff --git a/src/args.rs b/src/args.rs index 4b8a94ca6..61195ec6b 100644 --- a/src/args.rs +++ b/src/args.rs @@ -134,11 +134,11 @@ pub fn parse(args: &[String]) -> Result<Command> { /// Parse the arguments after `architect`: at most one focus, and its flags, /// each at most once, in any order. `base` must be followed by the Base -/// branch. [`PLAN_ONLY`] dispatches nothing, so the -/// flags for the dispatched run, `merge`, `no-merge`, `parallel`, `base-fix`, -/// `no-base-fix`, `harness`, `model` and `effort`, can't go with it. `email` and `no-email` are for the -/// Architect run's own Run notification, and `base` is for the Architecture -/// review too, so they can. None of a Run's flags, nor `base`, +/// branch. [`PLAN_ONLY`] dispatches nothing, so the flags for the dispatched +/// run, `merge`, `no-merge`, `parallel`, `base-fix`, `no-base-fix`, +/// `harness`, `model` and `effort`, can't go with it. `email` and `no-email` +/// are for the Architect run's own Run notification, and `base` is for the +/// Architecture review too, so they can. None of a Run's flags, nor `base`, /// is ever the focus, and any other argument that starts with a dash is /// unexpected rather than a focus. fn parse_architect(args: &[String]) -> Result<ArchitectArgs> { diff --git a/src/config.rs b/src/config.rs index 570effded..301e29ce5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -165,15 +165,12 @@ impl UserConfig { ("harness", "default", _) => { bail!("harness.default must be a quoted name in {file}") } - ("harness", "claude" | "codex", Value::Table(settings)) => { - let set = match key.as_str() { - "claude" => &mut config.harness.claude, - _ => &mut config.harness.codex, + ("harness", name, value) if Harness::named(name).is_some() => { + let Value::Table(settings) = value else { + bail!("harness.{key} must be the section [harness.{key}] in {file}") }; - *set = model_and_effort(settings, key, &file)?; - } - ("harness", "claude" | "codex", _) => { - bail!("harness.{key} must be the section [harness.{key}] in {file}") + let harness = Harness::named(name).expect("a Harness's name"); + *config.harness.of_mut(harness) = model_and_effort(settings, key, &file)?; } _ => bail!("unknown key {section}.{key} in {file}"), } @@ -681,7 +678,7 @@ fn model_and_effort(settings: &Table, harness: &str, file: &str) -> Result<Model "harness.{harness}.{key} must be a quoted string, blank for {harness}'s own default, in {file}" ); }; - *setting = Some(value.trim().to_string()).filter(|value| !value.is_empty()); + *setting = Some(value.clone()).filter(|value| !value.is_empty()); } Ok(set) } diff --git a/src/delivery.rs b/src/delivery.rs index 173fe6d74..f8cdfcde1 100644 --- a/src/delivery.rs +++ b/src/delivery.rs @@ -21,7 +21,7 @@ use crate::base_fix::BaseFix; use crate::ci::{self, Ci, FailedChecks}; use crate::failed_run::{FailedRun, PolicyRefusal, interrupted_or}; use crate::github::{self, Mergeable, PullRequest}; -use crate::harness::{self, Choice}; +use crate::harness::Choice; use crate::host; use crate::interrupt; use crate::issue::IssueUrl; @@ -65,17 +65,16 @@ impl Delivery<'_> { /// Take the pull request from the branch checked out in `worktree` to the /// goal. With `opening.catch_up_from_origin`, first fast-forward the /// branch to origin. Then run the opening session, push the branch, for - /// any commit the session left unpushed, write the line that says what - /// it was built with in the pull request's body, only warning if that - /// fails, run `before_ready`, and mark the - /// pull request ready, failing unless it exists, is open and targets the - /// Base branch. Then keep it mergeable and its CI green through the - /// Repair loop, and for [`Goal::Merged`], Self-merge it. A merge that - /// fails goes back round the Repair loop and is tried again on the new - /// head; if that round finds nothing to fix, this fails with a - /// `PolicyRefusal`. Any failure goes through the Failed run path, which - /// keeps the worktree if its work did not reach origin; otherwise it is - /// cleaned up when this returns. + /// any commit the session left unpushed, write the line that says what it + /// was built with in the pull request's body, only warning if that fails, + /// run `before_ready`, and mark the pull request ready, failing unless it + /// exists, is open and targets the Base branch. Then keep it mergeable + /// and its CI green through the Repair loop, and for [`Goal::Merged`], + /// Self-merge it. A merge that fails goes back round the Repair loop and + /// is tried again on the new head; if that round finds nothing to fix, + /// this fails with a `PolicyRefusal`. Any failure goes through the Failed + /// run path, which keeps the worktree if its work did not reach origin; + /// otherwise it is cleaned up when this returns. pub fn deliver( self, worktree: Worktree, @@ -88,30 +87,28 @@ impl Delivery<'_> { branch: worktree.branch(), goal: self.goal, }; - let sessions_on = self.harness; - let (delivered, log) = - Sessions::within(self.logs, worktree.path(), sessions_on, |sessions| { - let mut outside = InWorktree { - issue: self.issue, - worktree: &worktree, - base: self.base, - base_fix: self.base_fix, - sessions, - }; - route.steps(&mut outside, &opening, |_| { - if let Err(error) = write_built_with(self.issue, worktree.branch(), sessions_on) - { - progress::warn( - &error, - format_args!( - "could not write \"{}\" in the pull request's body", - sessions_on.built_with() - ), - ); - } - before_ready() - }) - }); + let harness = self.harness; + let (delivered, log) = Sessions::within(self.logs, worktree.path(), harness, |sessions| { + let mut outside = InWorktree { + issue: self.issue, + worktree: &worktree, + base: self.base, + base_fix: self.base_fix, + sessions, + }; + route.steps(&mut outside, &opening, |_| { + if let Err(error) = write_built_with(self.issue, worktree.branch(), harness) { + progress::warn( + &error, + format_args!( + "could not write \"{}\" in the pull request's body", + harness.built_with() + ), + ); + } + before_ready() + }) + }); match delivered { Ok(pr_url) => Ok(Reached { pr_url, @@ -141,7 +138,7 @@ fn write_built_with(issue: &IssueUrl, branch: &str, harness: &Choice) -> Result< return Ok(()); }; let body = github::pr_body(issue, pr.number)?; - let written = harness::with_built_with(&body, harness); + let written = with_built_with(&body, harness); if written != body { progress::step(format_args!( "writing \"{}\" in the pull request's body", @@ -152,6 +149,33 @@ fn write_built_with(issue: &IssueUrl, branch: &str, harness: &Choice) -> Result< Ok(()) } +/// The marker that ends the line thirdshift writes in a pull request's body, +/// so it can be found and replaced. +const BUILT_WITH_MARKER: &str = "<!-- thirdshift:built-with -->"; + +/// `body` with `choice`'s line in it: in place of the one thirdshift wrote +/// before, or else added at the end. +pub fn with_built_with(body: &str, choice: &Choice) -> String { + let line = format!("{} {BUILT_WITH_MARKER}", choice.built_with()); + if body.contains(BUILT_WITH_MARKER) { + return body + .split_inclusive('\n') + .map(|old| { + if old.trim_end().ends_with(BUILT_WITH_MARKER) { + let end = &old[old.trim_end_matches(['\r', '\n']).len()..]; + format!("{line}{end}") + } else { + old.to_string() + } + }) + .collect(); + } + if body.trim().is_empty() { + return format!("{line}\n"); + } + format!("{}\n\n{line}\n", body.trim_end()) +} + /// The steps of a Delivery of the pull request for `issue`, from `branch` /// into the Base branch `base`, to `goal`. struct Route<'a> { @@ -1425,4 +1449,26 @@ mod tests { Some(Path::new("/logs/7-implement.jsonl")) ); } + + #[test] + fn the_built_with_line_is_added_to_a_body_and_replaces_the_one_written_before() { + let opus = Choice { + model: Some("opus".to_string()), + effort: Some("high".to_string()), + ..Choice::default() + }; + let line = "Built with claude · opus · high <!-- thirdshift:built-with -->"; + + let added = with_built_with("Adds a button.\n\nCloses #7\n", &opus); + assert_eq!(added, format!("Adds a button.\n\nCloses #7\n\n{line}\n")); + assert_eq!(with_built_with("", &opus), format!("{line}\n")); + + let old = "Adds a button.\n\nBuilt with claude · default model · default effort \ + <!-- thirdshift:built-with -->\n\nCloses #7\n"; + assert_eq!( + with_built_with(old, &opus), + format!("Adds a button.\n\n{line}\n\nCloses #7\n") + ); + assert_eq!(with_built_with(&added, &opus), added); + } } diff --git a/src/harness.rs b/src/harness.rs index cbccd93fe..616c8e760 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -69,6 +69,14 @@ impl Settings { Harness::Codex => &self.codex, } } + + /// The Model and Effort set for `harness`, to set them. + pub fn of_mut(&mut self, harness: Harness) -> &mut ModelAndEffort { + match harness { + Harness::Claude => &mut self.claude, + Harness::Codex => &mut self.codex, + } + } } /// What a command asked for, each none if it said nothing about it. @@ -260,33 +268,6 @@ fn on_path(cli: &str) -> bool { }) } -/// The marker that ends the line thirdshift writes in a pull request's body, -/// so it can be found and replaced. -const BUILT_WITH_MARKER: &str = "<!-- thirdshift:built-with -->"; - -/// `body` with `choice`'s line in it: in place of the one thirdshift wrote -/// before, or else added at the end. -pub fn with_built_with(body: &str, choice: &Choice) -> String { - let line = format!("{} {BUILT_WITH_MARKER}", choice.built_with()); - if body.contains(BUILT_WITH_MARKER) { - return body - .split_inclusive('\n') - .map(|old| { - if old.trim_end().ends_with(BUILT_WITH_MARKER) { - let end = &old[old.trim_end_matches(['\r', '\n']).len()..]; - format!("{line}{end}") - } else { - old.to_string() - } - }) - .collect(); - } - if body.trim().is_empty() { - return format!("{line}\n"); - } - format!("{}\n\n{line}\n", body.trim_end()) -} - #[cfg(test)] mod tests { use super::*; @@ -436,22 +417,4 @@ mod tests { assert_eq!(chosen.built_with(), line); } } - - #[test] - fn the_built_with_line_is_added_to_a_body_and_replaces_the_one_written_before() { - let opus = choice(Harness::Claude, "opus", "high", ChosenBy::Command); - let line = "Built with claude · opus · high <!-- thirdshift:built-with -->"; - - let added = with_built_with("Adds a button.\n\nCloses #7\n", &opus); - assert_eq!(added, format!("Adds a button.\n\nCloses #7\n\n{line}\n")); - assert_eq!(with_built_with("", &opus), format!("{line}\n")); - - let old = "Adds a button.\n\nBuilt with claude · default model · default effort \ - <!-- thirdshift:built-with -->\n\nCloses #7\n"; - assert_eq!( - with_built_with(old, &opus), - format!("Adds a button.\n\n{line}\n\nCloses #7\n") - ); - assert_eq!(with_built_with(&added, &opus), added); - } } diff --git a/src/notification.rs b/src/notification.rs index 56b1f2246..6a612d702 100644 --- a/src/notification.rs +++ b/src/notification.rs @@ -186,9 +186,10 @@ fn architect_subject(repo: Option<&Repo>, outcome: &str) -> String { /// The plain-text body of the notification for a command that ended as /// `account` tells, with the line that says what its sessions were /// `built_with`, if known, naming `command_log`, if the command keeps one, -/// and `host`, for a command that `took` so long. An Architect run's starts with -/// how its review ended, and the outcome of the run it dispatched, if any. -/// The cause is left out of an interrupted command's, whose outcome says so. +/// and `host`, for a command that `took` so long. An Architect run's starts +/// with how its review ended, and the outcome of the run it dispatched, if +/// any. The cause is left out of an interrupted command's, whose outcome says +/// so. fn body( account: &Account, built_with: Option<&str>, diff --git a/src/session.rs b/src/session.rs index 65d5c7165..75691f7ee 100644 --- a/src/session.rs +++ b/src/session.rs @@ -77,12 +77,12 @@ pub struct Sessions<'a> { impl<'a> Sessions<'a> { /// Write the Factory skills plugin, then take `steps`, which run their - /// sessions on `harness` through the `Sessions` they are given. Returns what `steps` - /// came to, with the most recent session's log, if a session created it. - /// If a session's last ending, its Resume's if it got one, left - /// background work to be killed and `steps` then fail, the failure names - /// that work ahead of its own cause: the session may have stopped short - /// of its job. The plugin directory is gone when this returns. + /// sessions on `harness` through the `Sessions` they are given. Returns + /// what `steps` came to, with the most recent session's log, if a session + /// created it. If a session's last ending, its Resume's if it got one, + /// left background work to be killed and `steps` then fail, the failure + /// names that work ahead of its own cause: the session may have stopped + /// short of its job. The plugin directory is gone when this returns. pub fn within<T>( logs: &'a Logs, worktree: &'a Path, diff --git a/src/spec_run.rs b/src/spec_run.rs index ca12ddbdc..65bf0e9ec 100644 --- a/src/spec_run.rs +++ b/src/spec_run.rs @@ -140,9 +140,8 @@ trait Outside { /// The outside world of the Ticket loop of a Spec run on `spec`, from its /// Spec branch checked out in `worktree`: child `thirdshift` Runs, each of /// which may start a Base fix if `base_fix` allows one, and runs its sessions -/// on `harness`, whose threads send -/// how each ended on `ended`, received from `endings`, GitHub and the Spec -/// PR `spec_pr`. +/// on `harness`, whose threads send how each ended on `ended`, received from +/// `endings`, GitHub and the Spec PR `spec_pr`. struct ChildRunsAndGitHub<'a, 'pr> { spec: &'a IssueUrl, worktree: &'a Worktree, From ee85338c7fe6aa7d49135e21e9e86b90439245a2 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 00:55:53 -0400 Subject: [PATCH 05/14] Setup asks for the Harness, Model and Effort (#393) Setup's first questions choose the Harness, listing both and refusing one sessions can't run on here, then the Model and Effort for it, a Claude Model checked with the test call a Run makes. The User config gains [harness], [harness.claude] and [harness.codex] with every key, and completing an existing one now recurses into subsections. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- README.md | 22 ++- src/config.rs | 189 ++++++++++++++++----- src/harness.rs | 58 ++++++- src/questions.rs | 92 +++++++++++ tests/setup.rs | 383 +++++++++++++++++++++++++++++++++++++++++-- tests/setup_offer.rs | 31 +++- 6 files changed, 705 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index cdf7c7583..f706b9176 100644 --- a/README.md +++ b/README.md @@ -242,15 +242,27 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 [pickup] limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 + +[harness] +default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude + +[harness.claude] +model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own +effort = "" # how hard that Model reasons; default blank, for Claude Code's own + +[harness.codex] +model = "" # the Model Codex's sessions run on; default blank, for Codex's own +effort = "" # how hard that Model reasons; default blank, for Codex's own ``` Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. Every Run a Merge run? (`merge.always`) -2. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. -3. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) -4. Run notifications? (`email.always`). If yes, the address (`email.to`), asked again until it has an `@`, then the sender (`email.from`). -5. With notifications on, the Resend API key, with input hidden. With none saved it asks `Resend API key (input hidden, Enter to skip):`; with one in the [Credentials](#email), `Resend API key (input hidden, Enter keeps the saved one):`, and a new one replaces it. Surrounding spaces are trimmed, and anything that doesn't start with `re_` is asked again. A key you give is saved in the Credentials, `~/.thirdshift/credentials.toml`, created with mode 600 (and `~/.thirdshift` with it) or edited in place, keeping its comments and anything else in it and changing only `resend.key`; `setup` then prints `wrote the Credentials <path>`. The key is never printed, nor written to the User config. Skipping writes no Credentials and says how to add a key later: rerun `thirdshift setup`, or set `RESEND_API_KEY`. With `RESEND_API_KEY` set and not empty, which wins over the Credentials, nothing is asked, and it says the key comes from `RESEND_API_KEY`. With a key found or given, it offers to send a test email (default No), as `thirdshift email-test` does, once the files are written. +1. The Harness for every Run's sessions (`harness.default`), listing both, each marked where sessions can't run on it here: `claude (not installed)` when `claude` isn't on `PATH`, and `codex (not supported yet)` until Codex sessions are. A marked Harness, or a name that isn't one, is refused and asked again. The default is the User config's Harness if sessions can run on it, else `claude`. Then the Model (`harness.<name>.model`) and the Effort (`harness.<name>.effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. A named Model gets the test call a Run makes (see [Harness, Model and Effort](#harness-model-and-effort)); if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. The other Harness's section is left as it is. With no Harness sessions can run on here, Setup says so, asks none of this, and leaves the harness settings as they are. +2. Every Run a Merge run? (`merge.always`) +3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. +4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) +5. Run notifications? (`email.always`). If yes, the address (`email.to`), asked again until it has an `@`, then the sender (`email.from`). +6. With notifications on, the Resend API key, with input hidden. With none saved it asks `Resend API key (input hidden, Enter to skip):`; with one in the [Credentials](#email), `Resend API key (input hidden, Enter keeps the saved one):`, and a new one replaces it. Surrounding spaces are trimmed, and anything that doesn't start with `re_` is asked again. A key you give is saved in the Credentials, `~/.thirdshift/credentials.toml`, created with mode 600 (and `~/.thirdshift` with it) or edited in place, keeping its comments and anything else in it and changing only `resend.key`; `setup` then prints `wrote the Credentials <path>`. The key is never printed, nor written to the User config. Skipping writes no Credentials and says how to add a key later: rerun `thirdshift setup`, or set `RESEND_API_KEY`. With `RESEND_API_KEY` set and not empty, which wins over the Credentials, nothing is asked, and it says the key comes from `RESEND_API_KEY`. With a key found or given, it offers to send a test email (default No), as `thirdshift email-test` does, once the files are written. Pressing Enter takes the default shown, which is the file's current value, or else the setting's default, and for the address the suggested email above. `logs.dir`, `activity.quiet_skips`, `spec.parallel` and `pickup.limit` are not asked about. The answers are written like everything else below: in place, keeping your comments. Ctrl-C during the questions, the key included, writes nothing: neither the User config nor the Credentials. With notifications off, nothing about a key is asked, and saved Credentials stay as they were, so `--email` on a single Run still works. Credentials a Run would refuse (see [Email](#email)) are refused before any question, exit `1`, and not touched. With no terminal, as from cron or `thirdshift setup </dev/null`, `setup` asks nothing and never writes the Credentials. Either way it prints the file's path on stderr and exits `0` with stdout empty. Over a User config that is already there, `setup` edits it in place: its comments and key order stay, as do the values it didn't ask about, and each key it lacks is added at its default with its comment, so afterwards the file lists every setting this version knows. One that already does, down to the commented-out `email.to` line, is left byte for byte as it was. A key added to an inline table, such as `launch = { pull = true }`, gets no comment, since TOML has no place for one there. One a Run would refuse is refused the same way, exit `1`, and not touched. Any argument after `setup` is an argument error (exit `2`). diff --git a/src/config.rs b/src/config.rs index 301e29ce5..04a766377 100644 --- a/src/config.rs +++ b/src/config.rs @@ -367,17 +367,28 @@ fn with_answers(text: &str, answers: &Answers) -> Result<String> { set(&mut document, "email", "from", notifications.from.as_str()); set_email_to(&mut document, ¬ifications.to); } + if let Some((harness, chosen)) = &answers.harness { + set(&mut document, "harness", "default", harness.name()); + let section = format!("harness.{}", harness.name()); + for (key, value) in [("model", &chosen.model), ("effort", &chosen.effort)] { + set(&mut document, §ion, key, value.as_deref().unwrap_or("")); + } + } Ok(document.to_string()) } /// Set `section.key`, which `document` holds, to `value`, keeping the /// spacing and comment around the old value, and the comment in the same /// column where the spaces before it allow. An equal value is left as it was -/// written. +/// written. `section` may be a subsection, as `harness.claude`. fn set(document: &mut DocumentMut, section: &str, key: &str, value: impl Into<toml_edit::Value>) { let value = value.into(); - let old = document[section] - .as_table_like_mut() + let old = section + .split('.') + .try_fold(document.as_item_mut(), |item, name| { + item.as_table_like_mut()?.get_mut(name) + }) + .and_then(Item::as_table_like_mut) .and_then(|settings| settings.get_mut(key)) .and_then(Item::as_value_mut) .unwrap_or_else(|| panic!("a completed User config has {section}.{key}")); @@ -475,46 +486,23 @@ pub fn replace(path: &Path, text: &str) -> std::io::Result<()> { /// `text`, a User config a Run accepts, with each key it lacks added at its /// default, as `DEFAULTS` writes it. Everything already in `text` stays as /// it was: a missing key goes at the end of its section, and a missing -/// section after the last line of `text`. A key added to an inline table +/// section, or a key of a section `text` names only in its subsections' +/// headers, after the last line of `text`. A key added to an inline table /// gets no comment, as TOML has no place for one there. fn complete(text: &str) -> Result<String> { let mut document: DocumentMut = text.parse().context("can't parse the User config")?; let defaults: DocumentMut = DEFAULTS.parse().expect("DEFAULTS is valid TOML"); let mut missing = DocumentMut::new(); - for (section, default_settings) in defaults.iter() { - let default_settings = default_settings - .as_table() - .expect("every key in DEFAULTS is in a section"); - match document.get_mut(section) { - None => { - missing.insert(section, Item::Table(default_settings.clone())); - } - Some(Item::Table(settings)) => { - for (key, item) in default_settings.iter() { - if !settings.contains_key(key) { - let mut key = default_settings - .key(key) - .expect("the key is in DEFAULTS") - .clone(); - key.leaf_decor_mut().set_prefix(""); - settings.insert_formatted(&key, item.clone()); - } - } - if section == "email" && !settings.is_dotted() { - note_email_to(settings, default_settings, text); - } - } - Some(Item::Value(toml_edit::Value::InlineTable(settings))) => { - for (key, item) in default_settings.iter() { - if !settings.contains_key(key) { - let mut value = item.as_value().expect("DEFAULTS has only values").clone(); - value.decor_mut().clear(); - settings.insert(key, value); - } - } - } - Some(_) => unreachable!("a Run refuses {section} that isn't a section"), - } + complete_table( + document.as_table_mut(), + defaults.as_table(), + missing.as_table_mut(), + ); + if let Some(Item::Table(email)) = document.get_mut("email") + && !email.is_dotted() + { + let defaults = defaults["email"].as_table().expect("DEFAULTS has [email]"); + note_email_to(email, defaults, text); } let mut completed = document.to_string(); let missing = missing.to_string(); @@ -529,6 +517,78 @@ fn complete(text: &str) -> Result<String> { Ok(completed) } +/// Add to `settings`, a table of a User config, each key of `defaults`, the +/// same table in `DEFAULTS`, that it lacks, and the same for each of its +/// subsections. A missing key goes at the end of `settings`, unless +/// `settings` has no header of its own, as `[harness]` with only +/// `[harness.claude]` written; then it goes, as does a missing subsection, in +/// `missing`, the same table of what is written after the file. +fn complete_table( + settings: &mut toml_edit::Table, + defaults: &toml_edit::Table, + missing: &mut toml_edit::Table, +) { + let implicit = settings.is_implicit() && !settings.is_dotted(); + for (key, item) in defaults.iter() { + let default_key = defaults.key(key).expect("the key is in DEFAULTS"); + match (item, settings.get_mut(key)) { + (Item::Table(default_settings), Some(Item::Table(settings))) => { + let missing = missing.entry(key).or_insert_with(|| { + let mut table = toml_edit::Table::new(); + table.set_implicit(true); + table.set_position(default_settings.position()); + Item::Table(table) + }); + let missing = missing + .as_table_mut() + .expect("a missing section is a table"); + complete_table(settings, default_settings, missing); + } + ( + Item::Table(default_settings), + Some(Item::Value(toml_edit::Value::InlineTable(settings))), + ) => complete_inline(settings, default_settings), + (Item::Table(_), Some(_)) => unreachable!("a Run refuses {key} that isn't a section"), + (Item::Table(_), None) => { + missing.insert_formatted(default_key, item.clone()); + } + (_, Some(_)) => {} + (_, None) if implicit => { + missing.set_implicit(false); + missing.insert_formatted(default_key, item.clone()); + } + (_, None) => { + let mut key = default_key.clone(); + key.leaf_decor_mut().set_prefix(""); + settings.insert_formatted(&key, item.clone()); + } + } + } +} + +/// Add to `settings`, an inline table of a User config, each key of +/// `defaults`, the same table in `DEFAULTS`, that it lacks, with no comment. +fn complete_inline(settings: &mut toml_edit::InlineTable, defaults: &toml_edit::Table) { + for (key, item) in defaults.iter() { + match (item, settings.get_mut(key)) { + (Item::Table(defaults), Some(toml_edit::Value::InlineTable(settings))) => { + complete_inline(settings, defaults) + } + (_, Some(_)) => {} + (Item::Table(defaults), None) => { + let mut table = toml_edit::InlineTable::new(); + complete_inline(&mut table, defaults); + settings.insert(key, toml_edit::Value::InlineTable(table)); + } + (_, None) => { + let mut value = item.as_value().expect("DEFAULTS has only values").clone(); + value.decor_mut().clear(); + settings.insert(key, value); + } + } + } +} + /// With no `email.to` in `email`, the `[email]` section of `text`, and no /// commented-out one either, add the commented-out example line from /// `defaults`, the `[email]` section of `DEFAULTS`, just before `email.from`: @@ -620,6 +680,17 @@ parallel = 3 # how many Tickets a Spec run runs at once; default 3 [pickup] limit = 3 # how many open issues labelled in-progress stop a Pickup run taking another; default 3 + +[harness] +default = "claude" # the Harness every Run's sessions run on, claude or codex; default claude + +[harness.claude] +model = "" # the Model Claude Code's sessions run on; default blank, for Claude Code's own +effort = "" # how hard that Model reasons; default blank, for Claude Code's own + +[harness.codex] +model = "" # the Model Codex's sessions run on; default blank, for Codex's own +effort = "" # how hard that Model reasons; default blank, for Codex's own "#; /// The line `DEFAULTS` holds for `email.to`, which has no default. @@ -730,6 +801,8 @@ mod tests { Some(crate::email::DEFAULT_FROM) ); config.email.from = None; + assert_eq!(config.harness.default, Some(Harness::Claude)); + config.harness.default = None; assert_eq!(config, UserConfig::defaults(Path::new("/home/me"))); } @@ -994,7 +1067,8 @@ mod tests { } /// `text` completed, after checking a Run reads it with the settings it - /// had, plus the defaults for the keys it lacked. + /// had, plus the defaults for the keys it lacked, which change nothing a + /// Run does. fn completed(text: &str) -> String { let completed = complete(text).unwrap(); let before = parse(text).unwrap(); @@ -1006,6 +1080,10 @@ mod tests { ); after.email.from = None; } + if before.harness.default.is_none() { + assert_eq!(after.harness.default, Some(Harness::Claude)); + after.harness.default = None; + } assert_eq!(after, before, "{completed}"); let again = complete(&completed).unwrap(); assert_eq!(again, completed, "completing twice changed it"); @@ -1044,6 +1122,38 @@ mod tests { } } + #[test] + fn completing_adds_each_harness_key_a_user_config_lacks() { + for text in [ + "[harness]\ndefault = \"claude\"\n", + "[harness.claude]\nmodel = \"opus\" # mine\n", + "[harness.codex]\neffort = \"max\"\n\n[merge]\nalways = true\n", + "[harness]\n\n[harness.claude]\neffort = \"high\"\n", + "harness.default = \"claude\"\nharness.claude.model = \"opus\"\n", + "harness = { claude = { model = \"opus\" } }\n", + "[harness]\nclaude = { effort = \"low\" }\n", + ] { + let completed = completed(text); + let config: toml::Table = completed.parse().unwrap(); + let harness = config["harness"].as_table().unwrap(); + assert!(harness.contains_key("default"), "{text:?}:\n{completed}"); + for name in ["claude", "codex"] { + let set = harness[name].as_table().unwrap(); + for key in ["model", "effort"] { + assert!(set.contains_key(key), "{text:?}:\n{completed}"); + } + } + // An inline table gains its missing keys on its own line. + let mut lines = completed.lines(); + for line in text.lines().filter(|line| !line.contains('{')) { + assert!( + lines.any(|kept| kept == line), + "{line:?} of {text:?} is lost or moved:\n{completed}" + ); + } + } + } + #[test] fn completing_adds_email_to_commented_out_once() { for text in [ @@ -1066,6 +1176,7 @@ mod tests { fn notifications_to(to: &str) -> Answers { Answers { + harness: None, merge_always: false, base_fix: false, launch_pull: false, diff --git a/src/harness.rs b/src/harness.rs index 616c8e760..c1f56bf77 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -37,6 +37,40 @@ impl Harness { Harness::Codex => "codex", } } + + /// Every Harness, in the order Setup lists them. + pub const ALL: [Harness; 2] = [Harness::Claude, Harness::Codex]; + + /// Why sessions can't run on it on this machine, if they can't: Codex is + /// not supported yet, and otherwise its CLI must be on `PATH`. + pub fn unavailable(self) -> Option<Unavailable> { + if self == Harness::Codex { + Some(Unavailable::NotSupportedYet) + } else if !on_path(self.name()) { + Some(Unavailable::NotInstalled) + } else { + None + } + } +} + +/// Why sessions can't run on a Harness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Unavailable { + /// thirdshift can't run its sessions yet. + NotSupportedYet, + /// Its CLI isn't on `PATH`. + NotInstalled, +} + +impl fmt::Display for Unavailable { + /// As Setup marks a Harness with it, as in `codex (not supported yet)`. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(match self { + Unavailable::NotSupportedYet => "not supported yet", + Unavailable::NotInstalled => "not installed", + }) + } } /// The names a Harness is chosen by, for the messages that list them. @@ -63,7 +97,7 @@ pub struct Settings { impl Settings { /// The Model and Effort set for `harness`. - fn of(&self, harness: Harness) -> &ModelAndEffort { + pub fn of(&self, harness: Harness) -> &ModelAndEffort { match harness { Harness::Claude => &self.claude, Harness::Codex => &self.codex, @@ -170,20 +204,26 @@ impl Choice { /// Each failure says what to change. pub fn check(&self) -> Result<()> { let cli = self.harness.name(); - if self.harness == Harness::Codex { - bail!( - "harness codex, chosen by {}, is not supported yet: Codex sessions come in a \ + match self.harness.unavailable() { + Some(Unavailable::NotSupportedYet) => bail!( + "harness {cli}, chosen by {}, is not supported yet: Codex sessions come in a \ later thirdshift; choose harness claude", self.chosen_by - ); - } - if !on_path(cli) { - bail!( + ), + Some(Unavailable::NotInstalled) => bail!( "{cli} is not on PATH, and the Harness {cli} is chosen by {}: install it, \ or choose another Harness", self.chosen_by - ); + ), + None => {} } + self.test_call() + } + + /// Check that Claude takes a minimal test call on the Model, if one is + /// named, with the Effort, if any. A refusal says what Claude said. + pub fn test_call(&self) -> Result<()> { + let cli = self.harness.name(); let Some(model) = &self.model else { return Ok(()); }; diff --git a/src/questions.rs b/src/questions.rs index baf80a117..25e6bcbcb 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -13,10 +13,15 @@ use signal_hook::low_level; use crate::config::UserConfig; use crate::email::DEFAULT_FROM; +use crate::harness::{self, Choice, ChosenBy, Harness, ModelAndEffort, Unavailable}; use crate::resend_key::{self, Credentials, Source}; /// What the user chose. pub struct Answers { + /// `harness.default`, and the Model and Effort of the Harness it names, + /// each none for the Harness's own default; `None` when sessions can run + /// on no Harness here, and the harness settings stay as they were. + pub harness: Option<(Harness, ModelAndEffort)>, /// `merge.always`. pub merge_always: bool, /// `base.fix`: asked only with `merge.always` on, and otherwise its @@ -76,6 +81,7 @@ pub fn offer(path: &Path) -> Result<bool> { /// Ask the Setup questions, with the settings in `current` as the default /// answers, and `suggested_address` for `email.to` when `current` has none. +/// The Harness, Model and Effort come first, as [`ask_harness`] asks them. /// Base fixes are asked about only with every Run a Merge run; otherwise /// the answer is `base.fix`'s default, no. /// The address is re-asked until it has an `@`. With Run notifications on, @@ -87,6 +93,7 @@ pub fn ask( credentials: &Credentials, suggested_address: impl FnOnce() -> Option<String>, ) -> Result<Answers> { + let harness = ask_harness(¤t.harness)?; let merge_always = yes_or_no("Every Run a Merge run?", current.merge_always)?; let base_fix = merge_always && yes_or_no( @@ -102,6 +109,7 @@ pub fn ask( current.email.always, )? { return Ok(Answers { + harness, merge_always, base_fix, launch_pull, @@ -138,6 +146,7 @@ pub fn ask( false }; Ok(Answers { + harness, merge_always, base_fix, launch_pull, @@ -150,6 +159,89 @@ pub fn ask( }) } +/// Ask which Harness every Run's sessions run on, listing each with why +/// sessions can't run on it here, if they can't, and refusing that one; then +/// its Model and Effort, with those `current` sets for it as the defaults. +/// The Harness's default is `current`'s, if sessions can run on it, else the +/// first one they can run on. A Model is checked with a test call, as a Run +/// checks it, and on a refusal the Model and Effort are asked again. With no +/// Harness to choose, nothing is asked, and the answer is `None`. +fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndEffort)>> { + let listed: Vec<String> = Harness::ALL + .iter() + .map(|harness| match harness.unavailable() { + Some(why) => format!("{} ({why})", harness.name()), + None => harness.name().to_string(), + }) + .collect(); + let listed = listed.join(" or "); + let available = |harness: &Harness| harness.unavailable().is_none(); + let default = current + .default + .filter(available) + .or_else(|| Harness::ALL.into_iter().find(available)); + let Some(default) = default else { + say(&format!( + "Harness for every Run's sessions: {listed}. Sessions can run on neither here, so \ + the harness settings stay as they are; install claude, then rerun `thirdshift setup`." + )); + return Ok(None); + }; + let harness = loop { + let name = answer( + &format!("Harness for every Run's sessions, {listed}"), + Some(default.name()), + )?; + match Harness::named(&name) { + None => say(&format!("Choose {}.", harness::NAMES)), + Some(harness) => match harness.unavailable() { + None => break harness, + Some(Unavailable::NotSupportedYet) => say(&format!( + "{name} is not supported yet: Codex sessions come in a later thirdshift." + )), + Some(Unavailable::NotInstalled) => { + say(&format!("{name} is not installed: it isn't on PATH.")) + } + }, + } + }; + let set = current.of(harness); + loop { + let model = ask_setting("Model", harness, set.model.as_deref())?; + let effort = ask_setting("Effort", harness, set.effort.as_deref())?; + let choice = Choice { + harness, + model, + effort, + chosen_by: ChosenBy::UserConfig, + }; + match choice.test_call() { + Ok(()) => { + return Ok(Some(( + harness, + ModelAndEffort { + model: choice.model, + effort: choice.effort, + }, + ))); + } + Err(error) => say(&format!("{error:#}")), + } + } +} + +/// Ask for the `setting`, the Model or the Effort, of `harness`, with +/// `current` as the default; `-` is none, the Harness's own default. +fn ask_setting(setting: &str, harness: Harness, current: Option<&str>) -> Result<Option<String>> { + let name = harness.name(); + let own = format!("{name}'s own default"); + let answer = match current { + Some(current) => answer(&format!("{setting} for {name}, - for {own}"), Some(current))?, + None => read(&format!("{setting} for {name} [{own}]: "))?, + }; + Ok(Some(answer).filter(|answer| !answer.is_empty() && answer != "-")) +} + /// Ask `question` until the answer is yes, no or nothing, which is `default`. fn yes_or_no(question: &str, default: bool) -> Result<bool> { let choices = if default { "[Y/n]" } else { "[y/N]" }; diff --git a/tests/setup.rs b/tests/setup.rs index d7d77d86c..c2c9e3106 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -4,7 +4,9 @@ //! existing one, it keeps its values and comments and adds the keys it lacks. //! From a terminal, it first asks the Setup questions on stderr, each with the //! current value as its default answer. Base fixes are asked about only -//! with every Run a Merge run. +//! with every Run a Merge run. The Harness question lists both Harnesses, +//! refusing one sessions can't run on here, and the Model and Effort are +//! asked for the Harness chosen, a Claude Model checked with a test call. mod support; @@ -73,7 +75,10 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { "[logs]", "[activity]", "[spec]", - "[pickup]" + "[pickup]", + "[harness]", + "[harness.claude]", + "[harness.codex]" ], "{text}" ); @@ -90,6 +95,12 @@ fn with_no_terminal_and_no_user_config_setup_writes_the_defaults() { assert_eq!(config["activity"]["quiet_skips"].as_bool(), Some(false)); assert_eq!(config["spec"]["parallel"].as_integer(), Some(3)); assert_eq!(config["pickup"]["limit"].as_integer(), Some(3)); + assert_eq!(config["harness"]["default"].as_str(), Some("claude")); + for harness in ["claude", "codex"] { + for key in ["model", "effort"] { + assert_eq!(config["harness"][harness][key].as_str(), Some(""), "{text}"); + } + } assert!( result.stderr.contains(".thirdshift/config.toml"), "stderr: {}", @@ -125,7 +136,12 @@ fn every_key_is_written_with_a_comment_giving_what_it_does_and_its_default() { "logs.dir", "activity.quiet_skips", "spec.parallel", - "pickup.limit" + "pickup.limit", + "harness.default", + "harness.claude.model", + "harness.claude.effort", + "harness.codex.model", + "harness.codex.effort" ], "{text}" ); @@ -224,12 +240,17 @@ fn setup_with_an_argument_is_an_argument_error_and_writes_nothing() { } /// Every key this version knows, as `section.key`, sorted. -const EVERY_KEY: [&str; 10] = [ +const EVERY_KEY: [&str; 15] = [ "activity.quiet_skips", "base.fix", "email.always", "email.from", "email.to", + "harness.claude.effort", + "harness.claude.model", + "harness.codex.effort", + "harness.codex.model", + "harness.default", "launch.pull", "logs.dir", "merge.always", @@ -340,7 +361,7 @@ fn setup_writes_pickup_limit_at_3_with_its_comment_and_keeps_one_already_there() let text = user_config(&scenario).unwrap(); let written = "\n[pickup]\nlimit = 3 # how many open issues labelled in-progress stop a \ Pickup run taking another; default 3\n"; - assert!(text.ends_with(written), "{text}"); + assert!(text.contains(&format!("{written}\n[harness]\n")), "{text}"); let mine = "[pickup]\nlimit = 7 # I review quickly\n"; scenario.user_config_is(mine); @@ -518,6 +539,9 @@ fn setup_never_replaces_an_existing_email_to() { // Setup from a terminal. +const HARNESS: &str = "Harness for every Run's sessions"; +const MODEL: &str = "Model for claude"; +const EFFORT: &str = "Effort for claude"; const MERGE: &str = "Merge run?"; const BASE_FIX: &str = "Every Run may start a Base fix when the Base branch's CI is red?"; const PULL: &str = "fast-forward"; @@ -551,6 +575,9 @@ fn setup_on_terminal( /// the default sender, then `rest`. fn notifications_on<'a>(rest: &[Keystrokes<'a>]) -> Vec<Keystrokes<'a>> { let mut keystrokes = vec![ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -574,7 +601,18 @@ fn on_a_terminal_pressing_enter_throughout_writes_what_setup_with_no_terminal_wr let scenario = Scenario::new(); scenario.git_email_is(None); - let result = setup_on_terminal(&scenario, &[], &[(MERGE, ""), (PULL, ""), (NOTIFY, "")]); + let result = setup_on_terminal( + &scenario, + &[], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, ""), + (PULL, ""), + (NOTIFY, ""), + ], + ); assert_eq!(result.user_config, user_config(&unattended)); for prompt in [MERGE, PULL, NOTIFY] { @@ -595,6 +633,9 @@ fn on_a_terminal_the_answers_are_written_with_the_comments_on_each_key() { &scenario, &[("RESEND_API_KEY", KEY)], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, "y"), (PULL, "yes"), @@ -625,7 +666,12 @@ fn on_a_terminal_the_answers_are_written_with_the_comments_on_each_key() { "logs.dir", "activity.quiet_skips", "spec.parallel", - "pickup.limit" + "pickup.limit", + "harness.default", + "harness.claude.model", + "harness.claude.effort", + "harness.codex.model", + "harness.codex.effort" ], "{text}" ); @@ -656,7 +702,15 @@ fn on_a_terminal_with_merging_on_setup_asks_about_base_fixes_and_writes_the_answ let result = setup_on_terminal( &scenario, &[], - &[(MERGE, "y"), (BASE_FIX, answer), (PULL, ""), (NOTIFY, "")], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, "y"), + (BASE_FIX, answer), + (PULL, ""), + (NOTIFY, ""), + ], ); let config = table(&result); @@ -676,8 +730,18 @@ fn on_a_terminal_with_merging_off_setup_asks_nothing_about_base_fixes_and_writes let scenario = Scenario::new(); scenario.git_email_is(None); - let result = - setup_on_terminal(&scenario, &[], &[(MERGE, answer), (PULL, ""), (NOTIFY, "")]); + let result = setup_on_terminal( + &scenario, + &[], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, answer), + (PULL, ""), + (NOTIFY, ""), + ], + ); assert!( !result.stderr.contains(BASE_FIX), @@ -699,7 +763,15 @@ fn on_a_terminal_the_base_fix_question_defaults_to_the_user_configs_base_fix() { let result = setup_on_terminal( &scenario, &[], - &[(MERGE, ""), (BASE_FIX, ""), (PULL, ""), (NOTIFY, "")], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, ""), + (BASE_FIX, ""), + (PULL, ""), + (NOTIFY, ""), + ], ); assert!( @@ -717,7 +789,18 @@ fn on_a_terminal_turning_merging_off_writes_base_fix_at_its_default() { scenario.git_email_is(None); scenario.user_config_is("[merge]\nalways = true\n\n[base]\nfix = true # mine\n"); - let result = setup_on_terminal(&scenario, &[], &[(MERGE, "n"), (PULL, ""), (NOTIFY, "")]); + let result = setup_on_terminal( + &scenario, + &[], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, "n"), + (PULL, ""), + (NOTIFY, ""), + ], + ); assert!( !result.stderr.contains(BASE_FIX), @@ -740,6 +823,9 @@ fn on_a_terminal_the_address_defaults_to_the_suggested_github_email() { &scenario, &[], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -793,6 +879,17 @@ parallel = 5 [pickup] limit = 5 + +[harness] +default = \"claude\" + +[harness.claude] +model = \"opus\" +effort = \"high\" + +[harness.codex] +model = \"gpt-6.1-sol\" +effort = \"max\" "; scenario.user_config_is(mine); @@ -800,6 +897,9 @@ limit = 5 &scenario, &[("RESEND_API_KEY", KEY)], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (BASE_FIX, ""), (PULL, ""), @@ -817,7 +917,13 @@ limit = 5 "terminal: {}", result.stderr ); - for current in ["[Y/n]", "mine@example.net", "ts@acme.dev"] { + for current in [ + "[Y/n]", + "mine@example.net", + "ts@acme.dev", + "[opus]", + "[high]", + ] { assert!( result.stderr.contains(current), "terminal: {}", @@ -846,6 +952,9 @@ always = false # quiet, please &scenario, &[], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, ""), (PULL, ""), @@ -881,6 +990,9 @@ fn on_a_terminal_an_address_without_an_at_is_asked_again() { &scenario, &[], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -911,7 +1023,14 @@ fn on_a_terminal_with_notifications_off_nothing_about_email_is_asked() { let result = setup_on_terminal( &scenario, &[("THIRDSHIFT_RESEND_URL", resend.url())], - &[(MERGE, ""), (PULL, ""), (NOTIFY, "n")], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, ""), + (PULL, ""), + (NOTIFY, "n"), + ], ); assert_eq!(scenario.credentials().as_deref(), Some(saved)); @@ -1187,6 +1306,9 @@ fn on_a_terminal_accepting_the_test_email_sends_one_and_declining_sends_none() { ("THIRDSHIFT_RESEND_URL", resend.url()), ], &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -1219,7 +1341,14 @@ fn ctrl_c_during_the_questions_writes_no_user_config() { let result = scenario.run_on_terminal( &["setup"], &[], - &[(MERGE, "y"), (BASE_FIX, ""), (PULL, CTRL_C)], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, "y"), + (BASE_FIX, ""), + (PULL, CTRL_C), + ], ); assert_ne!(result.code, Some(0), "terminal: {}", result.stderr); @@ -1236,7 +1365,15 @@ fn ctrl_c_during_the_questions_leaves_an_existing_user_config_unchanged() { let result = scenario.run_on_terminal( &["setup"], &[], - &[(MERGE, "n"), (PULL, "y"), (NOTIFY, "y"), (TO, CTRL_C)], + &[ + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, "n"), + (PULL, "y"), + (NOTIFY, "y"), + (TO, CTRL_C), + ], ); assert_ne!(result.code, Some(0), "terminal: {}", result.stderr); @@ -1260,3 +1397,217 @@ fn on_a_terminal_setup_still_refuses_a_broken_user_config_before_asking() { ); assert_eq!(result.user_config.as_deref(), Some(broken)); } + +// The Harness, Model and Effort. + +/// What Claude says, and how it exits, when it refuses the Model it is +/// asked to run on. +const CLAUDE_REFUSES_THE_MODEL: &str = r#" +echo "There's an issue with the selected model (Opus 5.5). It may not exist or you may not have access to it." +exit 1 +"#; + +/// The answers that leave every setting but the Harness's as it is. +const NOTHING_ELSE: [Keystrokes; 3] = [(MERGE, ""), (PULL, ""), (NOTIFY, "")]; + +/// `PATH` with the scenario's fakes and the system's tools, and so no +/// `codex`, whatever is installed on the machine the tests run on. +fn fakes_only_path(scenario: &Scenario) -> String { + format!("{}:/usr/bin:/bin", scenario.path("bin").display()) +} + +/// Each call to `claude` that was the test call checking a Model, by its +/// arguments. +fn test_calls(scenario: &Scenario) -> Vec<Vec<String>> { + scenario + .claude_calls() + .into_iter() + .map(|call| { + call["argv"] + .as_array() + .unwrap() + .iter() + .map(|arg| arg.as_str().unwrap().to_string()) + .collect() + }) + .collect() +} + +#[test] +fn on_a_terminal_the_harness_question_lists_both_harnesses_and_defaults_to_claude() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![(HARNESS, ""), (MODEL, ""), (EFFORT, "")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + assert!( + result.stderr.contains( + "Harness for every Run's sessions, claude or codex (not supported yet) [claude]: " + ), + "terminal: {}", + result.stderr + ); + assert!( + !result.stderr.contains("for codex"), + "terminal: {}", + result.stderr + ); + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("claude")); + for harness in ["claude", "codex"] { + for key in ["model", "effort"] { + assert_eq!(config["harness"][harness][key].as_str(), Some("")); + } + } + assert!(test_calls(&scenario).is_empty()); +} + +#[test] +fn on_a_terminal_a_harness_sessions_cant_run_on_is_refused_and_asked_again() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![ + (HARNESS, "codex"), + (HARNESS, "gemini"), + (HARNESS, "claude"), + (MODEL, ""), + (EFFORT, ""), + ]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + for refusal in [ + "codex is not supported yet: Codex sessions come in a later thirdshift.", + "Choose claude or codex.", + ] { + assert!( + result.stderr.contains(refusal), + "terminal: {}", + result.stderr + ); + } + assert_eq!( + table(&result)["harness"]["default"].as_str(), + Some("claude") + ); +} + +#[test] +fn on_a_terminal_with_claude_not_installed_the_harness_is_left_as_it_was() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.user_config_is("[harness.claude]\nmodel = \"opus\"\n"); + fs::remove_file(scenario.path("bin/claude")).unwrap(); + let path = fakes_only_path(&scenario); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &NOTHING_ELSE); + + assert!( + result + .stderr + .contains("claude (not installed) or codex (not supported yet)"), + "terminal: {}", + result.stderr + ); + assert!( + !result.stderr.contains(MODEL), + "terminal: {}", + result.stderr + ); + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("claude")); + assert_eq!(config["harness"]["claude"]["model"].as_str(), Some("opus")); +} + +#[test] +fn on_a_terminal_the_model_and_effort_answers_are_written_after_a_test_call_with_them() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let mut keystrokes = vec![(HARNESS, ""), (MODEL, "claude-opus-5-5"), (EFFORT, "high")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[], &keystrokes); + + let config = table(&result); + assert_eq!( + config["harness"]["claude"]["model"].as_str(), + Some("claude-opus-5-5") + ); + assert_eq!(config["harness"]["claude"]["effort"].as_str(), Some("high")); + assert_eq!(config["harness"]["codex"]["model"].as_str(), Some("")); + let text = result.user_config.unwrap(); + assert!( + text.contains("model = \"claude-opus-5-5\" # the Model Claude Code's sessions run on"), + "{text}" + ); + assert_eq!( + test_calls(&scenario), + [["-p", "--model", "claude-opus-5-5", "--effort", "high"]] + ); +} + +#[test] +fn on_a_terminal_the_model_and_effort_default_to_the_current_ones_and_a_dash_clears_one() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.user_config_is( + "[harness.claude]\nmodel = \"opus\" # mine\neffort = \"high\"\n\n\ + [harness.codex]\nmodel = \"gpt-6.1-sol\"\n", + ); + let mut keystrokes = vec![(HARNESS, ""), (MODEL, ""), (EFFORT, "-")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[], &keystrokes); + + for shown in [ + "Model for claude, - for claude's own default [opus]: ", + "Effort for claude, - for claude's own default [high]: ", + ] { + assert!(result.stderr.contains(shown), "terminal: {}", result.stderr); + } + let text = result.user_config.clone().unwrap(); + assert!(text.contains("model = \"opus\" # mine\n"), "{text}"); + let config = table(&result); + assert_eq!(config["harness"]["claude"]["effort"].as_str(), Some("")); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-6.1-sol") + ); + assert_eq!(test_calls(&scenario), [["-p", "--model", "opus"]]); +} + +#[test] +fn on_a_terminal_a_model_claude_refuses_is_asked_again_with_claudes_error() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.agent_does_in_session(1, CLAUDE_REFUSES_THE_MODEL); + let mut keystrokes = vec![ + (HARNESS, ""), + (MODEL, "Opus 5.5"), + (EFFORT, ""), + (MODEL, "opus"), + (EFFORT, ""), + ]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[], &keystrokes); + + assert!( + result.stderr.contains( + "claude refused a test call on the Model Opus 5.5: There's an issue with the \ + selected model (Opus 5.5)." + ), + "terminal: {}", + result.stderr + ); + assert_eq!( + table(&result)["harness"]["claude"]["model"].as_str(), + Some("opus") + ); + assert_eq!(test_calls(&scenario).len(), 2); +} diff --git a/tests/setup_offer.rs b/tests/setup_offer.rs index 3b155178f..113054753 100644 --- a/tests/setup_offer.rs +++ b/tests/setup_offer.rs @@ -23,6 +23,9 @@ gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" const PR_URL: &str = "https://github.com/acme/widgets/pull/1"; const OFFER: &str = "Set your defaults now? [Y/n]"; +const HARNESS: &str = "Harness for every Run's sessions"; +const MODEL: &str = "Model for claude"; +const EFFORT: &str = "Effort for claude"; const MERGE: &str = "Merge run?"; const BASE_FIX: &str = "Every Run may start a Base fix when the Base branch's CI is red?"; const PULL: &str = "fast-forward"; @@ -79,6 +82,9 @@ fn accepting_and_choosing_merge_always_makes_that_run_a_merge_run() { &[], &[ (OFFER, ""), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, ""), (PULL, ""), @@ -115,6 +121,9 @@ gh fake checks "$(git rev-parse HEAD)" '[{"name": "test", "conclusion": "success &[], &[ (OFFER, ""), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, "y"), (PULL, ""), @@ -140,6 +149,9 @@ fn no_merge_in_the_command_wins_over_a_fresh_merge_always() { &[], &[ (OFFER, "y"), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, ""), (PULL, ""), @@ -166,6 +178,9 @@ fn no_email_in_the_command_wins_over_fresh_run_notifications() { ], &[ (OFFER, "y"), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -192,6 +207,9 @@ fn accepting_asks_for_the_key_saves_it_and_the_run_notification_goes_with_it() { &[("THIRDSHIFT_RESEND_URL", resend.url())], &[ (OFFER, "y"), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, ""), (PULL, ""), (NOTIFY, "y"), @@ -332,6 +350,9 @@ fn an_unwritable_user_config_is_a_warning_and_the_run_completes_on_the_defaults( &[], &[ (OFFER, "y"), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), (MERGE, "y"), (BASE_FIX, ""), (PULL, ""), @@ -385,7 +406,15 @@ fn ctrl_c_during_the_questions_writes_no_file_and_does_no_work() { let result = scenario.run_on_terminal( &[&scenario.issue_url(7)], &[], - &[(OFFER, "y"), (MERGE, "y"), (BASE_FIX, ""), (PULL, CTRL_C)], + &[ + (OFFER, "y"), + (HARNESS, ""), + (MODEL, ""), + (EFFORT, ""), + (MERGE, "y"), + (BASE_FIX, ""), + (PULL, CTRL_C), + ], ); assert_ne!(result.code, Some(0), "terminal: {}", result.stderr); From 2c45b87f0bcab4dd0c0b11a94d786fbee7d121ae Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 00:56:41 -0400 Subject: [PATCH 06/14] Run a Command's sessions on Codex (#394) harness codex runs every session as codex exec --json, unsandboxed, with stdin null, the Model and Effort as Codex names them and the CLAUDE.md fallback. The Factory skills are linked into .agents/skills/, and each prompt loads its skill as $thirdshift-<skill>. Codex's JSONL stream gives the progress lines, session id, final message and failures. The Model and Effort are checked against codex debug models before any work, replacing the not-supported-yet refusal. The fake gains a codex personality. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- README.md | 9 +- prompts/README.md | 4 +- site/prompts/index.html | 1 + src/architect.rs | 3 +- src/base_fix.rs | 6 + src/harness.rs | 323 +++++++++++++++++++++++++-- src/main.rs | 47 ++-- src/pickup.rs | 2 +- src/progress.rs | 84 +++++-- src/progress/codex.rs | 473 ++++++++++++++++++++++++++++++++++++++++ src/prompts_page.rs | 46 +++- src/run.rs | 16 +- src/session.rs | 133 ++++++++--- src/skills.rs | 47 ++-- tests/codex_sessions.rs | 422 +++++++++++++++++++++++++++++++++++ tests/fakes.rs | 11 +- tests/fakes/claude.rs | 35 +-- tests/fakes/codex.rs | 152 +++++++++++++ tests/harness_choice.rs | 29 --- tests/support/fakes.rs | 9 +- tests/support/mod.rs | 55 +++-- 21 files changed, 1715 insertions(+), 192 deletions(-) create mode 100644 src/progress/codex.rs create mode 100644 tests/codex_sessions.rs create mode 100644 tests/fakes/codex.rs diff --git a/README.md b/README.md index cdf7c7583..8c8d1656e 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ One statically linked Linux binary covers the first three, so it doesn't depend The user account that runs thirdshift needs: - **`claude`** (Claude Code), logged in. +- **`codex`** (Codex CLI), logged in, only for `harness codex`. - **`gh`** (GitHub CLI), logged in. - **`git`** with a global `user.name` and `user.email`, and credentials that can push to the repository (`gh auth setup-git` makes git use `gh`'s login). The agents commit as this identity; without it, an agent may borrow the author of the last commit. @@ -108,7 +109,7 @@ Sessions run headless in Claude Code's auto mode (`claude -p --permission-mode a ### What sessions leave behind -Cleanup removes only thirdshift's own worktree, local Issue branch and the temporary directory it wrote the Factory skills out to. The `.git/info/exclude` entry that keeps the linked skills out of git, `/.claude/skills/thirdshift-*`, is added once and left in place, as every worktree of the repository shares the file. Anything else an agent does as your user stays. For example, one Run downloaded a JDK to `~/.local/jdk/` because the server had no Java, installed Playwright in `/tmp/pw`, and left a pull request body draft in `/tmp/`. This is by design, but worth knowing: +Cleanup removes only thirdshift's own worktree, local Issue branch and the temporary directory it wrote the Factory skills out to. The `.git/info/exclude` entry that keeps the linked skills out of git, `/.claude/skills/thirdshift-*`, or `/.agents/skills/thirdshift-*` on Codex, is added once and left in place, as every worktree of the repository shares the file. Anything else an agent does as your user stays. For example, one Run downloaded a JDK to `~/.local/jdk/` because the server had no Java, installed Playwright in `/tmp/pw`, and left a pull request body draft in `/tmp/`. This is by design, but worth knowing: - **Install the target repository's toolchains up front** (language runtimes, package managers, test tools), so the agents don't download their own on every Run. - **For UI work, install the headless browser dependencies**, e.g. `libgbm1` on Ubuntu server, which headless Chromium needs. Without it, agents can't take screenshots to check a UI change and fall back to reading the code. @@ -282,9 +283,11 @@ Every session of a command, its implement session, each Repair and Resume, a Spe thirdshift https://github.com/<owner>/<repo>/issues/<n> model claude-opus-5-5 effort high ``` -For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Only `claude` works for now: `harness codex` fails with `not supported yet`. +For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Codex takes them as it names them, which the check below settles: `-m <model>` and `-c model_reasoning_effort="<effort>"` when set. -Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and a named Model gets a minimal test call, `claude -p --model <model>` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. +With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-<skill>`. A Resume runs `codex exec … resume <session id>`, with every flag given again. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. + +Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model <model>` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. The Command log's opening lines (`sessions run on claude · claude-opus-5-5 · high`), the Activity log's start line, the pull request's body and the Run notification each name the Harness, Model and Effort, `default model` or `default effort` where the Harness's own is used. In the pull request's body, the Spec PR's included, thirdshift writes the line itself once the opening session has ended, such as `Built with claude · claude-opus-5-5 · high`, replacing any it wrote there before. diff --git a/prompts/README.md b/prompts/README.md index b1012d114..6b0f38373 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -1,4 +1,4 @@ -<!-- Generated by src/prompts_page.rs from the prompts' titles and when sentences in src/prompts_page.rs and the claude arguments in src/session.rs; don't edit. Regenerate with UPDATE_PROMPTS=1 cargo test prompts_page --> +<!-- Generated by src/prompts_page.rs from the prompts' titles and when sentences in src/prompts_page.rs and the claude and codex arguments in src/session.rs; don't edit. Regenerate with UPDATE_PROMPTS=1 cargo test prompts_page --> # Session prompts @@ -16,6 +16,8 @@ A [Resume](resume.md) continues the session that ended: claude -p --permission-mode auto --output-format stream-json --verbose --resume <session id> <prompt> ``` +With `harness codex`, each session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox -m <model> -c model_reasoning_effort="<effort>" -c project_doc_fallback_filenames=["CLAUDE.md"] <prompt>` instead, without `-m <model>` or the Effort's `-c` where they are not set, and with `resume <session id>` before the prompt for a Resume. The Factory skills are linked into `.agents/skills/`, and a prompt's first line loads its skill as `$thirdshift-<skill>`. + | Prompt | When it is sent | File | | --- | --- | --- | | Fresh | Starts the implement session, which goes on to review and open the pull request, when the Run starts a new Issue branch. | [fresh.md](fresh.md) | diff --git a/site/prompts/index.html b/site/prompts/index.html index 7607c4f82..31c82a603 100644 --- a/site/prompts/index.html +++ b/site/prompts/index.html @@ -38,6 +38,7 @@ <h1 class="misreg">Prompts and skills</h1> <pre class="job-text" aria-label="A new session"><code>claude -p --permission-mode auto --output-format stream-json --verbose <mark class="ph"><prompt></mark></code></pre> <p class="lede">A <a href="#prompt-resume">Resume</a> continues the session that ended:</p> <pre class="job-text" aria-label="A Resume"><code>claude -p --permission-mode auto --output-format stream-json --verbose --resume <mark class="ph"><session id></mark> <mark class="ph"><prompt></mark></code></pre> + <p class="lede">With <code>harness codex</code>, each session runs <code>codex exec --json --dangerously-bypass-approvals-and-sandbox -m <mark class="ph"><model></mark> -c model_reasoning_effort="<mark class="ph"><effort></mark>" -c project_doc_fallback_filenames=["CLAUDE.md"] <mark class="ph"><prompt></mark></code> instead, without <code>-m <mark class="ph"><model></mark></code> or the Effort's <code>-c</code> where they are not set, and with <code>resume <mark class="ph"><session id></mark></code> before the prompt for a Resume. The Factory skills are linked into <code>.agents/skills/</code>, and a prompt's first line loads its skill as <code>$thirdshift-<skill></code>.</p> </div> </section> <section class="sec" aria-labelledby="prompts-title"> diff --git a/src/architect.rs b/src/architect.rs index 14be9be77..aea25565c 100644 --- a/src/architect.rs +++ b/src/architect.rs @@ -220,7 +220,7 @@ pub fn run( focus: Option<&str>, base: Option<&str>, launch_pull: bool, - harness: &Choice, + harness: &mut Choice, ) -> Result<Outcome, FailedRun> { let started = Utc::now(); let Launch { @@ -242,6 +242,7 @@ pub fn run( return Ok(skip(&repo, skipped)); } harness.check()?; + let harness = &*harness; logs::started(Work::ArchitectRun(&repo), harness); if launch_pull { directory.pull(&base); diff --git a/src/base_fix.rs b/src/base_fix.rs index 1ab90a3f1..8a0f97146 100644 --- a/src/base_fix.rs +++ b/src/base_fix.rs @@ -157,6 +157,12 @@ impl BaseFix { } } + /// Have a Base fix it starts run its sessions on `harness`, the Run's + /// once checked. + pub fn runs_on(&mut self, harness: Choice) { + self.harness = harness; + } + /// Whether the Run compares its red checks with the Base branch's, so /// that some may be Inherited failures. pub fn sees_inherited_failures(&self) -> bool { diff --git a/src/harness.rs b/src/harness.rs index 616c8e760..0a9c00a08 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -6,9 +6,10 @@ use std::fmt; use std::io::Write; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Command, Output, Stdio}; use anyhow::{Context, Result, bail}; +use serde_json::Value; use crate::progress; @@ -164,19 +165,31 @@ impl Choice { args } - /// Check, before any work, that sessions can run on this: Codex is not - /// supported yet; the Harness's CLI must be on `PATH`; and a Model named - /// for Claude must take a minimal test call, with the Effort, if any. - /// Each failure says what to change. - pub fn check(&self) -> Result<()> { - let cli = self.harness.name(); - if self.harness == Harness::Codex { - bail!( - "harness codex, chosen by {}, is not supported yet: Codex sessions come in a \ - later thirdshift; choose harness claude", - self.chosen_by - ); + /// The arguments that run a Codex session on its Model and Effort, where + /// they are set. + pub fn codex_args(&self) -> Vec<String> { + let mut args = Vec::new(); + if let Some(model) = &self.model { + args.extend(["-m".to_string(), model.clone()]); } + if let Some(effort) = &self.effort { + args.extend([ + "-c".to_string(), + format!("model_reasoning_effort=\"{effort}\""), + ]); + } + args + } + + /// Check, before any work, that sessions can run on this, and settle the + /// Model and Effort on the names the Harness knows them by. The + /// Harness's CLI must be on `PATH`. A Model named for Claude must take a + /// minimal test call, with the Effort, if any. A Model or Effort named + /// for Codex must be in its catalog, which costs no tokens to read: each + /// is matched regardless of case, a Model by its slug or display name, + /// and becomes the name Codex takes. Each failure says what to change. + pub fn check(&mut self) -> Result<()> { + let cli = self.harness.name(); if !on_path(cli) { bail!( "{cli} is not on PATH, and the Harness {cli} is chosen by {}: install it, \ @@ -184,6 +197,15 @@ impl Choice { self.chosen_by ); } + match self.harness { + Harness::Claude => self.check_claude(), + Harness::Codex => self.check_codex(), + } + } + + /// Make a test call to Claude on the Model, if one is named. + fn check_claude(&self) -> Result<()> { + let cli = self.harness.name(); let Some(model) = &self.model else { return Ok(()); }; @@ -208,17 +230,166 @@ impl Choice { if output.status.success() { return Ok(()); } - let said = [&output.stderr, &output.stdout] - .map(|said| String::from_utf8_lossy(said).trim().to_string()) - .into_iter() - .filter(|said| !said.is_empty()) - .collect::<Vec<_>>() - .join("\n"); let effort = match &self.effort { Some(effort) => format!(" with the Effort {effort}"), None => String::new(), }; - bail!("{cli} refused a test call on the Model {model}{effort}: {said}") + bail!( + "{cli} refused a test call on the Model {model}{effort}: {}", + said(&output) + ) + } + + /// Settle the Model and Effort, if either is named, on Codex's names for + /// them, from its catalog. + fn check_codex(&mut self) -> Result<()> { + if self.model.is_none() && self.effort.is_none() { + return Ok(()); + } + progress::step("checking the Model and Effort against codex debug models"); + let output = Command::new("codex") + .args(["debug", "models"]) + .stdin(Stdio::null()) + .output() + .context("could not run codex debug models to check the Model and Effort")?; + if !output.status.success() { + bail!( + "codex debug models failed, so the Model and Effort can't be checked: {}", + said(&output) + ); + } + let catalog = Catalog::parse(&String::from_utf8_lossy(&output.stdout))?; + let (model, effort) = catalog.settle(self.model.as_deref(), self.effort.as_deref())?; + self.model = model; + self.effort = effort; + Ok(()) + } +} + +/// What a CLI said on stderr and stdout, trimmed, each on its own lines. +fn said(output: &Output) -> String { + [&output.stderr, &output.stdout] + .map(|said| String::from_utf8_lossy(said).trim().to_string()) + .into_iter() + .filter(|said| !said.is_empty()) + .collect::<Vec<_>>() + .join("\n") +} + +/// One Model in Codex's catalog. +#[derive(Debug, Clone, PartialEq, Eq)] +struct CatalogModel { + /// The name Codex takes. + slug: String, + display_name: Option<String>, + /// The Efforts it supports. + efforts: Vec<String>, + /// Whether the catalog lists it to choose from. + listed: bool, +} + +/// The Models Codex can run, as `codex debug models` prints them. +#[derive(Debug)] +struct Catalog { + models: Vec<CatalogModel>, +} + +impl Catalog { + /// The catalog `codex debug models` printed as `json`. + fn parse(json: &str) -> Result<Catalog> { + let parsed: Value = serde_json::from_str(json) + .context("codex debug models printed no JSON catalog of Models")?; + let models = parsed["models"] + .as_array() + .context("codex debug models printed no list of Models")? + .iter() + .filter_map(|model| { + Some(CatalogModel { + slug: model["slug"].as_str()?.to_string(), + display_name: model["display_name"].as_str().map(String::from), + efforts: model["supported_reasoning_levels"] + .as_array() + .into_iter() + .flatten() + .filter_map(|level| level["effort"].as_str().map(String::from)) + .collect(), + listed: model["visibility"] != "hide", + }) + }) + .collect(); + Ok(Catalog { models }) + } + + /// `model` and `effort` as Codex names them: the slug of the Model whose + /// slug or display name is `model`, regardless of case, and the Effort, + /// of those that Model supports, or with no Model of those any Model + /// supports, that is `effort`, regardless of case. Fails naming the + /// valid choices for a Model or Effort the catalog doesn't have. + fn settle( + &self, + model: Option<&str>, + effort: Option<&str>, + ) -> Result<(Option<String>, Option<String>)> { + let found = match model { + Some(model) => Some(self.model(model)?), + None => None, + }; + let Some(effort) = effort else { + return Ok((found.map(|found| found.slug.clone()), None)); + }; + let efforts: Vec<&str> = match found { + Some(found) => found.efforts.iter().map(String::as_str).collect(), + None => { + let mut efforts = Vec::new(); + for each in self.models.iter().flat_map(|model| &model.efforts) { + if !efforts.contains(&each.as_str()) { + efforts.push(each.as_str()); + } + } + efforts + } + }; + let Some(settled) = efforts + .iter() + .find(|supported| supported.eq_ignore_ascii_case(effort)) + else { + let of = match found { + Some(found) => format!("the Codex Model {}", found.slug), + None => "any Codex Model".to_string(), + }; + bail!( + "the Effort {effort} is not one {of} supports: choose one of {}", + efforts.join(", ") + ); + }; + Ok(( + found.map(|found| found.slug.clone()), + Some(settled.to_string()), + )) + } + + /// The Model whose slug or display name is `name`, regardless of case. + fn model(&self, name: &str) -> Result<&CatalogModel> { + let named = |candidate: &&CatalogModel| { + candidate.slug.eq_ignore_ascii_case(name) + || candidate + .display_name + .as_deref() + .is_some_and(|display| display.eq_ignore_ascii_case(name)) + }; + if let Some(found) = self.models.iter().find(named) { + return Ok(found); + } + let listed: Vec<&str> = self + .models + .iter() + .filter(|model| model.listed) + .map(|model| model.slug.as_str()) + .collect(); + bail!( + "the Model {name} is not in Codex's catalog: choose one of {}", + listed.join(", ") + ) } } @@ -417,4 +588,116 @@ mod tests { assert_eq!(chosen.built_with(), line); } } + + #[test] + fn codex_is_given_the_model_and_the_effort_as_its_config_setting() { + for (model, effort, args) in [ + ("", "", vec![]), + ("gpt-6.1-sol", "", vec!["-m", "gpt-6.1-sol"]), + ( + "gpt-6.1-sol", + "max", + vec!["-m", "gpt-6.1-sol", "-c", "model_reasoning_effort=\"max\""], + ), + ] { + let chosen = choice(Harness::Codex, model, effort, ChosenBy::Command); + + assert_eq!(chosen.codex_args(), args); + } + } + + /// A catalog as `codex debug models` prints one, trimmed to what is read. + fn catalog() -> Catalog { + Catalog::parse( + r#"{"models": [ + {"slug": "gpt-6.1-sol", "display_name": "GPT-6.1-Sol", "visibility": "list", + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "high"}, {"effort": "max"}]}, + {"slug": "gpt-5.5", "display_name": "GPT-5.5 Classic", "visibility": "list", + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "xhigh"}]}, + {"slug": "codex-auto-review", "display_name": "Auto review", "visibility": "hide", + "supported_reasoning_levels": [{"effort": "minimal"}]} + ]}"#, + ) + .unwrap() + } + + fn settled( + model: Option<&str>, + effort: Option<&str>, + ) -> Result<(Option<String>, Option<String>)> { + catalog().settle(model, effort) + } + + fn named(model: &str, effort: &str) -> (Option<String>, Option<String>) { + let set = set(model, effort); + (set.model, set.effort) + } + + #[test] + fn a_codex_model_and_effort_match_regardless_of_case_and_become_codexs_names() { + for (model, effort, settled_as) in [ + ( + Some("GPT-6.1-Sol"), + Some("Max"), + named("gpt-6.1-sol", "max"), + ), + (Some("gpt-6.1-sol"), None, named("gpt-6.1-sol", "")), + ( + Some("gpt-5.5 classic"), + Some("XHIGH"), + named("gpt-5.5", "xhigh"), + ), + ( + Some("codex-auto-review"), + None, + named("codex-auto-review", ""), + ), + (None, Some("High"), named("", "high")), + (None, None, named("", "")), + ] { + assert_eq!( + settled(model, effort).unwrap(), + settled_as, + "{model:?} {effort:?}" + ); + } + } + + #[test] + fn an_unknown_codex_model_fails_naming_the_listed_models() { + let error = settled(Some("gpt-7"), Some("max")).unwrap_err(); + + assert_eq!( + error.to_string(), + "the Model gpt-7 is not in Codex's catalog: choose one of gpt-6.1-sol, gpt-5.5" + ); + } + + #[test] + fn an_effort_the_codex_model_does_not_support_fails_naming_those_it_does() { + let error = settled(Some("GPT-5.5"), Some("max")).unwrap_err(); + + assert_eq!( + error.to_string(), + "the Effort max is not one the Codex Model gpt-5.5 supports: choose one of low, xhigh" + ); + } + + #[test] + fn with_no_codex_model_the_effort_must_be_one_some_model_supports() { + let error = settled(None, Some("ultra")).unwrap_err(); + + assert_eq!( + error.to_string(), + "the Effort ultra is not one any Codex Model supports: choose one of low, high, max, \ + xhigh, minimal" + ); + } + + #[test] + fn output_that_is_no_catalog_fails() { + for output in ["", "not json", r#"{"models": 3}"#] { + assert!(Catalog::parse(output).is_err(), "{output}"); + } + } } diff --git a/src/main.rs b/src/main.rs index 3392fd5d1..7009b6af8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,10 +85,12 @@ default takes [harness.claude]: model = \"opus\" effort = \"high\" -Before any work, the Harness's CLI must be on PATH, and a named Model gets a minimal test call -with its Effort, which must succeed: a failure stops the command naming what to fix, before -the Claim, the worktree and any Command log. Only claude works for now: harness codex fails as -not supported yet. The Command log, the Activity log's start line, the pull request's body, +Before any work, the Harness's CLI must be on PATH. On claude, a named Model gets a minimal +test call with its Effort, which must succeed. On codex, a named Model and Effort must be in +codex debug models, matched regardless of case, a Model by slug or display name, and are +passed on as Codex names them. A failure stops the command naming what to fix, before the +Claim, the worktree and any Command log. Codex sessions run codex exec --json +--dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, as in Built with claude · opus · high, and the Run notification each name all three. --email sends one Run notification when the Run ends, whatever the outcome: ready for @@ -383,12 +385,13 @@ fn main() -> ExitCode { Ok(started) => started, Err(failure) => return failure, }; - let asks = asks_of(&config); - started.built_with(&asks.harness); + let mut asks = asks_of(&config); let started_by = given .as_ref() .map_or(StartedBy::Command, |given| StartedBy::Child(&given.kind)); - started.finish(Ending::Run(run::run_to_end(&issue, &asks, started_by))) + let ended = run::run_to_end(&issue, &mut asks, started_by); + started.built_with(&asks.harness); + started.finish(Ending::Run(ended)) } /// An Architect run: the Architecture review and its plan marked ready, then, @@ -420,14 +423,15 @@ fn architect(args: ArchitectArgs) -> ExitCode { Ok(started) => started, Err(failure) => return failure, }; - let harness = args.flags.harness(&config); - started.built_with(&harness); - let review = match architect::run( + let mut harness = args.flags.harness(&config); + let review = architect::run( args.focus.as_deref(), args.base.as_deref(), config.launch_pull, - &harness, - ) { + &mut harness, + ); + started.built_with(&harness); + let review = match review { Ok(Outcome::Skipped(skipped)) => { return started.finish(Ending::Skipped(skipped.into())); } @@ -440,9 +444,12 @@ fn architect(args: ArchitectArgs) -> ExitCode { "dispatching the plan {url}, as thirdshift {url} would", url = plan.url )); - let asks = Asks::of_architect_plan(plan, &args.flags, &config); + let mut asks = Asks { + harness: harness.clone(), + ..Asks::of_architect_plan(plan, &args.flags, &config) + }; let started_by = StartedBy::Dispatch { base }; - Some(run::run_to_end(plan, &asks, started_by)) + Some(run::run_to_end(plan, &mut asks, started_by)) } _ => None, }; @@ -470,9 +477,10 @@ fn pickup(args: PickupArgs) -> ExitCode { Ok(started) => started, Err(failure) => return failure, }; - let harness = args.flags.harness(&config); + let mut harness = args.flags.harness(&config); + let taken = pickup::run(args.base.as_deref(), config.pickup_limit, &mut harness); started.built_with(&harness); - let taken = match pickup::run(args.base.as_deref(), config.pickup_limit, &harness) { + let taken = match taken { Ok(pickup::Outcome::Taken(taken)) => taken, Ok(pickup::Outcome::Skipped(skipped)) => { return started.finish(Ending::Skipped(skipped.into())); @@ -480,11 +488,14 @@ fn pickup(args: PickupArgs) -> ExitCode { Err(error) => return failure(&error), }; started.took(&taken.issue, taken.title); - let asks = Asks::of_ready_issue(&taken.issue, taken.is_spec, &args.flags, &config); + let mut asks = Asks { + harness, + ..Asks::of_ready_issue(&taken.issue, taken.is_spec, &args.flags, &config) + }; let started_by = StartedBy::Dispatch { base: &taken.base }; started.finish(Ending::Run(run::run_to_end( &taken.issue, - &asks, + &mut asks, started_by, ))) } diff --git a/src/pickup.rs b/src/pickup.rs index e67bc7569..f899d39be 100644 --- a/src/pickup.rs +++ b/src/pickup.rs @@ -112,7 +112,7 @@ impl fmt::Display for Skipped { /// it dispatches will run its sessions on, failing before any work if they /// can't run; then it records that it started work on the issue, which /// keeps its Command log. -pub fn run(base: Option<&str>, limit: NonZeroUsize, harness: &Choice) -> Result<Outcome> { +pub fn run(base: Option<&str>, limit: NonZeroUsize, harness: &mut Choice) -> Result<Outcome> { let Launch { directory, repo, diff --git a/src/progress.rs b/src/progress.rs index 8433d0fa2..9052d604b 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1,15 +1,20 @@ //! Progress lines on stderr: thirdshift's own steps, and a session's -//! stream-json output condensed to one short line per notable event. Each +//! stream, Claude's stream-json or Codex's JSONL, condensed to one short +//! line per notable event. Each //! line is stamped with the local time it was printed, so a stalled Run can //! be told from a busy one. Each is kept in the Command log too, if the //! command keeps one. +mod codex; + use std::collections::HashMap; use std::fmt::Display; +use std::path::Path; use chrono::Local; use serde_json::Value; +use crate::harness::Harness; use crate::logs; /// The longest detail a session line shows before it is cut short. @@ -108,7 +113,53 @@ fn split_stamp(line: &str) -> Option<(&str, &str)> { is_stamp.then_some((time, message)) } -/// Condenses one session's stream, a line at a time. Unknown and malformed +/// What a session's stream shows, read a line at a time: the progress lines +/// it condenses to, and once it has ended, how the session went. Each +/// Harness streams its own events: see [`for_harness`]. +pub trait Stream: Send { + /// The stderr lines, without the `thirdshift: ` prefix, for one line of + /// the stream: one per notable event in it, often none. Unknown and + /// malformed lines give none, never an error. + fn condense(&mut self, raw: &str) -> Vec<String>; + + /// What the session-ended line says the session took, if the stream + /// told. + fn summary(&self) -> Option<String>; + + /// The session's final message: what the agent said as it ended its + /// last turn, if anything. + fn final_message(&self) -> Option<&str>; + + /// The session's id, which a Resume continues. + fn session_id(&self) -> Option<&str>; + + /// Descriptions of the background work killed as the session ended: the + /// work still running when it ended its last turn, which it may have + /// been waiting on. None if that turn failed: the session ended for + /// another reason. + fn killed_background_work(&self) -> Vec<&str>; + + /// Whether the stream says the session failed, whatever its exit code. + fn failed(&self) -> bool { + false + } + + /// The error the stream gave for the session, for a failure's cause: + /// what failed its turn, else the last error it reported, if any. + fn error(&self) -> Option<&str> { + None + } +} + +/// The [`Stream`] of a session on `harness` in `worktree`. +pub fn for_harness(harness: Harness, worktree: &Path) -> Box<dyn Stream> { + match harness { + Harness::Claude => Box::new(Progress::default()), + Harness::Codex => Box::new(codex::CodexProgress::in_worktree(worktree)), + } +} + +/// Condenses one Claude session's stream-json, a line at a time. Unknown and malformed /// lines are skipped, never an error. /// /// A `result` event doesn't mean the session is over: a session that waits on @@ -140,10 +191,8 @@ pub struct Progress { final_message: Option<String>, } -impl Progress { - /// The stderr lines, without the `thirdshift: ` prefix, for one line of - /// the stream: one per notable event in it, often none. - pub fn condense(&mut self, raw: &str) -> Vec<String> { +impl Stream for Progress { + fn condense(&mut self, raw: &str) -> Vec<String> { let Ok(event) = serde_json::from_str::<Value>(raw) else { return Vec::new(); }; @@ -186,7 +235,7 @@ impl Progress { /// Turns and cost from the last `result` event that reported them. On a /// claude.ai subscription the cost is the API-price equivalent, not a /// charge, and says so. - pub fn summary(&self) -> Option<String> { + fn summary(&self) -> Option<String> { let (turns, cost) = self.turns_and_cost?; let basis = if self.subscription { " at API prices" @@ -196,24 +245,21 @@ impl Progress { Some(format!("{turns} turns, ${cost:.2}{basis}")) } - /// The session's final message: the text of its last `result` event, - /// what the agent said as it ended its last turn. None if that `result` - /// had no text, as one that is an error may not. - pub fn final_message(&self) -> Option<&str> { + /// The text of the last `result` event. None if that `result` had no + /// text, as one that is an error may not. + fn final_message(&self) -> Option<&str> { self.final_message.as_deref() } /// The session's id, from its first `init` event. - pub fn session_id(&self) -> Option<&str> { + fn session_id(&self) -> Option<&str> { self.session_id.as_deref() } - /// Descriptions of the background tasks killed after the last `result`: - /// the work still running when the session ended, which it may have been - /// waiting on. The stream doesn't say whether it was, or had been - /// abandoned. None if that `result` was an error: the session ended for - /// another reason. - pub fn killed_background_work(&self) -> Vec<&str> { + /// Descriptions of the background tasks killed after the last `result`. + /// The stream doesn't say whether the session was waiting on them, or + /// had abandoned them. None if that `result` was an error. + fn killed_background_work(&self) -> Vec<&str> { if self.failed { return Vec::new(); } @@ -222,7 +268,9 @@ impl Progress { .map(|(_, description)| description.as_str()) .collect() } +} +impl Progress { /// Track a background task's description and whether it was killed. fn track_task(&mut self, event: &Value) { let Some(id) = event["task_id"].as_str() else { diff --git a/src/progress/codex.rs b/src/progress/codex.rs new file mode 100644 index 000000000..1c6a07abc --- /dev/null +++ b/src/progress/codex.rs @@ -0,0 +1,473 @@ +//! Codex's `exec --json` stream, condensed: `thread.started` gives the +//! session id, `item.*` events give a progress line per command, file +//! change and tool call, `turn.completed` the token totals and +//! `turn.failed` the error the session failed with. See +//! `docs/research/codex-headless-harness.md`. + +use std::collections::HashSet; +use std::path::Path; + +use serde_json::Value; + +use super::{Stream, bash, shorten}; + +/// Condenses one Codex session's JSONL stream, a line at a time. +/// +/// `codex exec` runs one turn per process. A command the agent left running +/// shows as a `command_execution` item still `in_progress` when the turn +/// completes, and it is killed as Codex exits: see +/// [`Stream::killed_background_work`]. +#[derive(Default)] +pub struct CodexProgress { + /// The worktree the session runs in, which Codex's stream doesn't name. + cwd: String, + session_id: Option<String>, + /// The ids of the items that have had their line. + seen: HashSet<String>, + /// The commands still running, by item id, in the order they started. + running: Vec<(String, String)>, + /// The commands still running when the turn completed. + killed: Vec<String>, + /// Token totals from `turn.completed`: input, of which cached, and + /// output. + tokens: Option<(u64, u64, u64)>, + final_message: Option<String>, + /// What failed the turn, from `turn.failed`. + failure: Option<String>, + /// The last error the stream reported, which may have been retried. + last_error: Option<String>, +} + +impl CodexProgress { + /// The stream of a session running in `worktree`. + pub fn in_worktree(worktree: &Path) -> Self { + CodexProgress { + cwd: worktree.display().to_string(), + ..CodexProgress::default() + } + } + + /// Track an item's state, and give its line the first time it is seen. + fn item(&mut self, item: &Value) -> Vec<String> { + let id = item["id"].as_str().unwrap_or("").to_string(); + match item["type"].as_str() { + Some("command_execution") => { + self.running.retain(|(running, _)| *running != id); + if item["status"] == "in_progress" { + let command = unwrapped(item["command"].as_str().unwrap_or("")); + self.running.push((id.clone(), command)); + } + } + Some("agent_message") => { + if let Some(text) = item["text"].as_str() { + self.final_message = Some(text.to_string()); + } + } + Some("error") => { + if let Some(message) = item["message"].as_str() { + self.last_error = Some(message.to_string()); + } + } + _ => {} + } + if !self.seen.insert(id) { + return Vec::new(); + } + self.lines_for(item) + } + + /// The progress lines an item gives. + fn lines_for(&self, item: &Value) -> Vec<String> { + let text = |key: &str| item[key].as_str().unwrap_or("?"); + match item["type"].as_str() { + Some("command_execution") => vec![bash(&unwrapped(text("command")))], + Some("file_change") => item["changes"] + .as_array() + .into_iter() + .flatten() + .map(|change| { + let path = change["path"].as_str().unwrap_or("?"); + let kind = match change["kind"].as_str() { + Some("add") => "Add", + Some("delete") => "Delete", + _ => "Edit", + }; + format!("{kind} {}", shorten(self.relative(path))) + }) + .collect(), + Some("mcp_tool_call") => vec![format!("{} {}", text("server"), text("tool"))], + Some("web_search") => vec![format!("web search {}", shorten(text("query")))], + Some("collab_tool_call") => vec![format!("agent {}", text("tool"))], + _ => Vec::new(), + } + } + + /// `path` relative to the worktree, if it is inside it. + fn relative<'a>(&self, path: &'a str) -> &'a str { + path.strip_prefix(&self.cwd) + .and_then(|rest| rest.strip_prefix('/')) + .unwrap_or(path) + } +} + +impl Stream for CodexProgress { + fn condense(&mut self, raw: &str) -> Vec<String> { + let Ok(event) = serde_json::from_str::<Value>(raw) else { + return Vec::new(); + }; + match event["type"].as_str() { + Some("thread.started") if self.session_id.is_none() => { + self.session_id = event["thread_id"].as_str().map(String::from); + vec!["session started".to_string()] + } + Some("item.started" | "item.updated" | "item.completed") => self.item(&event["item"]), + Some("turn.completed") => { + let usage = &event["usage"]; + let count = |key: &str| usage[key].as_u64(); + if let (Some(input), Some(output)) = (count("input_tokens"), count("output_tokens")) + { + let cached = count("cached_input_tokens").unwrap_or(0); + self.tokens = Some((input, cached, output)); + } + self.killed = self + .running + .iter() + .map(|(_, command)| command.clone()) + .collect(); + Vec::new() + } + Some("turn.failed") => { + if self.failure.is_none() { + let message = event["error"]["message"].as_str().unwrap_or("turn failed"); + self.failure = Some(message.to_string()); + } + Vec::new() + } + Some("error") => { + if let Some(message) = event["message"].as_str() { + self.last_error = Some(message.to_string()); + } + Vec::new() + } + _ => Vec::new(), + } + } + + /// The token totals from `turn.completed`: Codex reports no turns and no + /// cost. + fn summary(&self) -> Option<String> { + let (input, cached, output) = self.tokens?; + Some(format!( + "{input} input tokens ({cached} cached), {output} output tokens" + )) + } + + /// The text of the last `agent_message` item. + fn final_message(&self) -> Option<&str> { + self.final_message.as_deref() + } + + /// The thread id from `thread.started`, which `codex exec resume` takes. + fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + + /// The commands still running when the turn completed. + fn killed_background_work(&self) -> Vec<&str> { + if self.failure.is_some() { + return Vec::new(); + } + self.killed.iter().map(String::as_str).collect() + } + + fn failed(&self) -> bool { + self.failure.is_some() + } + + fn error(&self) -> Option<&str> { + self.failure.as_deref().or(self.last_error.as_deref()) + } +} + +/// `command` without the shell Codex wraps it in, as in `/bin/bash -lc 'git +/// push'`, if it is wrapped so. +fn unwrapped(command: &str) -> String { + let mut words = command.splitn(3, ' '); + let (Some(shell), Some(flag), Some(script)) = (words.next(), words.next(), words.next()) else { + return command.to_string(); + }; + let is_shell = ["bash", "sh", "zsh"] + .iter() + .any(|name| shell == *name || shell.ends_with(&format!("/{name}"))); + if !is_shell || !matches!(flag, "-c" | "-lc") { + return command.to_string(); + } + if let Some(quoted) = script + .strip_prefix('\'') + .and_then(|rest| rest.strip_suffix('\'')) + { + return quoted.replace("'\\''", "'"); + } + if let Some(quoted) = script + .strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + { + return quoted.replace("\\\"", "\""); + } + script.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn lines(events: &[Value]) -> (CodexProgress, Vec<Vec<String>>) { + let mut progress = CodexProgress::in_worktree(Path::new("/work/widgets-issue-7")); + let lines = events + .iter() + .map(|event| progress.condense(&event.to_string())) + .collect(); + (progress, lines) + } + + fn item(kind: &str, id: &str, item: Value) -> Value { + let mut item = item; + item["id"] = json!(id); + json!({ "type": kind, "item": item }) + } + + fn command(kind: &str, id: &str, command: &str, status: &str) -> Value { + item( + kind, + id, + json!({ "type": "command_execution", "command": command, "status": status }), + ) + } + + fn completed(input: u64, output: u64) -> Value { + json!({ "type": "turn.completed", "usage": { + "input_tokens": input, "cached_input_tokens": 100, "output_tokens": output + } }) + } + + #[test] + fn the_first_thread_started_starts_the_session_and_gives_its_id() { + let (progress, lines) = lines(&[ + json!({ "type": "thread.started", "thread_id": "019a-thread" }), + json!({ "type": "turn.started" }), + ]); + + assert_eq!(lines, [vec!["session started".to_string()], vec![]]); + assert_eq!(progress.session_id(), Some("019a-thread")); + } + + #[test] + fn a_command_gets_one_line_without_its_shell_naming_commits_and_pushes() { + let (_, lines) = lines(&[ + command( + "item.started", + "item_1", + "/bin/bash -lc 'cargo test'", + "in_progress", + ), + command( + "item.completed", + "item_1", + "/bin/bash -lc 'cargo test'", + "completed", + ), + command( + "item.started", + "item_2", + "bash -lc 'git add -A && git commit -m '\\''x'\\'' && git push'", + "in_progress", + ), + command("item.completed", "item_3", "git push", "completed"), + ]); + + assert_eq!( + lines, + [ + vec!["$ cargo test".to_string()], + vec![], + vec!["commit and push".to_string()], + vec!["push".to_string()], + ] + ); + } + + #[test] + fn file_changes_tool_calls_and_searches_get_a_line_each() { + let (_, lines) = lines(&[ + item( + "item.completed", + "item_1", + json!({ "type": "file_change", "status": "completed", "changes": [ + { "path": "/work/widgets-issue-7/src/run.rs", "kind": "update" }, + { "path": "/work/widgets-issue-7/NOTES.md", "kind": "add" }, + { "path": "/etc/hosts", "kind": "delete" } + ] }), + ), + item( + "item.started", + "item_2", + json!({ "type": "mcp_tool_call", "server": "github", "tool": "get_issue" }), + ), + item( + "item.started", + "item_3", + json!({ "type": "web_search", "query": "codex exec json" }), + ), + item( + "item.started", + "item_4", + json!({ "type": "collab_tool_call", "tool": "spawn_agent" }), + ), + ]); + + assert_eq!( + lines, + [ + vec![ + "Edit src/run.rs".to_string(), + "Add NOTES.md".to_string(), + "Delete /etc/hosts".to_string() + ], + vec!["github get_issue".to_string()], + vec!["web search codex exec json".to_string()], + vec!["agent spawn_agent".to_string()], + ] + ); + } + + #[test] + fn messages_reasoning_and_turn_events_give_no_line() { + let (_, lines) = lines(&[ + item( + "item.completed", + "item_1", + json!({ "type": "reasoning", "text": "Thinking." }), + ), + item( + "item.completed", + "item_2", + json!({ "type": "agent_message", "text": "Done." }), + ), + json!({ "type": "turn.started" }), + completed(10, 2), + ]); + + assert_eq!(lines, [vec![], vec![], vec![], vec![]] as [Vec<String>; 4]); + } + + #[test] + fn the_final_message_is_the_last_agent_message() { + let said = |id: &str, text: &str| { + item( + "item.completed", + id, + json!({ "type": "agent_message", "text": text }), + ) + }; + let (progress, _) = lines(&[said("item_1", "Running tests."), said("item_5", "Done.")]); + + assert_eq!(progress.final_message(), Some("Done.")); + } + + #[test] + fn the_summary_is_the_token_totals() { + let (progress, _) = lines(&[completed(12000, 345)]); + + assert_eq!( + progress.summary().as_deref(), + Some("12000 input tokens (100 cached), 345 output tokens") + ); + assert_eq!(lines(&[]).0.summary(), None); + } + + #[test] + fn a_command_still_running_when_the_turn_completes_is_killed_background_work() { + let (progress, _) = lines(&[ + command( + "item.started", + "item_1", + "/bin/bash -lc 'sleep 188'", + "in_progress", + ), + command("item.started", "item_2", "cargo build", "in_progress"), + command("item.completed", "item_2", "cargo build", "completed"), + completed(10, 2), + ]); + + assert_eq!(progress.killed_background_work(), ["sleep 188"]); + assert!(!progress.failed()); + } + + #[test] + fn a_command_reconciled_as_in_progress_at_turn_end_is_killed_background_work() { + let (progress, _) = lines(&[ + command("item.started", "item_1", "npm run dev", "in_progress"), + command("item.completed", "item_1", "npm run dev", "in_progress"), + completed(10, 2), + ]); + + assert_eq!(progress.killed_background_work(), ["npm run dev"]); + } + + #[test] + fn a_failed_turn_fails_with_its_error_and_kills_no_background_work() { + let (progress, _) = lines(&[ + command("item.started", "item_1", "sleep 188", "in_progress"), + json!({ "type": "error", "message": "Reconnecting... 1/5" }), + json!({ "type": "turn.failed", "error": { "message": "The 'GPT-6.1-Sol' model is not supported." } }), + ]); + + assert!(progress.failed()); + assert_eq!( + progress.error(), + Some("The 'GPT-6.1-Sol' model is not supported.") + ); + assert!(progress.killed_background_work().is_empty()); + } + + #[test] + fn an_error_event_alone_does_not_fail_the_session_but_is_its_last_error() { + let (progress, _) = lines(&[ + json!({ "type": "error", "message": "Reconnecting... 1/5" }), + completed(10, 2), + ]); + + assert!(!progress.failed()); + assert_eq!(progress.error(), Some("Reconnecting... 1/5")); + } + + #[test] + fn unknown_and_malformed_lines_are_skipped() { + let mut progress = CodexProgress::default(); + for raw in [ + "", + "not json", + "[1, 2]", + r#"{"type": "item.started""#, + r#"{"type": "item.started", "item": 3}"#, + r#"{"type": "item.started", "item": {"type": "command_execution"}}"#, + r#"{"type": "turn.completed", "usage": {"input_tokens": "many"}}"#, + ] { + let lines = progress.condense(raw); + assert!(lines.len() <= 1, "{raw}: {lines:?}"); + } + assert_eq!(progress.summary(), None); + } + + #[test] + fn a_command_not_wrapped_in_a_shell_is_kept_as_it_is() { + for command in ["cargo test", "bash script.sh", "bash -lc", "python -c 'x'"] { + assert_eq!(unwrapped(command), command); + } + assert_eq!( + unwrapped("/usr/bin/zsh -c \"echo \\\"hi\\\"\""), + "echo \"hi\"" + ); + } +} diff --git a/src/prompts_page.rs b/src/prompts_page.rs index 39a2421e9..aa22c1540 100644 --- a/src/prompts_page.rs +++ b/src/prompts_page.rs @@ -13,10 +13,10 @@ use include_dir::{Dir, File}; use crate::ci::FailedChecks; use crate::github::{Check, CheckState}; -use crate::harness::Choice; +use crate::harness::{Choice, Harness}; use crate::issue::IssueUrl; use crate::prompt; -use crate::session::claude_args; +use crate::session::{claude_args, codex_args}; use crate::skills::SKILLS; use Sender::{ArchitectRun, Units}; @@ -58,7 +58,9 @@ const BACKGROUND_WORK: &str = "<background work>"; const FOCUS: &str = "<focus>"; const SESSION_ID: &str = "<session id>"; const PROMPT: &str = "<prompt>"; -const PLACEHOLDERS: [&str; 15] = [ +const MODEL: &str = "<model>"; +const EFFORT: &str = "<effort>"; +const PLACEHOLDERS: [&str; 17] = [ ISSUE_URL, SPEC_URL, NUMBER, @@ -74,6 +76,8 @@ const PLACEHOLDERS: [&str; 15] = [ FOCUS, SESSION_ID, PROMPT, + MODEL, + EFFORT, ]; /// Who sends a prompt. @@ -259,7 +263,7 @@ fn readme(prompts: &[Prompt]) -> String { .find(|prompt| prompt.id == "prompt-resume") .expect("there is a Resume prompt"); let mut readme = format!( - "<!-- Generated by src/prompts_page.rs from the prompts' titles and when sentences in src/prompts_page.rs and the claude arguments in src/session.rs; don't edit. Regenerate with {UPDATE} --> + "<!-- Generated by src/prompts_page.rs from the prompts' titles and when sentences in src/prompts_page.rs and the claude and codex arguments in src/session.rs; don't edit. Regenerate with {UPDATE} --> # Session prompts @@ -277,12 +281,15 @@ A [Resume]({resume_file}) continues the session that ended: {resume} ``` +{codex} + | Prompt | When it is sent | File | | --- | --- | --- | ", new = command(None), resume = command(Some(SESSION_ID)), resume_file = file_name(resume), + codex = codex_note(|code| format!("`{code}`")), ); for prompt in prompts { let _ = writeln!( @@ -354,6 +361,33 @@ fn command(resume: Option<&str>) -> String { format!("claude {}", claude_args(&harness, resume, PROMPT).join(" ")) } +/// The `codex` command line a session on a Model and Effort starts with, +/// with the page's placeholders. +fn codex_command() -> String { + let harness = Choice { + harness: Harness::Codex, + model: Some(MODEL.to_string()), + effort: Some(EFFORT.to_string()), + ..Choice::default() + }; + format!("codex {}", codex_args(&harness, None, PROMPT).join(" ")) +} + +/// The note on how a session runs on Codex instead, with `code` marking up +/// each piece of code in it. +fn codex_note(code: impl Fn(&str) -> String) -> String { + format!( + "With {harness}, each session runs {command} instead, without {model} or the Effort's {c} where they are not set, and with {resume} before the prompt for a Resume. The Factory skills are linked into {dir}, and a prompt's first line loads its skill as {sigil}.", + harness = code("harness codex"), + command = code(&codex_command()), + model = code(&format!("-m {MODEL}")), + c = code("-c"), + resume = code(&format!("resume {SESSION_ID}")), + dir = code(".agents/skills/"), + sigil = code("$thirdshift-<skill>"), + ) +} + fn command_line(html: &mut String) { let _ = write!( html, @@ -364,11 +398,13 @@ fn command_line(html: &mut String) { <pre class="job-text" aria-label="A new session"><code>{new}</code></pre> <p class="lede">A <a href="#prompt-resume">Resume</a> continues the session that ended:</p> <pre class="job-text" aria-label="A Resume"><code>{resume}</code></pre> + <p class="lede">{codex}</p> </div> </section> "##, new = highlighted(&command(None)), resume = highlighted(&command(Some(SESSION_ID))), + codex = codex_note(|code| format!("<code>{}</code>", highlighted(code))), ); } @@ -577,7 +613,7 @@ mod tests { } assert!( checked_in == rendered, - "{} is out of date with the prompts, the claude arguments or the skills:\n{}\nRegenerate it with: {UPDATE}", + "{} is out of date with the prompts, the claude and codex arguments or the skills:\n{}\nRegenerate it with: {UPDATE}", page_path.display(), diff(checked_in, &rendered, before.lines().count() + 2), ); diff --git a/src/run.rs b/src/run.rs index d35bcbe00..fab341e20 100644 --- a/src/run.rs +++ b/src/run.rs @@ -111,9 +111,10 @@ impl<'a> StartedBy<'a> { } /// [`run`] the Run on `issue` that `started_by` started and that is asked -/// `asks`, to its end. Its Run notification, if `asks` has it send one, is -/// for what started the Run to send, once it has ended. -pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended { +/// `asks`, to its end, with the Model and Effort in `asks` settled on the +/// Harness's names for them once checked. Its Run notification, if `asks` +/// has it send one, is for what started the Run to send, once it has ended. +pub fn run_to_end(issue: &IssueUrl, asks: &mut Asks, started_by: StartedBy) -> Ended { let mut base_fix = BaseFix::new( started_by.child(), asks.base_fix.clone(), @@ -139,8 +140,9 @@ pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended /// The Launch directory is opened first, with its Origin match, the check /// that `issue` is open and the git identity check. A Run started by its /// command then checks the Harness, Model and Effort its sessions run on, as -/// `asks` say: one started by another thirdshift, or dispatched by a pass, -/// runs on what that command checked. The rest is [`start`], through the +/// `asks` say, settling the Model and Effort in them, and in `base_fix`, on +/// the Harness's names: one started by another thirdshift, or dispatched by +/// a pass, runs on what that command checked. The rest is [`start`], through the /// Launch directory, GitHub and the logs. /// /// `base_fix` is the one Base fix the Run, or a Spec run for its Spec PR, may @@ -148,14 +150,16 @@ pub fn run_to_end(issue: &IssueUrl, asks: &Asks, started_by: StartedBy) -> Ended /// `asks` ask about one is already in it. fn run( issue: &IssueUrl, - asks: &Asks, + asks: &mut Asks, started_by: StartedBy, base_fix: &mut BaseFix, ) -> Result<Reached, FailedRun> { let directory = LaunchDirectory::open_for_run(issue)?; if let StartedBy::Command = started_by { asks.harness.check()?; + base_fix.runs_on(asks.harness.clone()); } + let asks = &*asks; let mut outside = LaunchAndGitHub { directory: &directory, issue, diff --git a/src/session.rs b/src/session.rs index 5497338bd..a6e50c8b3 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,4 +1,4 @@ -//! Headless Claude Code sessions and their logs. +//! Headless agent sessions, on Claude Code or Codex, and their logs. use std::cell::RefCell; use std::fs::{self, File}; @@ -11,11 +11,11 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail}; -use crate::harness::Choice; +use crate::harness::{Choice, Harness}; use crate::interrupt; use crate::issue::{IssueUrl, Repo}; use crate::logs; -use crate::progress::{self, Progress}; +use crate::progress::{self, Stream}; use crate::prompt; use crate::skills; @@ -86,7 +86,7 @@ impl<'a> Sessions<'a> { harness: &'a Choice, steps: impl FnOnce(&Self) -> Result<T>, ) -> (Result<T>, Option<PathBuf>) { - if let Err(error) = skills::link_into(worktree) { + if let Err(error) = skills::link_into(worktree, harness.harness) { return (Err(error), None); } let sessions = Sessions { @@ -149,12 +149,14 @@ impl<'a> Sessions<'a> { /// Run one session as `kind`, logged under its own path, which becomes /// the last log. - fn start(&self, kind: &str, resume: Option<&str>, prompt: &str) -> Result<Progress> { + fn start(&self, kind: &str, resume: Option<&str>, prompt: &str) -> Result<Box<dyn Stream>> { let log = self.logs.path(kind); progress::step(format_args!("logging the session to {}", log.display())); *self.last_log.borrow_mut() = Some(log.clone()); - let args = claude_args(self.harness, resume, prompt); - run(kind, self.worktree, &args, &log) + let harness = self.harness.harness; + let args = session_args(self.harness, resume, prompt); + let stream = progress::for_harness(harness, self.worktree); + run(kind, harness, self.worktree, &args, &log, stream) } } @@ -170,19 +172,29 @@ fn ending_with(killed: &[&str]) -> String { } } -/// Run `claude` with `args`, as [`claude_args`] gives them, in `worktree`, -/// where it finds the Factory skills, streaming its output to `log` and -/// condensing it to progress lines on stderr, each labelled `kind`. Returns -/// what the stream showed once `claude` has exited cleanly. An interrupt -/// stops the session and fails with `interrupted`. -fn run(kind: &str, worktree: &Path, args: &[&str], log: &Path) -> Result<Progress> { +/// Run `harness`'s CLI with `args`, as [`session_args`] gives them, in +/// `worktree`, where it finds the Factory skills, streaming its output to +/// `log` and condensing it through `stream` to progress lines on stderr, +/// each labelled `kind`. Returns what the stream showed once the CLI has +/// exited cleanly, its turn not failed. Otherwise fails with the error the +/// stream gave, if any. An interrupt stops the session and fails with +/// `interrupted`. +fn run( + kind: &str, + harness: Harness, + worktree: &Path, + args: &[String], + log: &Path, + mut stream: Box<dyn Stream>, +) -> Result<Box<dyn Stream>> { + let cli = harness.name(); if let Some(dir) = log.parent() { fs::create_dir_all(dir).with_context(|| format!("could not create {}", dir.display()))?; } let mut log_file = File::create(log).with_context(|| format!("could not create {}", log.display()))?; let started = Instant::now(); - let mut child = Command::new("claude") + let mut child = Command::new(cli) .args(args) .current_dir(worktree) .stdin(Stdio::null()) @@ -191,60 +203,96 @@ fn run(kind: &str, worktree: &Path, args: &[&str], log: &Path) -> Result<Progres // can stop everything it started. .process_group(0) .spawn() - .context("could not run claude")?; + .with_context(|| format!("could not run {cli}"))?; // Follow the stream on its own thread, so this one can watch for an // interrupt while the session runs. - let stream = child.stdout.take().context("no stdout from claude")?; + let output = child + .stdout + .take() + .with_context(|| format!("no stdout from {cli}"))?; let group = -(child.id() as libc::pid_t); let (kind_owned, log_owned) = (kind.to_string(), log.to_path_buf()); let follower = thread::spawn(move || { - let mut progress = Progress::default(); let followed = follow( &kind_owned, - stream, + output, &mut log_file, &log_owned, - &mut progress, + stream.as_mut(), ); if followed.is_err() { // Nothing reads its output any more, so it could block forever. // SAFETY: kill has no memory-safety preconditions. unsafe { libc::kill(group, libc::SIGKILL) }; } - (followed, progress) + (followed, stream) }); let status = loop { if interrupt::requested() { stop(&mut child); bail!("interrupted"); } - if let Some(status) = child.try_wait().context("could not wait for claude")? { + if let Some(status) = child + .try_wait() + .with_context(|| format!("could not wait for {cli}"))? + { break status; } thread::sleep(POLL); }; - let (followed, progress) = follower + let (followed, stream) = follower .join() .map_err(|_| anyhow!("the session stream reader panicked"))?; let elapsed = minutes_and_seconds(started.elapsed()); - let summary = progress + let summary = stream .summary() .map_or(String::new(), |summary| format!(": {summary}")); progress::step(format_args!( "{kind}: session ended after {elapsed}{summary}" )); followed?; - if !status.success() { - bail!( - "claude exited {}", + if status.success() && !stream.failed() { + return Ok(stream); + } + let ended = if status.success() { + "'s turn failed".to_string() + } else { + format!( + " exited {}", status .code() .map_or("by signal".to_string(), |code| code.to_string()) - ); + ) + }; + match stream.error() { + Some(error) => bail!("{cli}{ended}: {error}"), + None => bail!("{cli}{ended}"), + } +} + +/// The arguments a session runs `harness`'s CLI with, as [`claude_args`] or +/// [`codex_args`] give them, its prompt loading its skill with that +/// Harness's sigil. +fn session_args(harness: &Choice, resume: Option<&str>, prompt: &str) -> Vec<String> { + match harness.harness { + Harness::Claude => claude_args(harness, resume, prompt) + .into_iter() + .map(String::from) + .collect(), + Harness::Codex => codex_args(harness, resume, &codex_prompt(prompt)), + } +} + +/// `prompt` as Codex takes it: a first line that loads a Factory skill, +/// `/thirdshift-<skill>` as Claude's prompts write it, loads it as +/// `$thirdshift-<skill>`. +pub fn codex_prompt(prompt: &str) -> String { + match prompt.strip_prefix("/thirdshift-") { + Some(rest) => format!("$thirdshift-{rest}"), + None => prompt.to_string(), } - Ok(progress) } /// The arguments every session runs `claude` with: headless in auto mode, @@ -265,6 +313,33 @@ pub fn claude_args<'a>( args } +/// The arguments every session runs `codex` with: `exec`, streaming JSONL, +/// with no approvals and no sandbox (ADR-0012), on the Model and Effort +/// `harness` sets, if any, reading `CLAUDE.md` where a directory has no +/// `AGENTS.md`. With `resume`, the session with that id continues, given +/// every one of those again, as Codex keeps none of them. The prompt comes +/// last. +pub fn codex_args(harness: &Choice, resume: Option<&str>, prompt: &str) -> Vec<String> { + let mut args: Vec<String> = [ + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + ] + .map(String::from) + .to_vec(); + args.extend(harness.codex_args()); + args.extend(["-c".to_string(), CLAUDE_MD_FALLBACK.to_string()]); + if let Some(session_id) = resume { + args.extend(["resume".to_string(), session_id.to_string()]); + } + args.push(prompt.to_string()); + args +} + +/// The config setting that has Codex read `CLAUDE.md` where a directory has +/// no `AGENTS.md`. +const CLAUDE_MD_FALLBACK: &str = r#"project_doc_fallback_filenames=["CLAUDE.md"]"#; + const POLL: Duration = Duration::from_millis(100); /// How long a session gets to exit after SIGTERM before it is killed. @@ -295,7 +370,7 @@ fn follow( stream: impl Read, log_file: &mut File, log: &Path, - progress: &mut Progress, + progress: &mut dyn Stream, ) -> Result<()> { let mut stream = BufReader::new(stream); let mut line = Vec::new(); diff --git a/src/skills.rs b/src/skills.rs index 3043887ec..7a6d646fb 100644 --- a/src/skills.rs +++ b/src/skills.rs @@ -1,6 +1,6 @@ //! The Factory skills, embedded at compile time (ADR-0001), written out once -//! per Command and linked into each worktree its sessions run in, where -//! Claude Code finds project skills (ADR-0012). +//! per Command and linked into each worktree its sessions run in, where the +//! Harness finds project skills (ADR-0012). use std::fs::{self, OpenOptions}; use std::io::{ErrorKind, Write}; @@ -13,29 +13,34 @@ use include_dir::{Dir, include_dir}; use tempfile::TempDir; use crate::git::Git; +use crate::harness::Harness; /// The Factory skills, as they are written out and the Prompts and skills /// page shows them, each in its `thirdshift-<skill>` directory. pub static SKILLS: Dir = include_dir!("$CARGO_MANIFEST_DIR/skills"); -/// Where in a worktree Claude Code finds its project skills. -const PROJECT_SKILLS: &str = ".claude/skills"; - -/// The `.git/info/exclude` pattern that keeps the linked skills out of git. -const EXCLUDE: &str = "/.claude/skills/thirdshift-*"; +/// Where in a worktree `harness` finds its project skills. +fn project_skills(harness: Harness) -> &'static str { + match harness { + Harness::Claude => ".claude/skills", + Harness::Codex => ".agents/skills", + } +} /// The temp directory the Command wrote the Factory skills out to, once it /// has. static WRITTEN: Mutex<Option<TempDir>> = Mutex::new(None); -/// Link every Factory skill into `worktree`'s `.claude/skills/`, writing them -/// out first if the Command has not yet, and make sure the repository's -/// `.git/info/exclude` keeps them out of git. The links are left in place: -/// they go with the worktree. -pub fn link_into(worktree: &Path) -> Result<()> { +/// Link every Factory skill into `worktree`'s project skills for `harness`, +/// `.claude/skills/` or `.agents/skills/`, writing them out first if the +/// Command has not yet, and make sure the repository's `.git/info/exclude` +/// keeps them out of git. The links are left in place: they go with the +/// worktree. +pub fn link_into(worktree: &Path, harness: Harness) -> Result<()> { let written = written_out()?; - exclude(&Git::new(worktree))?; - let project_skills = worktree.join(PROJECT_SKILLS); + let dir = project_skills(harness); + exclude(&Git::new(worktree), &format!("/{dir}/thirdshift-*"))?; + let project_skills = worktree.join(dir); fs::create_dir_all(&project_skills) .with_context(|| format!("could not create {}", project_skills.display()))?; for skill in SKILLS.dirs() { @@ -80,11 +85,11 @@ fn written_out() -> Result<PathBuf> { Ok(path) } -/// Add [`EXCLUDE`] to the `.git/info/exclude` of the repository `git` works -/// in, unless it is there already. The file is shared by every worktree, so -/// the entry is left there, under a lock other Runs from the same Launch -/// directory take too. -fn exclude(git: &Git) -> Result<()> { +/// Add the pattern `exclude` to the `.git/info/exclude` of the repository +/// `git` works in, unless it is there already. The file is shared by every +/// worktree, so the entry is left there, under a lock other Runs from the +/// same Launch directory take too. +fn exclude(git: &Git, exclude: &str) -> Result<()> { let _lock = git.lock("thirdshift-exclude.lock")?; let info = git.common_dir()?.join("info"); let path = info.join("exclude"); @@ -95,7 +100,7 @@ fn exclude(git: &Git) -> Result<()> { return Err(error).with_context(|| format!("could not read {}", path.display())); } }; - if existing.lines().any(|line| line == EXCLUDE) { + if existing.lines().any(|line| line == exclude) { return Ok(()); } fs::create_dir_all(&info).with_context(|| format!("could not create {}", info.display()))?; @@ -108,6 +113,6 @@ fn exclude(git: &Git) -> Result<()> { .create(true) .append(true) .open(&path) - .and_then(|mut file| writeln!(file, "{separator}{EXCLUDE}")) + .and_then(|mut file| writeln!(file, "{separator}{exclude}")) .with_context(|| format!("could not add the Factory skills to {}", path.display())) } diff --git a/tests/codex_sessions.rs b/tests/codex_sessions.rs new file mode 100644 index 000000000..02f54ffa2 --- /dev/null +++ b/tests/codex_sessions.rs @@ -0,0 +1,422 @@ +//! A Command's sessions on Codex, with `harness codex`: each runs `codex exec +//! --json` unsandboxed in the Run's worktree, with stdin set to null, on the +//! Model and Effort as Codex names them, reading `CLAUDE.md` where there is +//! no `AGENTS.md`, with the Factory skills linked into the worktree's +//! `.agents/skills/` and its prompt loading its skill as +//! `$thirdshift-<skill>`. Codex's stream gives the progress lines, the +//! Session log, the session id a Resume continues and the final message, and +//! a failed turn fails the Run with Codex's error. The Model and Effort are +//! checked against `codex debug models` before any work. + +mod support; + +use std::fs; + +use serde_json::Value; +use support::{RunResult, Scenario}; + +/// The agent commits its work and opens a PR that closes issue #7. +const AGENT_OPENS_PR: &str = r#" +echo "feature" > feature.txt +git add feature.txt +git commit -q -m "Add feature" +gh pr create --base main --head issue-7 --title "Add feature" --body "Closes #7" +"#; + +/// The scenario's repository's logs, relative to the scenario. +const LOGS: &str = "home/.thirdshift/logs/acme/widgets"; + +/// Bash that has the agent run `command` as Codex streams it, wrapped in its +/// shell, still running when its turn ends. +fn leaves_running(command: &str) -> String { + format!( + r#"echo '{{"type": "item.started", "item": {{"id": "item_9", "type": "command_execution", "command": "/bin/bash -lc '\''{command}'\''", "status": "in_progress"}}}}' +"# + ) +} + +/// Bash that has the agent run `command` to completion, as Codex streams it. +fn runs(command: &str) -> String { + format!( + r#"echo '{{"type": "item.started", "item": {{"id": "item_1", "type": "command_execution", "command": "{command}", "status": "in_progress"}}}}' +echo '{{"type": "item.completed", "item": {{"id": "item_1", "type": "command_execution", "command": "{command}", "status": "completed", "exit_code": 0}}}}' +"# + ) +} + +/// Bash that sets the check runs on the worktree's HEAD to `checks`. +fn checks_on_head(checks: &str) -> String { + format!("gh fake checks \"$(git rev-parse HEAD)\" '{checks}'\n") +} + +const RED: &str = + r#"[{"name": "test", "conclusion": "failure", "url": "https://ci.example/test"}]"#; +const GREEN: &str = + r#"[{"name": "test", "conclusion": "success", "url": "https://ci.example/test"}]"#; + +fn argv(call: &Value) -> Vec<String> { + call["argv"] + .as_array() + .unwrap() + .iter() + .map(|arg| arg.as_str().unwrap().to_string()) + .collect() +} + +/// The arguments a Codex session on `model` and `effort` starts with, before +/// any `resume` and its prompt. +fn codex_args(model: &str, effort: &str) -> Vec<String> { + [ + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "-m", + model, + "-c", + &format!("model_reasoning_effort=\"{effort}\""), + "-c", + "project_doc_fallback_filenames=[\"CLAUDE.md\"]", + ] + .map(String::from) + .to_vec() +} + +/// Assert every session ran on `codex exec` with the bypass flag, `model` +/// and `effort`, the `CLAUDE.md` fallback and stdin set to null, its prompt +/// last, and found every Factory skill in `.agents/skills/` with none of +/// them in `git status`; that its prompt's first line loads its skill with +/// Codex's sigil, if it loads one; and that none ran on `claude`. +fn assert_sessions_on_codex(scenario: &Scenario, model: &str, effort: &str) { + let calls = scenario.codex_calls(); + assert!(!calls.is_empty(), "no Codex session ran"); + assert!( + scenario.claude_calls().is_empty(), + "a session ran on claude" + ); + for call in &calls { + let args = argv(call); + let prompt = call["prompt"].as_str().unwrap(); + assert_eq!(args.last().map(String::as_str), Some(prompt)); + assert_eq!(args[..9], codex_args(model, effort), "{args:?}"); + assert_eq!(call["stdin_null"], true, "{args:?}"); + assert_eq!(call["git_status"], "", "{args:?}"); + assert!(!prompt.starts_with("/thirdshift-"), "{prompt}"); + } + scenario.assert_every_codex_session_found_the_factory_skills(); +} + +/// The first line of each Codex session's prompt. +fn first_lines(scenario: &Scenario) -> Vec<String> { + scenario + .codex_calls() + .iter() + .map(|call| { + let prompt = call["prompt"].as_str().unwrap(); + prompt.lines().next().unwrap().to_string() + }) + .collect() +} + +#[test] +fn a_runs_implement_session_and_its_repair_run_on_codex_with_the_model_and_effort_as_codex_names_them() + { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session( + 7, + 1, + &format!( + "{}{AGENT_OPENS_PR}{}", + runs("/bin/bash -lc 'cargo test'"), + checks_on_head(RED) + ), + ); + scenario.agent_does_for_in_session( + 7, + 2, + &format!( + "echo fix > fix.txt\ngit add fix.txt\ngit commit -q -m Fix\n{}", + checks_on_head(GREEN) + ), + ); + let url = scenario.issue_url(7); + + let result = scenario.run(&[ + "harness", + "codex", + "model", + "GPT-6.1-Sol", + &url, + "effort", + "Max", + ]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on_codex(&scenario, "gpt-6.1-sol", "max"); + assert_eq!(scenario.codex_calls().len(), 2); + assert_eq!( + first_lines(&scenario)[0], + format!("$thirdshift-implement {url}") + ); + for line in [ + "thirdshift: checking the Model and Effort against codex debug models\n", + "thirdshift: sessions run on codex · gpt-6.1-sol · max\n", + "thirdshift: implement: session started\n", + "thirdshift: implement: $ cargo test\n", + ": 1200 input tokens (200 cached), 300 output tokens\n", + ] { + assert!( + result.stderr.contains(line), + "{line:?} in {}", + result.stderr + ); + } + let body = scenario.gh_state()["prs"][0]["body"] + .as_str() + .unwrap() + .to_string(); + assert!( + body.ends_with("Built with codex · gpt-6.1-sol · max <!-- thirdshift:built-with -->\n"), + "{body}" + ); + let logs = scenario.entries(&format!("{LOGS}/sessions")); + let implement = logs + .iter() + .find(|log| log.ends_with("-implement.jsonl")) + .unwrap_or_else(|| panic!("no implement session log: {logs:?}")); + let log = fs::read_to_string(scenario.path(&format!("{LOGS}/sessions/{implement}"))).unwrap(); + assert!( + log.starts_with(r#"{"type": "thread.started", "thread_id": "fake-thread-1"}"#), + "{log}" + ); + assert!(log.contains(r#""type": "turn.completed""#), "{log}"); +} + +#[test] +fn a_session_left_with_a_command_running_is_resumed_on_its_thread_with_every_flag_again() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session( + 7, + 1, + &format!("{AGENT_OPENS_PR}{}", leaves_running("sleep 188")), + ); + let url = scenario.issue_url(7); + + let result = scenario.run(&[ + "harness", + "codex", + "model", + "gpt-6-luna", + "effort", + "high", + &url, + ]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_sessions_on_codex(&scenario, "gpt-6-luna", "high"); + let calls = scenario.codex_calls(); + assert_eq!(calls.len(), 2); + let resume = argv(&calls[1]); + assert_eq!(resume[9..11], ["resume", "fake-thread-1"], "{resume:?}"); + assert!( + calls[1]["prompt"] + .as_str() + .unwrap() + .starts_with("Your background work (sleep 188) was killed"), + "{resume:?}" + ); + assert!( + result.stderr.contains( + "implement: background work was killed as the session ended; resuming it once\n" + ), + "stderr: {}", + result.stderr + ); +} + +#[test] +fn a_failed_turn_fails_the_run_with_codexs_error_in_the_cause() { + for (script, cause) in [ + ( + "echo \"The 'gpt-6.1-sol' model is not supported.\" > \"$FAKE_CODEX_ERROR\"\nexit 1", + "codex exited 1: The 'gpt-6.1-sol' model is not supported.", + ), + ( + r#"echo '{"type": "turn.failed", "error": {"message": "stream disconnected"}}'"#, + "codex's turn failed: stream disconnected", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, script); + + let result = scenario.run(&["harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!( + result.stderr.contains(cause), + "{cause:?} in {}", + result.stderr + ); + assert_eq!(scenario.codex_calls().len(), 1); + } +} + +#[test] +fn a_retried_error_alone_does_not_fail_the_session() { + let scenario = Scenario::new(); + scenario.agent_does_for( + 7, + &format!( + "echo '{{\"type\": \"error\", \"message\": \"Reconnecting... 1/5\"}}'\n{AGENT_OPENS_PR}" + ), + ); + + let result = scenario.run(&["harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert!( + !result.stderr.contains("checking the Model"), + "a Model was checked with none named: {}", + result.stderr + ); +} + +/// Assert the Run failed with `error` before any work: no Claim, no +/// worktree, no session and no Command log. +fn assert_failed_before_any_work(scenario: &Scenario, result: &RunResult, error: &str) { + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert!( + result.stderr.contains(error), + "expected {error:?} in stderr: {}", + result.stderr + ); + assert!(scenario.codex_calls().is_empty(), "a session ran"); + assert_eq!(scenario.entries("work"), vec!["widgets"]); + assert_eq!( + scenario + .launch_git(&["worktree", "list", "--porcelain"]) + .matches("worktree ") + .count(), + 1 + ); + assert!(scenario.issue_labels(7).is_empty(), "the Claim was made"); + assert!( + !scenario.path(&format!("{LOGS}/commands")).exists(), + "a Command log was kept" + ); +} + +#[test] +fn a_model_or_effort_codex_lacks_fails_before_any_work_naming_the_valid_choices() { + for (model, effort, error) in [ + ( + "gpt-7", + "max", + "thirdshift: the Model gpt-7 is not in Codex's catalog: choose one of gpt-6.1-sol, \ + gpt-6-luna, gpt-5.5\n", + ), + ( + "GPT-5.5", + "Max", + "thirdshift: the Effort Max is not one the Codex Model gpt-5.5 supports: choose one \ + of low, medium, high, xhigh\n", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + + let result = scenario.run(&[ + &scenario.issue_url(7), + "harness", + "codex", + "model", + model, + "effort", + effort, + ]); + + assert_failed_before_any_work(&scenario, &result, error); + } +} + +/// A script in which the agent for issue `issue` commits its work and opens +/// its PR into `base`. +fn agent_opens_pr(issue: u32, base: &str) -> String { + format!( + r#" +echo "{issue}" > issue-{issue}.txt +git add issue-{issue}.txt +git commit -q -m "Work on {issue}" +gh pr create --base {base} --head issue-{issue} --title "Work on {issue}" --body "Closes #{issue}" +"# + ) +} + +#[test] +fn an_architecture_review_and_the_spec_run_it_dispatches_run_every_session_on_codex() { + let scenario = Scenario::new(); + scenario.repo_has_labels(&["needs-triage", "ready-for-agent", "architecture"]); + scenario.agent_does_in_session( + 1, + r#" +spec=$(gh issue create --title "Deepen the session module" --body "The Spec" --label needs-triage) +gh issue create --title "Move the logs" --body "A Ticket" --label ready-for-agent +gh issue create --title "Move the sessions" --body "A Ticket" --label ready-for-agent +gh fake sub-issues 8 '[9, 10]' +printf 'Architecture review plan: %s\n' "$spec" > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + scenario.agent_does_for(9, &agent_opens_pr(9, "issue-8")); + scenario.agent_does_for(10, &agent_opens_pr(10, "issue-8")); + + let result = scenario.run(&["architect", "harness", "codex", "model", "GPT-6-Luna"]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let gh = scenario.gh_state(); + let spec_pr = gh["prs"] + .as_array() + .unwrap() + .iter() + .find(|pr| pr["head"] == "issue-8") + .unwrap_or_else(|| panic!("no Spec PR: {gh}")); + assert_eq!(spec_pr["isDraft"], false); + let calls = scenario.codex_calls(); + assert_eq!(calls.len(), 4, "{calls:?}"); + for call in &calls { + let args = argv(call); + assert_eq!( + args[..7], + [ + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "-m", + "gpt-6-luna", + "-c", + "project_doc_fallback_filenames=[\"CLAUDE.md\"]", + ], + "{args:?}" + ); + assert_eq!(call["stdin_null"], true, "{args:?}"); + assert_eq!(call["git_status"], "", "{args:?}"); + } + assert!(scenario.claude_calls().is_empty()); + scenario.assert_every_codex_session_found_the_factory_skills(); + let mut first_lines = first_lines(&scenario); + assert_eq!( + first_lines.remove(0), + "$thirdshift-improve-codebase-architecture" + ); + assert!( + first_lines[2].starts_with("$thirdshift-code-review main, with the Spec "), + "{first_lines:?}" + ); + first_lines.pop(); + first_lines.sort(); + assert_eq!( + first_lines, + [ + "$thirdshift-implement https://github.com/acme/widgets/issues/10", + "$thirdshift-implement https://github.com/acme/widgets/issues/9", + ] + ); +} diff --git a/tests/fakes.rs b/tests/fakes.rs index eaa999dcc..0b197d18a 100644 --- a/tests/fakes.rs +++ b/tests/fakes.rs @@ -1,7 +1,7 @@ -//! The fake `gh` and `claude` the end-to-end tests put first on PATH: one -//! executable, which acts as whichever its name says. See `fakes/gh.rs` and -//! `fakes/claude.rs` for what each does, and `support/fakes.rs` for how the -//! harness builds it. +//! The fake `gh`, `claude` and `codex` the end-to-end tests put first on +//! PATH: one executable, which acts as whichever its name says. See +//! `fakes/gh.rs`, `fakes/claude.rs` and `fakes/codex.rs` for what each does, +//! and `support/fakes.rs` for how the harness builds it. //! //! It uses only the standard library, so the harness can build it with plain //! `rustc`. It started out as two Python scripts, whose startup cost was about @@ -15,6 +15,8 @@ #[path = "fakes/claude.rs"] mod claude; +#[path = "fakes/codex.rs"] +mod codex; #[path = "fakes/gh.rs"] mod gh; #[path = "fakes/json.rs"] @@ -107,6 +109,7 @@ fn main() { match name { "gh" => gh::main(args.collect()), "claude" => claude::main(args.collect()), + "codex" => codex::main(args.collect()), _ => die(&format!("fake: no fake is called {name}"), 2), } exit(0); diff --git a/tests/fakes/claude.rs b/tests/fakes/claude.rs index 684741aa5..a0b2d3d00 100644 --- a/tests/fakes/claude.rs +++ b/tests/fakes/claude.rs @@ -66,9 +66,10 @@ fn snapshot_into(files: &mut Json, root: &Path, dir: &Path) { } } -/// The `thirdshift-*` entries of the working directory's `.claude/skills/`. -fn linked_skills() -> Vec<PathBuf> { - let Ok(entries) = fs::read_dir(SKILLS) else { +/// The `thirdshift-*` entries of the working directory's `skills`, its +/// project skills. +fn linked_skills(skills: &str) -> Vec<PathBuf> { + let Ok(entries) = fs::read_dir(skills) else { return Vec::new(); }; entries @@ -82,22 +83,22 @@ fn linked_skills() -> Vec<PathBuf> { } /// Every file of the `thirdshift-*` skills in the working directory's -/// `.claude/skills/`, through their links, by its path relative to -/// `.claude/skills/`, with its contents. -fn skills_snapshot() -> Json { +/// `skills`, through their links, by its path relative to `skills`, with its +/// contents. +pub fn skills_snapshot(skills: &str) -> Json { let mut files = object([]); - for skill in linked_skills() { - snapshot_into(&mut files, Path::new(SKILLS), &skill); + for skill in linked_skills(skills) { + snapshot_into(&mut files, Path::new(skills), &skill); } files } /// The files written out beside the skills the `thirdshift-*` links in the -/// working directory's `.claude/skills/` point to, such as their licence, by -/// name, with their contents. -fn beside_skills_snapshot() -> Json { +/// working directory's `skills` point to, such as their licence, by name, +/// with their contents. +pub fn beside_skills_snapshot(skills: &str) -> Json { let mut files = object([]); - let written = linked_skills() + let written = linked_skills(skills) .first() .and_then(|link| fs::read_link(link).ok()) .and_then(|target| target.parent().map(Path::to_owned)); @@ -135,7 +136,7 @@ fn prompt_of(record: &Json) -> &str { } /// The script for the session just recorded last in `records`. -fn script_for(records: &[Json]) -> PathBuf { +pub fn script_for(records: &[Json]) -> PathBuf { let script = crate::env_path("FAKE_CLAUDE_SCRIPT").display().to_string(); let mut candidates = Vec::new(); if let Some(issue) = issue_of(prompt_of(records.last().unwrap())) { @@ -156,7 +157,7 @@ fn script_for(records: &[Json]) -> PathBuf { /// `git` in the working directory, with its stdout trimmed, and whether it /// succeeded. -fn git_here(args: &[&str]) -> (String, bool) { +pub fn git_here(args: &[&str]) -> (String, bool) { let output = crate::git(Path::new("."), args); let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); (stdout, output.status.success()) @@ -164,7 +165,7 @@ fn git_here(args: &[&str]) -> (String, bool) { /// The exit code a Python fake gave for `status`, which is negative for a /// signal and wraps as an exit code. -fn exit_code(status: ExitStatus) -> i32 { +pub fn exit_code(status: ExitStatus) -> i32 { status .code() .unwrap_or_else(|| -status.signal().unwrap() & 0xff) @@ -195,8 +196,8 @@ pub fn main(argv: Vec<String>) { ("cwd", string(cwd)), ("branch", string(branch)), ("merging", Bool(merging)), - ("skill_files", skills_snapshot()), - ("beside_skills", beside_skills_snapshot()), + ("skill_files", skills_snapshot(SKILLS)), + ("beside_skills", beside_skills_snapshot(SKILLS)), ("stdin", stdin), ]), ); diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs new file mode 100644 index 000000000..56b1471c5 --- /dev/null +++ b/tests/fakes/codex.rs @@ -0,0 +1,152 @@ +//! Fake `codex` for the thirdshift test harness. +//! +//! `codex debug models` prints [`CATALOG`], a fixed catalog of Models. +//! +//! `codex exec --json …` is an agent session, scripted as the fake `claude`'s +//! are: it runs the same per-test bash script, chosen the same way from +//! $FAKE_CLAUDE_SCRIPT by the session's issue and count, with the sessions +//! counted from its own record. Each call appends a record to the JSON list +//! in $FAKE_CODEX_RECORD: its argv, prompt, working directory, the branch +//! checked out there, whether its stdin was `/dev/null`, `git status +//! --porcelain` there, a snapshot of the `thirdshift-*` skills in its +//! `.agents/skills/`, through their links, and of the files written out +//! beside them. +//! +//! It emits `thread.started`, with thread id `fake-thread-<n>` for the n-th +//! call, or none if $FAKE_CLAUDE_NO_SESSION_ID is set, then `turn.started`, +//! then runs the script, whose stdout joins the stream. What the script +//! writes to $FAKE_CLAUDE_FINAL_MESSAGE becomes an `agent_message` item. A +//! script that succeeds ends the turn with `turn.completed`, and one that +//! fails with `turn.failed`, whose error is what it wrote to +//! $FAKE_CODEX_ERROR, else a default, and exits 1, as Codex does. + +use std::fs; +use std::io::Write; +use std::os::unix::fs::MetadataExt; +use std::process::Command; + +use crate::claude::{beside_skills_snapshot, exit_code, git_here, script_for, skills_snapshot}; +use crate::json::{Array, Bool, Json, Null, object, string}; + +/// Where Codex finds a worktree's project skills. +const SKILLS: &str = ".agents/skills"; + +/// What `codex debug models` prints: a few Models, each with its display +/// name and the Efforts it supports. +const CATALOG: &str = r#"{"models": [ + {"slug": "gpt-6.1-sol", "display_name": "GPT-6.1-Sol", "visibility": "list", "default_reasoning_level": "low", + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "medium"}, {"effort": "high"}, {"effort": "xhigh"}, {"effort": "max"}, {"effort": "ultra"}]}, + {"slug": "gpt-6-luna", "display_name": "GPT-6-Luna", "visibility": "list", "default_reasoning_level": "medium", + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "medium"}, {"effort": "high"}, {"effort": "xhigh"}, {"effort": "max"}]}, + {"slug": "gpt-5.5", "display_name": "GPT-5.5", "visibility": "list", "default_reasoning_level": "medium", + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "medium"}, {"effort": "high"}, {"effort": "xhigh"}]} +]}"#; + +/// Whether stdin is `/dev/null`. +fn stdin_is_null() -> bool { + let (Ok(stdin), Ok(null)) = (fs::metadata("/proc/self/fd/0"), fs::metadata("/dev/null")) else { + return false; + }; + stdin.rdev() == null.rdev() && stdin.ino() == null.ino() +} + +/// Print one line of the stream. +fn emit(event: Json) { + println!("{event}"); + std::io::stdout().flush().unwrap(); +} + +pub fn main(argv: Vec<String>) { + match argv.first().map(String::as_str) { + Some("debug") if argv.get(1).map(String::as_str) == Some("models") => { + println!("{CATALOG}"); + crate::exit(0); + } + Some("exec") if argv.iter().any(|arg| arg == "--json") => {} + _ => crate::die( + &format!( + "fake codex: unexpected arguments {}", + crate::python_list(&argv) + ), + 2, + ), + } + let (branch, _) = git_here(&["branch", "--show-current"]); + let (status, _) = git_here(&["status", "--porcelain"]); + let record_path = &crate::env_path("FAKE_CODEX_RECORD"); + + let lock = crate::lock_beside(record_path); + let cwd = std::env::current_dir().unwrap(); + let cwd = cwd.to_str().unwrap(); + let records = crate::append_record( + record_path, + object([ + ("argv", Array(argv.iter().map(string).collect())), + ("prompt", argv.last().map(string).unwrap_or(Null)), + ("cwd", string(cwd)), + ("branch", string(branch)), + ("stdin_null", Bool(stdin_is_null())), + ("git_status", string(status)), + ("skill_files", skills_snapshot(SKILLS)), + ("beside_skills", beside_skills_snapshot(SKILLS)), + ]), + ); + let records = records.items(); + let script = script_for(records); + drop(lock); + + let session = records.len(); + if std::env::var_os("FAKE_CLAUDE_NO_SESSION_ID").is_none() { + emit(object([ + ("type", string("thread.started")), + ("thread_id", string(format!("fake-thread-{session}"))), + ])); + } + emit(object([("type", string("turn.started"))])); + // Beside the record, not in $TMPDIR, which tests expect to be left empty. + let final_message = format!("{}.final-message.{session}", record_path.display()); + let error = format!("{}.error.{session}", record_path.display()); + let status = Command::new("bash") + .arg("-e") + .arg(&script) + .env("FAKE_CLAUDE_FINAL_MESSAGE", &final_message) + .env("FAKE_CODEX_ERROR", &error) + .status() + .unwrap(); + let code = exit_code(status); + if let Ok(message) = fs::read_to_string(&final_message) { + emit(object([ + ("type", string("item.completed")), + ( + "item", + object([ + ("id", string("item_final")), + ("type", string("agent_message")), + ("text", string(message)), + ]), + ), + ])); + } + if code == 0 { + emit(object([ + ("type", string("turn.completed")), + ( + "usage", + object([ + ("input_tokens", crate::json::number(1200)), + ("cached_input_tokens", crate::json::number(200)), + ("output_tokens", crate::json::number(300)), + ]), + ), + ])); + crate::exit(0); + } + let message = fs::read_to_string(&error) + .map(|message| message.trim_end().to_string()) + .unwrap_or_else(|_| format!("the agent's script exited {code}")); + emit(object([ + ("type", string("turn.failed")), + ("error", object([("message", string(message))])), + ])); + crate::exit(1); +} diff --git a/tests/harness_choice.rs b/tests/harness_choice.rs index 3ddc4677b..60cdca151 100644 --- a/tests/harness_choice.rs +++ b/tests/harness_choice.rs @@ -278,35 +278,6 @@ fn a_harness_missing_from_path_fails_the_run_before_any_work_naming_what_chose_i } } -#[test] -fn harness_codex_fails_the_run_before_any_work_as_not_supported_yet() { - for (args, config, chosen_by) in [ - (vec!["harness", "codex"], None, "the harness flag"), - ( - vec![], - Some("[harness]\ndefault = \"codex\"\n"), - "harness.default in the User config", - ), - ] { - let scenario = scenario(); - if let Some(config) = config { - scenario.user_config_is(config); - } - let url = scenario.issue_url(7); - let mut args = args; - args.push(&url); - - let result = scenario.run(&args); - - assert_failed_before_any_work( - &scenario, - &result, - &format!("harness codex, chosen by {chosen_by}, is not supported yet"), - ); - assert!(scenario.claude_calls().is_empty()); - } -} - #[test] fn the_command_log_the_activity_log_and_the_pr_body_name_the_harness_model_and_effort() { let scenario = scenario(); diff --git a/tests/support/fakes.rs b/tests/support/fakes.rs index 6a506d892..b851c6b75 100644 --- a/tests/support/fakes.rs +++ b/tests/support/fakes.rs @@ -1,4 +1,4 @@ -//! The fake `gh` and `claude`, built from `tests/fakes.rs` with plain +//! The fake `gh`, `claude` and `codex`, built from `tests/fakes.rs` with plain //! `rustc` the first time a test needs them and kept in Cargo's scratch //! directory for tests. They aren't a binary target, so neither the crate nor //! a release ships them. @@ -11,16 +11,17 @@ use std::sync::OnceLock; /// The fakes' sources. They name the built executable, and including them /// makes Cargo rebuild the tests when they change. -const SOURCES: [&str; 4] = [ +const SOURCES: [&str; 5] = [ include_str!("../fakes.rs"), include_str!("../fakes/gh.rs"), include_str!("../fakes/claude.rs"), + include_str!("../fakes/codex.rs"), include_str!("../fakes/json.rs"), ]; -/// Put the fake `gh` and `claude` in `bin`. +/// Put the fake `gh`, `claude` and `codex` in `bin`. pub fn install(bin: &Path) { - for name in ["gh", "claude"] { + for name in ["gh", "claude", "codex"] { std::os::unix::fs::symlink(executable(), bin.join(name)).unwrap(); } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index d3afd916b..f593337b0 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -1,5 +1,5 @@ //! Test harness: runs the compiled `thirdshift` binary against real git and -//! fake `gh` and `claude` executables, and, for email, a stand-in for Resend +//! fake `gh`, `claude` and `codex` executables, and, for email, a stand-in for Resend //! ([`resend::ResendStandIn`]). //! //! Layout of a scenario's temp root: @@ -13,7 +13,7 @@ //! home/.thirdshift/ the User config, config.toml, if the test writes one //! home/.config/ $XDG_CONFIG_HOME, where an install receipt would be //! installed/ a copy of thirdshift, if the test runs one to replace it -//! bin/ fake gh and claude, first on PATH +//! bin/ fake gh, claude and codex, first on PATH //! tmp/ $TMPDIR, so leftover temp directories are visible //! work/<repo>/ the launch clone, origin https://github.com/<owner>/<repo>.git //! gh-state.json fake GitHub state @@ -27,6 +27,11 @@ //! claude-calls.json.after-result.<n> stream lines the n-th session emits //! after its closing result //! claude-calls.json.final-message.<n> the n-th session's final message +//! codex-calls.json what the fake agent was asked to do on Codex, whose +//! sessions take the same scripts +//! codex-calls.json.final-message.<n>, codex-calls.json.error.<n> the n-th +//! Codex session's final message, and the error it +//! failed with //! gh-calls.json every gh command run, by thirdshift or the fake agent //! ``` @@ -597,6 +602,7 @@ test -f {root}/{COPY_REPLACED} .env("FAKE_GH_STATE", self.path("gh-state.json")) .env("FAKE_CLAUDE_SCRIPT", self.path("claude-script.sh")) .env("FAKE_CLAUDE_RECORD", self.path("claude-calls.json")) + .env("FAKE_CODEX_RECORD", self.path("codex-calls.json")) .env("FAKE_GH_RECORD", self.path("gh-calls.json")) // Seconds of waiting for CI become milliseconds. Each poll starts // the fake gh; at 100ms the grace period holds about three reads, @@ -801,22 +807,24 @@ test -f {root}/{COPY_REPLACED} } } + /// Every call the fake agent received on Codex, in order. + pub fn codex_calls(&self) -> Vec<Value> { + match fs::read_to_string(self.path("codex-calls.json")) { + Ok(text) => serde_json::from_str(&text).unwrap(), + Err(_) => Vec::new(), + } + } + /// Assert every `claude` call found every Factory skill, by its /// `thirdshift-<skill>` name, in its worktree's `.claude/skills/`. pub fn assert_every_session_found_the_factory_skills(&self) { - let calls = self.claude_calls(); - assert!(!calls.is_empty(), "no session ran"); - for call in calls { - for skill in FACTORY_SKILLS { - let skill_md = call["skill_files"][format!("{skill}/SKILL.md")].as_str(); - assert!( - skill_md.is_some_and(|text| text.contains(&format!("name: {skill}\n"))), - "the session in {} with the prompt {} found no {skill}", - call["cwd"], - call["prompt"] - ); - } - } + assert_found_the_factory_skills(self.claude_calls()); + } + + /// Assert every `codex` call found every Factory skill, by its + /// `thirdshift-<skill>` name, in its worktree's `.agents/skills/`. + pub fn assert_every_codex_session_found_the_factory_skills(&self) { + assert_found_the_factory_skills(self.codex_calls()); } /// The prompt of the first `claude` call. @@ -1080,3 +1088,20 @@ fn git_output(dir: &Path, args: &[&str]) -> Output { .output() .unwrap() } + +/// Assert each of `calls` found every Factory skill, by its +/// `thirdshift-<skill>` name, in its worktree's project skills. +fn assert_found_the_factory_skills(calls: Vec<Value>) { + assert!(!calls.is_empty(), "no session ran"); + for call in calls { + for skill in FACTORY_SKILLS { + let skill_md = call["skill_files"][format!("{skill}/SKILL.md")].as_str(); + assert!( + skill_md.is_some_and(|text| text.contains(&format!("name: {skill}\n"))), + "the session in {} with the prompt {} found no {skill}", + call["cwd"], + call["prompt"] + ); + } + } +} From 57e6904c0cbebd5d09157059dbe4fbcfffcb63df Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 00:57:35 -0400 Subject: [PATCH 07/14] Review clean-ups for Setup's Harness questions (#393) The Harness question defaults to the first Harness sessions can run on, claude when both can, as the issue says, rather than the User config's. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- README.md | 2 +- src/config.rs | 4 ++-- src/harness.rs | 5 +++-- src/questions.rs | 18 ++++++++---------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index f706b9176..0c0079daf 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ effort = "" # how hard that Model reasons; default blank, for Codex's own Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked where sessions can't run on it here: `claude (not installed)` when `claude` isn't on `PATH`, and `codex (not supported yet)` until Codex sessions are. A marked Harness, or a name that isn't one, is refused and asked again. The default is the User config's Harness if sessions can run on it, else `claude`. Then the Model (`harness.<name>.model`) and the Effort (`harness.<name>.effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. A named Model gets the test call a Run makes (see [Harness, Model and Effort](#harness-model-and-effort)); if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. The other Harness's section is left as it is. With no Harness sessions can run on here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing both, each marked where sessions can't run on it here: `claude (not installed)` when `claude` isn't on `PATH`, and `codex (not supported yet)` until Codex sessions are. A marked Harness, or a name that isn't one, is refused and asked again. The default is the one sessions can run on, `claude` when both can. Then the Model (`harness.<name>.model`) and the Effort (`harness.<name>.effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. A named Model gets the test call a Run makes (see [Harness, Model and Effort](#harness-model-and-effort)); if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. The other Harness's section is left as it is. With no Harness sessions can run on here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) diff --git a/src/config.rs b/src/config.rs index 04a766377..883906ac3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1138,9 +1138,9 @@ mod tests { let harness = config["harness"].as_table().unwrap(); assert!(harness.contains_key("default"), "{text:?}:\n{completed}"); for name in ["claude", "codex"] { - let set = harness[name].as_table().unwrap(); + let settings = harness[name].as_table().unwrap(); for key in ["model", "effort"] { - assert!(set.contains_key(key), "{text:?}:\n{completed}"); + assert!(settings.contains_key(key), "{text:?}:\n{completed}"); } } // An inline table gains its missing keys on its own line. diff --git a/src/harness.rs b/src/harness.rs index c1f56bf77..cedd7c603 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -220,8 +220,9 @@ impl Choice { self.test_call() } - /// Check that Claude takes a minimal test call on the Model, if one is - /// named, with the Effort, if any. A refusal says what Claude said. + /// Check that the Harness's CLI takes a minimal test call on the Model, + /// if one is named, with the Effort, if any. A refusal says what the CLI + /// said. Only Claude reaches it while Codex is not supported. pub fn test_call(&self) -> Result<()> { let cli = self.harness.name(); let Some(model) = &self.model else { diff --git a/src/questions.rs b/src/questions.rs index 25e6bcbcb..b538111ed 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -162,8 +162,8 @@ pub fn ask( /// Ask which Harness every Run's sessions run on, listing each with why /// sessions can't run on it here, if they can't, and refusing that one; then /// its Model and Effort, with those `current` sets for it as the defaults. -/// The Harness's default is `current`'s, if sessions can run on it, else the -/// first one they can run on. A Model is checked with a test call, as a Run +/// The Harness's default is the first one sessions can run on, so `claude` +/// when both can. A Model is checked with a test call, as a Run /// checks it, and on a refusal the Model and Effort are asked again. With no /// Harness to choose, nothing is asked, and the answer is `None`. fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndEffort)>> { @@ -175,11 +175,9 @@ fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndE }) .collect(); let listed = listed.join(" or "); - let available = |harness: &Harness| harness.unavailable().is_none(); - let default = current - .default - .filter(available) - .or_else(|| Harness::ALL.into_iter().find(available)); + let default = Harness::ALL + .into_iter() + .find(|harness| harness.unavailable().is_none()); let Some(default) = default else { say(&format!( "Harness for every Run's sessions: {listed}. Sessions can run on neither here, so \ @@ -205,10 +203,10 @@ fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndE }, } }; - let set = current.of(harness); + let current_settings = current.of(harness); loop { - let model = ask_setting("Model", harness, set.model.as_deref())?; - let effort = ask_setting("Effort", harness, set.effort.as_deref())?; + let model = ask_setting("Model", harness, current_settings.model.as_deref())?; + let effort = ask_setting("Effort", harness, current_settings.effort.as_deref())?; let choice = Choice { harness, model, From ffb42f2b9bbc91ac11774784802a4cf42d878edf Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 01:06:39 -0400 Subject: [PATCH 08/14] Review clean-ups for Codex sessions (#394) Name the Claude stream ClaudeProgress, settle a Codex Model and Effort into a ModelAndEffort, give the token totals their own type, and cover a Merge run, a bare non-zero exit and the prompts' sigil on Codex. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/base_fix.rs | 4 ++-- src/harness.rs | 53 ++++++++++++++++------------------------- src/main.rs | 5 ++-- src/progress.rs | 31 ++++++++++++------------ src/progress/codex.rs | 41 ++++++++++++++++++++++--------- src/session.rs | 33 +++++++++++++++++++++++++ tests/codex_sessions.rs | 19 ++++++++++++++- 7 files changed, 121 insertions(+), 65 deletions(-) diff --git a/src/base_fix.rs b/src/base_fix.rs index 8a0f97146..d3716d53b 100644 --- a/src/base_fix.rs +++ b/src/base_fix.rs @@ -157,8 +157,8 @@ impl BaseFix { } } - /// Have a Base fix it starts run its sessions on `harness`, the Run's - /// once checked. + /// Have any Base fix it starts run its sessions on `harness`: the Run's, + /// once the Run has checked it. pub fn runs_on(&mut self, harness: Choice) { self.harness = harness; } diff --git a/src/harness.rs b/src/harness.rs index 0a9c00a08..dda058629 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -259,9 +259,9 @@ impl Choice { ); } let catalog = Catalog::parse(&String::from_utf8_lossy(&output.stdout))?; - let (model, effort) = catalog.settle(self.model.as_deref(), self.effort.as_deref())?; - self.model = model; - self.effort = effort; + let settled = catalog.settle(self.model.as_deref(), self.effort.as_deref())?; + self.model = settled.model; + self.effort = settled.effort; Ok(()) } } @@ -325,17 +325,16 @@ impl Catalog { /// of those that Model supports, or with no Model of those any Model /// supports, that is `effort`, regardless of case. Fails naming the /// valid choices for a Model or Effort the catalog doesn't have. - fn settle( - &self, - model: Option<&str>, - effort: Option<&str>, - ) -> Result<(Option<String>, Option<String>)> { + fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result<ModelAndEffort> { let found = match model { Some(model) => Some(self.model(model)?), None => None, }; let Some(effort) = effort else { - return Ok((found.map(|found| found.slug.clone()), None)); + return Ok(ModelAndEffort { + model: found.map(|found| found.slug.clone()), + effort: None, + }); }; let efforts: Vec<&str> = match found { Some(found) => found.efforts.iter().map(String::as_str).collect(), @@ -362,10 +361,10 @@ impl Catalog { efforts.join(", ") ); }; - Ok(( - found.map(|found| found.slug.clone()), - Some(settled.to_string()), - )) + Ok(ModelAndEffort { + model: found.map(|found| found.slug.clone()), + effort: Some(settled.to_string()), + }) } /// The Model whose slug or display name is `name`, regardless of case. @@ -590,7 +589,7 @@ mod tests { } #[test] - fn codex_is_given_the_model_and_the_effort_as_its_config_setting() { + fn codex_is_given_the_model_and_the_effort_in_its_config() { for (model, effort, args) in [ ("", "", vec![]), ("gpt-6.1-sol", "", vec!["-m", "gpt-6.1-sol"]), @@ -621,39 +620,27 @@ mod tests { .unwrap() } - fn settled( - model: Option<&str>, - effort: Option<&str>, - ) -> Result<(Option<String>, Option<String>)> { + fn settled(model: Option<&str>, effort: Option<&str>) -> Result<ModelAndEffort> { catalog().settle(model, effort) } - fn named(model: &str, effort: &str) -> (Option<String>, Option<String>) { - let set = set(model, effort); - (set.model, set.effort) - } - #[test] fn a_codex_model_and_effort_match_regardless_of_case_and_become_codexs_names() { for (model, effort, settled_as) in [ - ( - Some("GPT-6.1-Sol"), - Some("Max"), - named("gpt-6.1-sol", "max"), - ), - (Some("gpt-6.1-sol"), None, named("gpt-6.1-sol", "")), + (Some("GPT-6.1-Sol"), Some("Max"), set("gpt-6.1-sol", "max")), + (Some("gpt-6.1-sol"), None, set("gpt-6.1-sol", "")), ( Some("gpt-5.5 classic"), Some("XHIGH"), - named("gpt-5.5", "xhigh"), + set("gpt-5.5", "xhigh"), ), ( Some("codex-auto-review"), None, - named("codex-auto-review", ""), + set("codex-auto-review", ""), ), - (None, Some("High"), named("", "high")), - (None, None, named("", "")), + (None, Some("High"), set("", "high")), + (None, None, set("", "")), ] { assert_eq!( settled(model, effort).unwrap(), diff --git a/src/main.rs b/src/main.rs index 7009b6af8..e0fc0020a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -90,8 +90,9 @@ test call with its Effort, which must succeed. On codex, a named Model and Effor codex debug models, matched regardless of case, a Model by slug or display name, and are passed on as Codex names them. A failure stops the command naming what to fix, before the Claim, the worktree and any Command log. Codex sessions run codex exec --json ---dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command log, the Activity log's start line, the pull request's body, -as in Built with claude · opus · high, and the Run notification each name all three. +--dangerously-bypass-approvals-and-sandbox, with the skills in .agents/skills/. The Command +log, the Activity log's start line, the pull request's body, as in Built with claude · opus · +high, and the Run notification each name all three. --email sends one Run notification when the Run ends, whatever the outcome: ready for review, merged, failed or interrupted. --email <address> sends it to <address>; a word diff --git a/src/progress.rs b/src/progress.rs index 9052d604b..7733170c0 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1,9 +1,8 @@ //! Progress lines on stderr: thirdshift's own steps, and a session's //! stream, Claude's stream-json or Codex's JSONL, condensed to one short -//! line per notable event. Each -//! line is stamped with the local time it was printed, so a stalled Run can -//! be told from a busy one. Each is kept in the Command log too, if the -//! command keeps one. +//! line per notable event. Each line is stamped with the local time it was +//! printed, so a stalled Run can be told from a busy one. Each is kept in the +//! Command log too, if the command keeps one. mod codex; @@ -154,26 +153,26 @@ pub trait Stream: Send { /// The [`Stream`] of a session on `harness` in `worktree`. pub fn for_harness(harness: Harness, worktree: &Path) -> Box<dyn Stream> { match harness { - Harness::Claude => Box::new(Progress::default()), + Harness::Claude => Box::new(ClaudeProgress::default()), Harness::Codex => Box::new(codex::CodexProgress::in_worktree(worktree)), } } -/// Condenses one Claude session's stream-json, a line at a time. Unknown and malformed -/// lines are skipped, never an error. +/// Condenses one Claude session's stream-json, a line at a time. Unknown and +/// malformed lines are skipped, never an error. /// /// A `result` event doesn't mean the session is over: a session that waits on /// background sub-agents resumes, with another `system`/`init` event, and ends /// each resumption with another `result`. So only the first `init` gives a -/// line, `result` events give none, and [`Progress::summary`] reports the last +/// line, `result` events give none, and [`ClaudeProgress::summary`] reports the last /// one once the process has exited. /// /// A background task still running when the session ends its turn for good is /// killed as the process exits, so a task killed after the last `result` is /// work the session may have been waiting on, or work it had given up on and -/// left running: see [`Progress::killed_background_work`]. +/// left running: see [`ClaudeProgress::killed_background_work`]. #[derive(Default)] -pub struct Progress { +pub struct ClaudeProgress { started: bool, cwd: Option<String>, session_id: Option<String>, @@ -191,7 +190,7 @@ pub struct Progress { final_message: Option<String>, } -impl Stream for Progress { +impl Stream for ClaudeProgress { fn condense(&mut self, raw: &str) -> Vec<String> { let Ok(event) = serde_json::from_str::<Value>(raw) else { return Vec::new(); @@ -270,7 +269,7 @@ impl Stream for Progress { } } -impl Progress { +impl ClaudeProgress { /// Track a background task's description and whether it was killed. fn track_task(&mut self, event: &Value) { let Some(id) = event["task_id"].as_str() else { @@ -358,8 +357,8 @@ mod tests { use super::*; use serde_json::json; - fn lines(events: &[Value]) -> (Progress, Vec<Vec<String>>) { - let mut progress = Progress::default(); + fn lines(events: &[Value]) -> (ClaudeProgress, Vec<Vec<String>>) { + let mut progress = ClaudeProgress::default(); let lines = events .iter() .map(|event| progress.condense(&event.to_string())) @@ -544,7 +543,7 @@ mod tests { #[test] fn no_result_means_no_summary() { - assert_eq!(Progress::default().summary(), None); + assert_eq!(ClaudeProgress::default().summary(), None); } fn task_started(id: &str, description: &str) -> Value { @@ -726,7 +725,7 @@ mod tests { #[test] fn unknown_and_malformed_lines_are_skipped() { - let mut progress = Progress::default(); + let mut progress = ClaudeProgress::default(); for raw in [ "", "not json", diff --git a/src/progress/codex.rs b/src/progress/codex.rs index 1c6a07abc..66168ba83 100644 --- a/src/progress/codex.rs +++ b/src/progress/codex.rs @@ -28,9 +28,8 @@ pub struct CodexProgress { running: Vec<(String, String)>, /// The commands still running when the turn completed. killed: Vec<String>, - /// Token totals from `turn.completed`: input, of which cached, and - /// output. - tokens: Option<(u64, u64, u64)>, + /// Token totals from `turn.completed`. + tokens: Option<Tokens>, final_message: Option<String>, /// What failed the turn, from `turn.failed`. failure: Option<String>, @@ -38,6 +37,14 @@ pub struct CodexProgress { last_error: Option<String>, } +/// A session's token totals. +struct Tokens { + input: u64, + /// Of the input, how many were cached. + cached: u64, + output: u64, +} + impl CodexProgress { /// The stream of a session running in `worktree`. pub fn in_worktree(worktree: &Path) -> Self { @@ -47,6 +54,14 @@ impl CodexProgress { } } + /// Keep the message of an `error`, an event or an item, as the last + /// error reported. + fn reported(&mut self, error: &Value) { + if let Some(message) = error["message"].as_str() { + self.last_error = Some(message.to_string()); + } + } + /// Track an item's state, and give its line the first time it is seen. fn item(&mut self, item: &Value) -> Vec<String> { let id = item["id"].as_str().unwrap_or("").to_string(); @@ -64,9 +79,7 @@ impl CodexProgress { } } Some("error") => { - if let Some(message) = item["message"].as_str() { - self.last_error = Some(message.to_string()); - } + self.reported(item); } _ => {} } @@ -127,7 +140,11 @@ impl Stream for CodexProgress { if let (Some(input), Some(output)) = (count("input_tokens"), count("output_tokens")) { let cached = count("cached_input_tokens").unwrap_or(0); - self.tokens = Some((input, cached, output)); + self.tokens = Some(Tokens { + input, + cached, + output, + }); } self.killed = self .running @@ -144,9 +161,7 @@ impl Stream for CodexProgress { Vec::new() } Some("error") => { - if let Some(message) = event["message"].as_str() { - self.last_error = Some(message.to_string()); - } + self.reported(&event); Vec::new() } _ => Vec::new(), @@ -156,7 +171,11 @@ impl Stream for CodexProgress { /// The token totals from `turn.completed`: Codex reports no turns and no /// cost. fn summary(&self) -> Option<String> { - let (input, cached, output) = self.tokens?; + let Tokens { + input, + cached, + output, + } = self.tokens.as_ref()?; Some(format!( "{input} input tokens ({cached} cached), {output} output tokens" )) diff --git a/src/session.rs b/src/session.rs index a6e50c8b3..d3185ae7f 100644 --- a/src/session.rs +++ b/src/session.rs @@ -400,3 +400,36 @@ fn minutes_and_seconds(duration: Duration) -> String { minutes => format!("{minutes}m {}s", seconds % 60), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_prompt_that_loads_a_skill_loads_it_with_codexs_sigil_on_codex() { + let issue = IssueUrl::parse("https://github.com/acme/widgets/issues/7").unwrap(); + for (prompt, first_line) in [ + ( + prompt::fresh(&issue, "main", "issue-7"), + "$thirdshift-implement https://github.com/acme/widgets/issues/7", + ), + ( + prompt::conflict_repair(&issue, "main", "issue-7", "<pr>"), + "$thirdshift-resolving-merge-conflicts", + ), + ( + prompt::review_repair(&issue, "issue-7", "<pr>", "abc123"), + "$thirdshift-code-review abc123", + ), + ( + prompt::resume(&["cargo test"]), + "Your background work (cargo test) was killed when your turn ended, because \ + ending the turn ends the session.", + ), + ] { + let codex_prompt = codex_prompt(&prompt); + assert_eq!(codex_prompt.lines().next(), Some(first_line)); + assert_eq!(codex_prompt[1..], prompt[1..]); + } + } +} diff --git a/tests/codex_sessions.rs b/tests/codex_sessions.rs index 02f54ffa2..eb4f75eba 100644 --- a/tests/codex_sessions.rs +++ b/tests/codex_sessions.rs @@ -234,7 +234,7 @@ fn a_session_left_with_a_command_running_is_resumed_on_its_thread_with_every_fla } #[test] -fn a_failed_turn_fails_the_run_with_codexs_error_in_the_cause() { +fn a_failed_turn_or_a_non_zero_exit_fails_the_run_with_codexs_error_in_the_cause() { for (script, cause) in [ ( "echo \"The 'gpt-6.1-sol' model is not supported.\" > \"$FAKE_CODEX_ERROR\"\nexit 1", @@ -244,6 +244,10 @@ fn a_failed_turn_fails_the_run_with_codexs_error_in_the_cause() { r#"echo '{"type": "turn.failed", "error": {"message": "stream disconnected"}}'"#, "codex's turn failed: stream disconnected", ), + ( + "echo '{\"type\": \"error\", \"message\": \"Reconnecting... 1/5\"}'\nkill -KILL $PPID", + "codex exited by signal: Reconnecting... 1/5", + ), ] { let scenario = Scenario::new(); scenario.agent_does_for(7, script); @@ -280,6 +284,19 @@ fn a_retried_error_alone_does_not_fail_the_session() { ); } +#[test] +fn a_merge_run_on_codex_merges_the_pr() { + let scenario = Scenario::new(); + scenario.agent_does_for(7, AGENT_OPENS_PR); + + let result = scenario.run(&["merge", "harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_eq!(scenario.gh_state()["prs"][0]["state"], "MERGED"); + assert_eq!(scenario.codex_calls().len(), 1); + assert!(scenario.claude_calls().is_empty()); +} + /// Assert the Run failed with `error` before any work: no Claim, no /// worktree, no session and no Command log. fn assert_failed_before_any_work(scenario: &Scenario, result: &RunResult, error: &str) { From 06c1ccc0daa144680d9ce301c97d2e67f5d8b5d1 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 01:19:30 -0400 Subject: [PATCH 09/14] Check the fake codex's stdin by fstat, as macOS has no /proc (#394) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- tests/fakes/codex.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 56b1471c5..7d317fd88 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -22,6 +22,7 @@ use std::fs; use std::io::Write; +use std::os::fd::AsFd; use std::os::unix::fs::MetadataExt; use std::process::Command; @@ -42,9 +43,14 @@ const CATALOG: &str = r#"{"models": [ "supported_reasoning_levels": [{"effort": "low"}, {"effort": "medium"}, {"effort": "high"}, {"effort": "xhigh"}]} ]}"#; -/// Whether stdin is `/dev/null`. +/// Whether stdin is `/dev/null`, by `fstat` on it, as macOS has no `/proc`. fn stdin_is_null() -> bool { - let (Ok(stdin), Ok(null)) = (fs::metadata("/proc/self/fd/0"), fs::metadata("/dev/null")) else { + let stdin = std::io::stdin() + .as_fd() + .try_clone_to_owned() + .map(fs::File::from) + .and_then(|file| file.metadata()); + let (Ok(stdin), Ok(null)) = (stdin, fs::metadata("/dev/null")) else { return false; }; stdin.rdev() == null.rdev() && stdin.ino() == null.ino() From c9161b141432ca27a12ef4606e7f5c6b63f1cd48 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 01:40:15 -0400 Subject: [PATCH 10/14] Resume Codex sessions that leave sub-agents running, and interrupt them with SIGINT first (#395) A Codex sub-agent call still running at turn end is now killed background work, as a command is, so its session gets the one Resume. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before SIGTERM and SIGKILL. Tests cover the Resume on Codex ending the same way, counting against no Repair cap, and the interrupt order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- README.md | 2 +- src/progress/codex.rs | 73 +++++++++++++--- src/session.rs | 41 +++++---- tests/codex_sessions.rs | 179 +++++++++++++++++++++++++++++++++++++++- tests/fakes/codex.rs | 22 +++++ 5 files changed, 288 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 8373ae56d..17094cc66 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ thirdshift https://github.com/<owner>/<repo>/issues/<n> model claude-opus-5-5 ef For each of the three, the command wins, then the [User config](#user-config), then the default. The Harness's default is `claude`; the Model's and the Effort's are nothing, which leaves them to the Harness, so with no flag and no `[harness]` section sessions run exactly as `claude -p` would. A Model and Effort set in the User config come from the chosen Harness's own section, so `harness claude` over a `codex` default takes `[harness.claude]`, while `model` and `effort` on the command apply to whichever Harness is chosen. Names are passed to Claude Code as given: `--model` and `--effort` when set. Codex takes them as it names them, which the check below settles: `-m <model>` and `-c model_reasoning_effort="<effort>"` when set. -With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-<skill>`. A Resume runs `codex exec … resume <session id>`, with every flag given again. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. +With `harness codex`, every session runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` in the Run's worktree, with stdin set to null, and with `-c project_doc_fallback_filenames=["CLAUDE.md"]`, so Codex reads `CLAUDE.md` where a repository has no `AGENTS.md`. Codex's sandbox is off: on a stock Ubuntu machine it can't start, and commits and pushes from a worktree write outside it, so the worktree is the only boundary, as with Claude's auto mode ([ADR-0012](docs/adr/0012-factory-skills-linked-into-the-worktree-codex-unsandboxed.md)). The Factory skills are linked into the worktree's `.agents/skills/`, kept out of git with `/.agents/skills/thirdshift-*` in `.git/info/exclude`, and each prompt's first line loads its skill as `$thirdshift-<skill>`. `codex exec` kills what the agent left running as it exits, so a session whose turn ends with a command or a sub-agent call still running (started, never completed) gets a Resume, as on Claude: `codex exec … resume <session id>`, with the Model, Effort, bypass flag and `CLAUDE.md` fallback given again, as Codex keeps none of them. An interrupt sends a Codex session SIGINT, the only signal Codex stops cleanly on, before the SIGTERM and SIGKILL every session gets, each 10 seconds apart. Progress lines come from Codex's `item.*` events, the session-ended line gives its token totals, as Codex reports no turns or cost, and the final message is the last `agent_message`. A session whose turn fails (`turn.failed`), or that exits non-zero, fails the Run with Codex's error in the cause; an `error` event Codex retries does not. Each Codex session adds a `trust_level = "trusted"` entry for the repository to `~/.codex/config.toml`, and your `~/.claude/` instructions and hooks don't reach it. Before any work, where an Origin match failure would stop it, and so before the Claim, the worktree and any Command log, a command checks what it chose: the Harness's CLI must be on `PATH`, else it fails naming the CLI and the setting that chose it; and on Claude a named Model gets a minimal test call, `claude -p --model <model>` with the Effort, if any, which must succeed, else the command fails with what Claude said. No test call is made when no Model is named. On Codex a named Model and Effort are checked against `codex debug models`, which costs no tokens: each matches regardless of case, a Model by its slug or its display name, and becomes the name Codex takes, so `model GPT-6.1-Sol effort Max` reaches Codex as `gpt-6.1-sol` and `max`. An unknown Model, or an Effort that Model doesn't support (with no Model, one no Model supports), fails naming the valid choices. What the command records is what the check settled on. A Pickup run or an Architect run makes these checks only once it has decided not to skip, and a Spec run makes them once for all its Tickets. diff --git a/src/progress/codex.rs b/src/progress/codex.rs index 66168ba83..a9d9a5e33 100644 --- a/src/progress/codex.rs +++ b/src/progress/codex.rs @@ -1,7 +1,8 @@ //! Codex's `exec --json` stream, condensed: `thread.started` gives the //! session id, `item.*` events give a progress line per command, file -//! change and tool call, `turn.completed` the token totals and -//! `turn.failed` the error the session failed with. See +//! change and tool call, `turn.completed` the token totals and the commands +//! and sub-agent calls killed as the session ends, and `turn.failed` the +//! error the session failed with. See //! `docs/research/codex-headless-harness.md`. use std::collections::HashSet; @@ -13,10 +14,10 @@ use super::{Stream, bash, shorten}; /// Condenses one Codex session's JSONL stream, a line at a time. /// -/// `codex exec` runs one turn per process. A command the agent left running -/// shows as a `command_execution` item still `in_progress` when the turn -/// completes, and it is killed as Codex exits: see -/// [`Stream::killed_background_work`]. +/// `codex exec` runs one turn per process. A command or sub-agent call the +/// agent left running shows as a `command_execution` or `collab_tool_call` +/// item still `in_progress` when the turn completes, and it is killed as +/// Codex exits: see [`Stream::killed_background_work`]. #[derive(Default)] pub struct CodexProgress { /// The worktree the session runs in, which Codex's stream doesn't name. @@ -24,9 +25,10 @@ pub struct CodexProgress { session_id: Option<String>, /// The ids of the items that have had their line. seen: HashSet<String>, - /// The commands still running, by item id, in the order they started. + /// The descriptions of the commands and sub-agent calls still running, + /// by item id, in the order they started. running: Vec<(String, String)>, - /// The commands still running when the turn completed. + /// The descriptions of those still running when the turn completed. killed: Vec<String>, /// Token totals from `turn.completed`. tokens: Option<Tokens>, @@ -66,11 +68,10 @@ impl CodexProgress { fn item(&mut self, item: &Value) -> Vec<String> { let id = item["id"].as_str().unwrap_or("").to_string(); match item["type"].as_str() { - Some("command_execution") => { + Some("command_execution" | "collab_tool_call") => { self.running.retain(|(running, _)| *running != id); if item["status"] == "in_progress" { - let command = unwrapped(item["command"].as_str().unwrap_or("")); - self.running.push((id.clone(), command)); + self.running.push((id.clone(), running_work(item))); } } Some("agent_message") => { @@ -191,7 +192,8 @@ impl Stream for CodexProgress { self.session_id.as_deref() } - /// The commands still running when the turn completed. + /// The commands and sub-agent calls still running when the turn + /// completed. fn killed_background_work(&self) -> Vec<&str> { if self.failure.is_some() { return Vec::new(); @@ -208,6 +210,24 @@ impl Stream for CodexProgress { } } +/// How a command or sub-agent call left running is described: a command as +/// it was run, without its shell, and a sub-agent call by its tool and the +/// first line of the prompt it gave, if any, as in `agent spawn_agent: +/// Review the diff`. +fn running_work(item: &Value) -> String { + if item["type"] == "command_execution" { + return unwrapped(item["command"].as_str().unwrap_or("")); + } + let tool = item["tool"].as_str().unwrap_or("?"); + match item["prompt"] + .as_str() + .and_then(|prompt| prompt.lines().next()) + { + Some(prompt) if !prompt.is_empty() => format!("agent {tool}: {}", shorten(prompt)), + _ => format!("agent {tool}"), + } +} + /// `command` without the shell Codex wraps it in, as in `/bin/bash -lc 'git /// push'`, if it is wrapped so. fn unwrapped(command: &str) -> String { @@ -434,6 +454,35 @@ mod tests { assert_eq!(progress.killed_background_work(), ["npm run dev"]); } + #[test] + fn a_sub_agent_call_still_running_when_the_turn_completes_is_killed_background_work() { + let call = |kind: &str, id: &str, tool: &str, prompt: &str, status: &str| { + item( + kind, + id, + json!({ "type": "collab_tool_call", "tool": tool, "prompt": prompt, "status": status }), + ) + }; + let (progress, _) = lines(&[ + call( + "item.started", + "item_1", + "spawn_agent", + "Review the diff\nThen report.", + "in_progress", + ), + call("item.started", "item_2", "wait", "", "in_progress"), + call("item.completed", "item_2", "wait", "", "completed"), + call("item.started", "item_3", "spawn_agent", "", "in_progress"), + completed(10, 2), + ]); + + assert_eq!( + progress.killed_background_work(), + ["agent spawn_agent: Review the diff", "agent spawn_agent"] + ); + } + #[test] fn a_failed_turn_fails_with_its_error_and_kills_no_background_work() { let (progress, _) = lines(&[ diff --git a/src/session.rs b/src/session.rs index d3185ae7f..6f42db211 100644 --- a/src/session.rs +++ b/src/session.rs @@ -177,8 +177,8 @@ fn ending_with(killed: &[&str]) -> String { /// `log` and condensing it through `stream` to progress lines on stderr, /// each labelled `kind`. Returns what the stream showed once the CLI has /// exited cleanly, its turn not failed. Otherwise fails with the error the -/// stream gave, if any. An interrupt stops the session and fails with -/// `interrupted`. +/// stream gave, if any. An interrupt stops the session, as [`stop`] does, +/// and fails with `interrupted`. fn run( kind: &str, harness: Harness, @@ -230,7 +230,7 @@ fn run( }); let status = loop { if interrupt::requested() { - stop(&mut child); + stop(&mut child, harness); bail!("interrupted"); } if let Some(status) = child @@ -342,27 +342,40 @@ const CLAUDE_MD_FALLBACK: &str = r#"project_doc_fallback_filenames=["CLAUDE.md"] const POLL: Duration = Duration::from_millis(100); -/// How long a session gets to exit after SIGTERM before it is killed. +/// How long a session gets to exit after each signal that asks it to stop. const STOP_GRACE: Duration = Duration::from_secs(10); -/// Stop `child`'s process group: SIGTERM, then SIGKILL if it outlives -/// `STOP_GRACE`. -fn stop(child: &mut Child) { +/// Stop `child`, a session on `harness`, by its process group: with the +/// signals that ask `harness` to stop, in turn, each given `STOP_GRACE`, +/// then SIGKILL. +fn stop(child: &mut Child, harness: Harness) { let group = -(child.id() as libc::pid_t); - // SAFETY: kill has no memory-safety preconditions. - unsafe { libc::kill(group, libc::SIGTERM) }; - let deadline = Instant::now() + STOP_GRACE; - while Instant::now() < deadline { - if let Ok(Some(_)) = child.try_wait() { - return; + for &signal in stop_signals(harness) { + // SAFETY: kill has no memory-safety preconditions. + unsafe { libc::kill(group, signal) }; + let deadline = Instant::now() + STOP_GRACE; + while Instant::now() < deadline { + if let Ok(Some(_)) = child.try_wait() { + return; + } + thread::sleep(POLL); } - thread::sleep(POLL); } // SAFETY: as above. unsafe { libc::kill(group, libc::SIGKILL) }; let _ = child.wait(); } +/// The signals that ask a session on `harness` to stop, in the order they +/// are sent: SIGTERM, after SIGINT for Codex, which stops cleanly only on +/// SIGINT, interrupting its turn. +fn stop_signals(harness: Harness) -> &'static [libc::c_int] { + match harness { + Harness::Claude => &[libc::SIGTERM], + Harness::Codex => &[libc::SIGINT, libc::SIGTERM], + } +} + /// Copy every line of `stream` to `log_file` and print the progress lines it /// condenses to, until the stream ends. fn follow( diff --git a/tests/codex_sessions.rs b/tests/codex_sessions.rs index eb4f75eba..646c04797 100644 --- a/tests/codex_sessions.rs +++ b/tests/codex_sessions.rs @@ -5,8 +5,10 @@ //! `.agents/skills/` and its prompt loading its skill as //! `$thirdshift-<skill>`. Codex's stream gives the progress lines, the //! Session log, the session id a Resume continues and the final message, and -//! a failed turn fails the Run with Codex's error. The Model and Effort are -//! checked against `codex debug models` before any work. +//! a failed turn fails the Run with Codex's error. A session that leaves a +//! command or sub-agent call running gets one Resume on its thread, as on +//! Claude, and an interrupt sends a session SIGINT before SIGTERM. The Model +//! and Effort are checked against `codex debug models` before any work. mod support; @@ -233,6 +235,179 @@ fn a_session_left_with_a_command_running_is_resumed_on_its_thread_with_every_fla ); } +/// Bash that has the agent leave a sub-agent call, spawning a sub-agent +/// given `prompt`, running when its turn ends, as Codex streams it. +fn leaves_a_sub_agent_running(prompt: &str) -> String { + format!( + r#"echo '{{"type": "item.started", "item": {{"id": "item_8", "type": "collab_tool_call", "tool": "spawn_agent", "prompt": "{prompt}", "status": "in_progress"}}}}' +"# + ) +} + +#[test] +fn a_session_left_with_a_sub_agent_running_is_resumed_too() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session( + 7, + 1, + &format!( + "{AGENT_OPENS_PR}{}", + leaves_a_sub_agent_running("Review the diff") + ), + ); + + let result = scenario.run(&["harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + let calls = scenario.codex_calls(); + assert_eq!(calls.len(), 2); + assert_eq!(argv(&calls[1])[5..7], ["resume", "fake-thread-1"]); + let prompt = calls[1]["prompt"].as_str().unwrap(); + assert!( + prompt.starts_with("Your background work (agent spawn_agent: Review the diff) was killed"), + "{prompt}" + ); +} + +#[test] +fn a_resume_that_ends_the_same_way_gets_no_second_resume_and_a_later_failure_names_the_work() { + let scenario = Scenario::new(); + // Every session, the Resume included, leaves its tests running and opens + // no PR. + scenario.agent_does(&format!( + "echo wip >> feature.txt\n{}", + leaves_running("cargo test") + )); + + let result = scenario.run(&["harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + let calls = scenario.codex_calls(); + assert_eq!(calls.len(), 2, "one Resume, never two"); + assert_eq!(argv(&calls[1])[5..7], ["resume", "fake-thread-1"]); + let ending = "ended with a background task still running (cargo test), which was killed"; + for line in [ + format!("thirdshift: implement: the Resume {ending}; carrying on"), + format!("thirdshift: implement session {ending}, and a later step failed: no PR found"), + ] { + assert!( + result.stderr.lines().any(|said| said.starts_with(&line)), + "{line:?} in {}", + result.stderr + ); + } + assert_eq!( + scenario.origin_log("issue-7").unwrap()[0], + format!( + "thirdshift: failed run (implement session {ending}, and a later step failed: no PR found)" + ) + ); +} + +#[test] +fn resumes_count_against_no_repair_cap() { + let scenario = Scenario::new(); + scenario.agent_does_for_in_session(7, 1, &format!("{AGENT_OPENS_PR}{}", checks_on_head(RED))); + // Repairs 1 to 4 each leave CI red and their tests running, so each gets + // a Resume, which does nothing; Repair 5, the cap's last, turns CI green. + for repair in 1..=4 { + scenario.agent_does_for_in_session( + 7, + repair + 1, + &format!( + "echo {repair} > fix.txt\ngit add fix.txt\ngit commit -q -m Fix\n{}{}", + checks_on_head(RED), + leaves_running("cargo test") + ), + ); + } + scenario.agent_does_for_in_session( + 7, + 6, + &format!( + "echo 5 > fix.txt\ngit add fix.txt\ngit commit -q -m Fix\n{}", + checks_on_head(GREEN) + ), + ); + + let result = scenario.run(&["harness", "codex", &scenario.issue_url(7)]); + + assert_eq!(result.code, Some(0), "stderr: {}", result.stderr); + assert_eq!(scenario.codex_calls().len(), 10); + for line in [ + "thirdshift: repair-4-resume: session started\n", + "starting Repair 5 of 5\n", + ] { + assert!( + result.stderr.contains(line), + "{line:?} in {}", + result.stderr + ); + } +} + +/// Bash that has the agent record each SIGINT and SIGTERM its session gets +/// to `signals` in the scenario, ending on SIGTERM and, if `ends_on_sigint`, +/// on SIGINT too, touch `agent-started` there and then keep working until a +/// signal ends it. +fn records_signals(scenario: &Scenario, ends_on_sigint: bool) -> String { + let on_sigint = if ends_on_sigint { "; exit 130" } else { "" }; + format!( + r#"trap 'echo INT >> "{signals}"{on_sigint}' INT +trap 'echo TERM >> "{signals}"; exit 143' TERM +echo 'half done' > wip.txt +touch "{started}" +while :; do sleep 0.1 || :; done +"#, + signals = scenario.path("signals").display(), + started = scenario.path("agent-started").display(), + ) +} + +#[test] +fn an_interrupt_sends_a_codex_session_sigint_and_the_run_ends_as_interrupted() { + let scenario = Scenario::new(); + let signals = scenario.path("signals"); + scenario.agent_does_for(7, &records_signals(&scenario, true)); + + let result = scenario.run_and_signal( + &["harness", "codex", &scenario.issue_url(7)], + "agent-started", + "INT", + ); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert_eq!(fs::read_to_string(&signals).unwrap(), "INT\n"); + assert_eq!( + scenario.origin_log("issue-7").unwrap()[0], + "thirdshift: failed run (interrupted)" + ); + assert_eq!( + scenario.origin_file("issue-7", "wip.txt"), + Some("half done\n".to_string()) + ); +} + +#[test] +fn a_codex_session_that_outlasts_sigint_is_sent_sigterm_after_it() { + let scenario = Scenario::new(); + let signals = scenario.path("signals"); + scenario.agent_does_for(7, &records_signals(&scenario, false)); + + let result = scenario.run_and_signal( + &["harness", "codex", &scenario.issue_url(7)], + "agent-started", + "TERM", + ); + + assert_eq!(result.code, Some(1), "stderr: {}", result.stderr); + assert_eq!(fs::read_to_string(&signals).unwrap(), "INT\nTERM\n"); + assert_eq!( + scenario.origin_log("issue-7").unwrap()[0], + "thirdshift: failed run (interrupted)" + ); +} + #[test] fn a_failed_turn_or_a_non_zero_exit_fails_the_run_with_codexs_error_in_the_cause() { for (script, cause) in [ diff --git a/tests/fakes/codex.rs b/tests/fakes/codex.rs index 7d317fd88..1da013cf1 100644 --- a/tests/fakes/codex.rs +++ b/tests/fakes/codex.rs @@ -19,6 +19,11 @@ //! script that succeeds ends the turn with `turn.completed`, and one that //! fails with `turn.failed`, whose error is what it wrote to //! $FAKE_CODEX_ERROR, else a default, and exits 1, as Codex does. +//! +//! SIGINT and SIGTERM to its process group reach its script, which decides +//! how the session stops: the fake waits for its script whatever signal +//! comes, short of SIGKILL, so what the script records of the signals is +//! there once the fake has exited. use std::fs; use std::io::Write; @@ -56,6 +61,22 @@ fn stdin_is_null() -> bool { stdin.rdev() == null.rdev() && stdin.ino() == null.ino() } +/// Catch SIGINT and SIGTERM, doing nothing on either, so the fake outlasts +/// them until its script ends. A caught signal, unlike an ignored one, is +/// back to its default in the script, which can trap it. +fn outlast_interrupts() { + // From the C library, which the standard library links: SIGINT and + // SIGTERM are 2 and 15 on Linux and macOS alike. + unsafe extern "C" { + fn signal(signum: i32, handler: extern "C" fn(i32)) -> usize; + } + extern "C" fn caught(_: i32) {} + for signum in [2, 15] { + // SAFETY: the handler does nothing, so it is async-signal-safe. + unsafe { signal(signum, caught) }; + } +} + /// Print one line of the stream. fn emit(event: Json) { println!("{event}"); @@ -102,6 +123,7 @@ pub fn main(argv: Vec<String>) { drop(lock); let session = records.len(); + outlast_interrupts(); if std::env::var_os("FAKE_CLAUDE_NO_SESSION_ID").is_none() { emit(object([ ("type", string("thread.started")), From 6afee411f37384def11860defd5ad50e63bc9d9e Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 01:44:09 -0400 Subject: [PATCH 11/14] Setup offers Codex (#396) With codex on PATH, Setup accepts it as the Harness, and it's the default answer when it's the only one installed. For Codex, Setup lists the Models codex debug models offers, by slug and display name, then the Efforts the chosen Model supports. Each answer is matched as a Run matches it and written as Codex names it; one the catalog doesn't have is asked again, with the valid choices. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- README.md | 2 +- src/harness.rs | 128 ++++++++++++++++++-------------- src/questions.rs | 116 +++++++++++++++++++---------- tests/setup.rs | 189 ++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 322 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 8373ae56d..1b60e347b 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ effort = "" # how hard that Model reasons; default blank, for Codex's own Every key holds its real value, so a Run reading it does exactly what it does with no file. `email.to` has no default, so `setup` suggests one: the public email of your GitHub profile (from `gh api user`), else your global git `user.email`, unless that is a `@users.noreply.github.com` address, which can't receive mail. With neither, `email.to` is the only line written commented out, as above. `setup` never asks `gh` for more scopes, so a private GitHub email is not read, and a Run never looks the suggestion up: `--email` with no address and no `email.to` still stops the Run. From a terminal (stdin and stderr both terminals), `setup` first asks, on stderr: -1. The Harness for every Run's sessions (`harness.default`), listing both, each marked where sessions can't run on it here: `claude (not installed)` when `claude` isn't on `PATH`, and `codex (not supported yet)` until Setup offers Codex. A marked Harness, or a name that isn't one, is refused and asked again. The default is the one sessions can run on, `claude` when both can. Then the Model (`harness.<name>.model`) and the Effort (`harness.<name>.effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. A named Model gets the test call a Run makes (see [Harness, Model and Effort](#harness-model-and-effort)); if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. The other Harness's section is left as it is. With no Harness sessions can run on here, Setup says so, asks none of this, and leaves the harness settings as they are. +1. The Harness for every Run's sessions (`harness.default`), listing both, each marked `(not installed)` where its CLI isn't on `PATH`. A marked Harness, or a name that isn't one, is refused and asked again. The default is the first one installed, `claude` when both are, and `codex` when only it is. Then the Model (`harness.<name>.model`) and the Effort (`harness.<name>.effort`) for the Harness chosen only, each defaulting to the User config's, where Enter on none keeps the Harness's own default and `-` clears one. They're checked as a Run checks them (see [Harness, Model and Effort](#harness-model-and-effort)). For Claude, a named Model gets the test call; if Claude refuses it, Setup shows what Claude said and asks the Model and Effort again. For Codex, Setup lists the Models `codex debug models` offers, by slug and display name, then the Efforts the Model chosen supports; each answer is matched regardless of case and written as Codex names it, so `GPT-6.1-Sol` and `Max` are written as `gpt-6.1-sol` and `max`, and a Model or Effort the catalog doesn't have is asked again, with the valid choices. The other Harness's section is left as it is. With no Harness installed here, Setup says so, asks none of this, and leaves the harness settings as they are. 2. Every Run a Merge run? (`merge.always`) 3. With that on, every Run may start a Base fix when the Base branch's CI is red? (`base.fix`). With it off, this is not asked, and `base.fix` is written at its default, `false`. 4. Every Run first fast-forwards your checkout of the Base branch? (`launch.pull`) diff --git a/src/harness.rs b/src/harness.rs index 42cc3a500..e6f3684eb 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -42,35 +42,9 @@ impl Harness { /// Every Harness, in the order Setup lists them. pub const ALL: [Harness; 2] = [Harness::Claude, Harness::Codex]; - /// Why Setup can't offer it on this machine, if it can't: Setup doesn't - /// offer Codex yet, and otherwise its CLI must be on `PATH`. - pub fn unavailable(self) -> Option<Unavailable> { - if self == Harness::Codex { - Some(Unavailable::NotSupportedYet) - } else if !on_path(self.name()) { - Some(Unavailable::NotInstalled) - } else { - None - } - } -} - -/// Why sessions can't run on a Harness. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Unavailable { - /// Setup doesn't offer it yet. - NotSupportedYet, - /// Its CLI isn't on `PATH`. - NotInstalled, -} - -impl fmt::Display for Unavailable { - /// As Setup marks a Harness with it, as in `codex (not supported yet)`. - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str(match self { - Unavailable::NotSupportedYet => "not supported yet", - Unavailable::NotInstalled => "not installed", - }) + /// Whether sessions can run on it here: its CLI is on `PATH`. + pub fn installed(self) -> bool { + on_path(self.name()) } } @@ -282,19 +256,7 @@ impl Choice { return Ok(()); } progress::step("checking the Model and Effort against codex debug models"); - let output = Command::new("codex") - .args(["debug", "models"]) - .stdin(Stdio::null()) - .output() - .context("could not run codex debug models to check the Model and Effort")?; - if !output.status.success() { - bail!( - "codex debug models failed, so the Model and Effort can't be checked: {}", - said(&output) - ); - } - let catalog = Catalog::parse(&String::from_utf8_lossy(&output.stdout))?; - let settled = catalog.settle(self.model.as_deref(), self.effort.as_deref())?; + let settled = Catalog::read()?.settle(self.model.as_deref(), self.effort.as_deref())?; self.model = settled.model; self.effort = settled.effort; Ok(()) @@ -325,11 +287,28 @@ struct CatalogModel { /// The Models Codex can run, as `codex debug models` prints them. #[derive(Debug)] -struct Catalog { +pub struct Catalog { models: Vec<CatalogModel>, } impl Catalog { + /// The catalog `codex debug models` prints, which costs no tokens to + /// read. + pub fn read() -> Result<Catalog> { + let output = Command::new("codex") + .args(["debug", "models"]) + .stdin(Stdio::null()) + .output() + .context("could not run codex debug models to check the Model and Effort")?; + if !output.status.success() { + bail!( + "codex debug models failed, so the Model and Effort can't be checked: {}", + said(&output) + ); + } + Catalog::parse(&String::from_utf8_lossy(&output.stdout)) + } + /// The catalog `codex debug models` printed as `json`. fn parse(json: &str) -> Result<Catalog> { let parsed: Value = serde_json::from_str(json) @@ -355,12 +334,41 @@ impl Catalog { Ok(Catalog { models }) } + /// The Models it lists to choose from, each as its slug and, where it + /// has one, its display name, as in `gpt-6.1-sol (GPT-6.1-Sol)`. + pub fn listed(&self) -> Vec<String> { + self.models + .iter() + .filter(|model| model.listed) + .map(|model| match &model.display_name { + Some(display) => format!("{} ({display})", model.slug), + None => model.slug.clone(), + }) + .collect() + } + + /// The Efforts the Model `slug` supports, or with no Model, those any + /// Model supports, each once, in the catalog's order. + pub fn efforts(&self, slug: Option<&str>) -> Vec<&str> { + let mut efforts = Vec::new(); + let of = self + .models + .iter() + .filter(|model| slug.is_none_or(|slug| model.slug == slug)); + for each in of.flat_map(|model| &model.efforts) { + if !efforts.contains(&each.as_str()) { + efforts.push(each.as_str()); + } + } + efforts + } + /// `model` and `effort` as Codex names them: the slug of the Model whose /// slug or display name is `model`, regardless of case, and the Effort, /// of those that Model supports, or with no Model of those any Model /// supports, that is `effort`, regardless of case. Fails naming the /// valid choices for a Model or Effort the catalog doesn't have. - fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result<ModelAndEffort> { + pub fn settle(&self, model: Option<&str>, effort: Option<&str>) -> Result<ModelAndEffort> { let found = match model { Some(model) => Some(self.model(model)?), None => None, @@ -371,18 +379,7 @@ impl Catalog { effort: None, }); }; - let efforts: Vec<&str> = match found { - Some(found) => found.efforts.iter().map(String::as_str).collect(), - None => { - let mut efforts = Vec::new(); - for each in self.models.iter().flat_map(|model| &model.efforts) { - if !efforts.contains(&each.as_str()) { - efforts.push(each.as_str()); - } - } - efforts - } - }; + let efforts = self.efforts(found.map(|found| found.slug.as_str())); let Some(settled) = efforts .iter() .find(|supported| supported.eq_ignore_ascii_case(effort)) @@ -716,6 +713,25 @@ mod tests { ); } + #[test] + fn the_catalog_lists_its_listed_models_by_slug_and_display_name() { + assert_eq!( + catalog().listed(), + ["gpt-6.1-sol (GPT-6.1-Sol)", "gpt-5.5 (GPT-5.5 Classic)"] + ); + } + + #[test] + fn the_efforts_are_the_models_or_with_no_model_any_models_each_once() { + let catalog = catalog(); + + assert_eq!(catalog.efforts(Some("gpt-5.5")), ["low", "xhigh"]); + assert_eq!( + catalog.efforts(None), + ["low", "high", "max", "xhigh", "minimal"] + ); + } + #[test] fn output_that_is_no_catalog_fails() { for output in ["", "not json", r#"{"models": 3}"#] { diff --git a/src/questions.rs b/src/questions.rs index b538111ed..a5bee431a 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -13,7 +13,7 @@ use signal_hook::low_level; use crate::config::UserConfig; use crate::email::DEFAULT_FROM; -use crate::harness::{self, Choice, ChosenBy, Harness, ModelAndEffort, Unavailable}; +use crate::harness::{self, Catalog, Choice, ChosenBy, Harness, ModelAndEffort}; use crate::resend_key::{self, Credentials, Source}; /// What the user chose. @@ -159,75 +159,117 @@ pub fn ask( }) } -/// Ask which Harness every Run's sessions run on, listing each with why -/// sessions can't run on it here, if they can't, and refusing that one; then -/// its Model and Effort, with those `current` sets for it as the defaults. -/// The Harness's default is the first one sessions can run on, so `claude` -/// when both can. A Model is checked with a test call, as a Run -/// checks it, and on a refusal the Model and Effort are asked again. With no +/// Ask which Harness every Run's sessions run on, listing each, marked where +/// it isn't installed, and refusing that one; then its Model and Effort, +/// with those `current` sets for it as the defaults, checked as a Run +/// checks them, as [`ask_claude`] and [`ask_codex`] ask them. The Harness's +/// default is the first one installed, so `claude` when both are. With no /// Harness to choose, nothing is asked, and the answer is `None`. fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndEffort)>> { let listed: Vec<String> = Harness::ALL .iter() - .map(|harness| match harness.unavailable() { - Some(why) => format!("{} ({why})", harness.name()), - None => harness.name().to_string(), + .map(|harness| { + if harness.installed() { + harness.name().to_string() + } else { + format!("{} (not installed)", harness.name()) + } }) .collect(); let listed = listed.join(" or "); - let default = Harness::ALL - .into_iter() - .find(|harness| harness.unavailable().is_none()); + let default = Harness::ALL.into_iter().find(|harness| harness.installed()); let Some(default) = default else { say(&format!( "Harness for every Run's sessions: {listed}. Sessions can run on neither here, so \ - the harness settings stay as they are; install claude, then rerun `thirdshift setup`." + the harness settings stay as they are; install claude or codex, then rerun \ + `thirdshift setup`." )); return Ok(None); }; - let harness = loop { + loop { let name = answer( &format!("Harness for every Run's sessions, {listed}"), Some(default.name()), )?; - match Harness::named(&name) { - None => say(&format!("Choose {}.", harness::NAMES)), - Some(harness) => match harness.unavailable() { - None => break harness, - Some(Unavailable::NotSupportedYet) => say(&format!( - "{name} is not supported yet: Codex sessions come in a later thirdshift." - )), - Some(Unavailable::NotInstalled) => { - say(&format!("{name} is not installed: it isn't on PATH.")) + let harness = match Harness::named(&name) { + None => { + say(&format!("Choose {}.", harness::NAMES)); + continue; + } + Some(harness) if !harness.installed() => { + say(&format!("{name} is not installed: it isn't on PATH.")); + continue; + } + Some(harness) => harness, + }; + let current = current.of(harness); + let chosen = match harness { + Harness::Claude => ask_claude(current)?, + Harness::Codex => match Catalog::read() { + Ok(catalog) => ask_codex(&catalog, current)?, + Err(error) => { + say(&format!("{error:#}")); + continue; } }, - } - }; - let current_settings = current.of(harness); + }; + return Ok(Some((harness, chosen))); + } +} + +/// Ask Claude's Model and Effort, with `current` as the defaults. A Model is +/// checked with a test call, as a Run checks it, and on a refusal the Model +/// and Effort are asked again. +fn ask_claude(current: &ModelAndEffort) -> Result<ModelAndEffort> { loop { - let model = ask_setting("Model", harness, current_settings.model.as_deref())?; - let effort = ask_setting("Effort", harness, current_settings.effort.as_deref())?; + let model = ask_setting("Model", Harness::Claude, current.model.as_deref())?; + let effort = ask_setting("Effort", Harness::Claude, current.effort.as_deref())?; let choice = Choice { - harness, + harness: Harness::Claude, model, effort, chosen_by: ChosenBy::UserConfig, }; match choice.test_call() { Ok(()) => { - return Ok(Some(( - harness, - ModelAndEffort { - model: choice.model, - effort: choice.effort, - }, - ))); + return Ok(ModelAndEffort { + model: choice.model, + effort: choice.effort, + }); } Err(error) => say(&format!("{error:#}")), } } } +/// Ask Codex's Model, listing those in its `catalog`, then its Effort, +/// listing those the Model chosen supports, with `current` as the defaults. +/// Each is matched against the catalog as a Run matches it, and written as +/// Codex names it; one the catalog doesn't have is asked again, with the +/// valid choices. +fn ask_codex(catalog: &Catalog, current: &ModelAndEffort) -> Result<ModelAndEffort> { + say(&format!("Codex's Models: {}", catalog.listed().join(", "))); + let model = loop { + let model = ask_setting("Model", Harness::Codex, current.model.as_deref())?; + match catalog.settle(model.as_deref(), None) { + Ok(settled) => break settled.model, + Err(error) => say(&format!("{error:#}")), + } + }; + let efforts = catalog.efforts(model.as_deref()).join(", "); + say(&match &model { + Some(model) => format!("Efforts {model} supports: {efforts}"), + None => format!("Efforts Codex's Models support: {efforts}"), + }); + loop { + let effort = ask_setting("Effort", Harness::Codex, current.effort.as_deref())?; + match catalog.settle(model.as_deref(), effort.as_deref()) { + Ok(settled) => return Ok(settled), + Err(error) => say(&format!("{error:#}")), + } + } +} + /// Ask for the `setting`, the Model or the Effort, of `harness`, with /// `current` as the default; `-` is none, the Harness's own default. fn ask_setting(setting: &str, harness: Harness, current: Option<&str>) -> Result<Option<String>> { diff --git a/tests/setup.rs b/tests/setup.rs index c2c9e3106..5317c59db 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -5,8 +5,10 @@ //! From a terminal, it first asks the Setup questions on stderr, each with the //! current value as its default answer. Base fixes are asked about only //! with every Run a Merge run. The Harness question lists both Harnesses, -//! refusing one sessions can't run on here, and the Model and Effort are -//! asked for the Harness chosen, a Claude Model checked with a test call. +//! refusing one that isn't installed, and the Model and Effort are asked for +//! the Harness chosen: a Claude Model checked with a test call, and a Codex +//! Model and Effort chosen from Codex's catalog and written as Codex names +//! them. mod support; @@ -1410,8 +1412,9 @@ exit 1 /// The answers that leave every setting but the Harness's as it is. const NOTHING_ELSE: [Keystrokes; 3] = [(MERGE, ""), (PULL, ""), (NOTIFY, "")]; -/// `PATH` with the scenario's fakes and the system's tools, and so no -/// `codex`, whatever is installed on the machine the tests run on. +/// `PATH` with the scenario's fakes and the system's tools, and so no other +/// `claude` or `codex`, whatever is installed on the machine the tests run +/// on. fn fakes_only_path(scenario: &Scenario) -> String { format!("{}:/usr/bin:/bin", scenario.path("bin").display()) } @@ -1444,9 +1447,9 @@ fn on_a_terminal_the_harness_question_lists_both_harnesses_and_defaults_to_claud let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); assert!( - result.stderr.contains( - "Harness for every Run's sessions, claude or codex (not supported yet) [claude]: " - ), + result + .stderr + .contains("Harness for every Run's sessions, claude or codex [claude]: "), "terminal: {}", result.stderr ); @@ -1466,23 +1469,24 @@ fn on_a_terminal_the_harness_question_lists_both_harnesses_and_defaults_to_claud } #[test] -fn on_a_terminal_a_harness_sessions_cant_run_on_is_refused_and_asked_again() { +fn on_a_terminal_a_harness_not_installed_is_refused_and_asked_again() { let scenario = Scenario::new(); scenario.git_email_is(None); + fs::remove_file(scenario.path("bin/claude")).unwrap(); let path = fakes_only_path(&scenario); let mut keystrokes = vec![ - (HARNESS, "codex"), - (HARNESS, "gemini"), (HARNESS, "claude"), - (MODEL, ""), - (EFFORT, ""), + (HARNESS, "gemini"), + (HARNESS, "codex"), + (CODEX_MODEL, ""), + (CODEX_EFFORT, ""), ]; keystrokes.extend(NOTHING_ELSE); let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); for refusal in [ - "codex is not supported yet: Codex sessions come in a later thirdshift.", + "claude is not installed: it isn't on PATH.", "Choose claude or codex.", ] { assert!( @@ -1491,18 +1495,16 @@ fn on_a_terminal_a_harness_sessions_cant_run_on_is_refused_and_asked_again() { result.stderr ); } - assert_eq!( - table(&result)["harness"]["default"].as_str(), - Some("claude") - ); + assert_eq!(table(&result)["harness"]["default"].as_str(), Some("codex")); } #[test] -fn on_a_terminal_with_claude_not_installed_the_harness_is_left_as_it_was() { +fn on_a_terminal_with_neither_harness_installed_the_harness_is_left_as_it_was() { let scenario = Scenario::new(); scenario.git_email_is(None); scenario.user_config_is("[harness.claude]\nmodel = \"opus\"\n"); fs::remove_file(scenario.path("bin/claude")).unwrap(); + fs::remove_file(scenario.path("bin/codex")).unwrap(); let path = fakes_only_path(&scenario); let result = setup_on_terminal(&scenario, &[("PATH", &path)], &NOTHING_ELSE); @@ -1510,7 +1512,7 @@ fn on_a_terminal_with_claude_not_installed_the_harness_is_left_as_it_was() { assert!( result .stderr - .contains("claude (not installed) or codex (not supported yet)"), + .contains("claude (not installed) or codex (not installed)"), "terminal: {}", result.stderr ); @@ -1611,3 +1613,152 @@ fn on_a_terminal_a_model_claude_refuses_is_asked_again_with_claudes_error() { ); assert_eq!(test_calls(&scenario).len(), 2); } + +// Codex. + +const CODEX_MODEL: &str = "Model for codex"; +const CODEX_EFFORT: &str = "Effort for codex"; + +/// How Setup lists the fake `codex`'s catalog of Models. +const CODEX_MODELS: &str = + "Codex's Models: gpt-6.1-sol (GPT-6.1-Sol), gpt-6-luna (GPT-6-Luna), gpt-5.5 (GPT-5.5)"; + +#[test] +fn on_a_terminal_codex_is_offered_with_its_models_and_the_chosen_models_efforts() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![ + (HARNESS, "codex"), + (CODEX_MODEL, "gpt-5.5"), + (CODEX_EFFORT, "high"), + ]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + for shown in [ + "Harness for every Run's sessions, claude or codex [claude]: ", + CODEX_MODELS, + "Efforts gpt-5.5 supports: low, medium, high, xhigh", + ] { + assert!(result.stderr.contains(shown), "terminal: {}", result.stderr); + } + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-5.5") + ); + assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("high")); + assert_eq!(config["harness"]["claude"]["model"].as_str(), Some("")); + assert!(test_calls(&scenario).is_empty()); + assert!(scenario.codex_calls().is_empty()); +} + +#[test] +fn on_a_terminal_with_only_codex_installed_codex_is_the_default_answer() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + fs::remove_file(scenario.path("bin/claude")).unwrap(); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![(HARNESS, ""), (CODEX_MODEL, ""), (CODEX_EFFORT, "")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + for shown in [ + "Harness for every Run's sessions, claude (not installed) or codex [codex]: ", + "Efforts Codex's Models support: low, medium, high, xhigh, max, ultra", + ] { + assert!(result.stderr.contains(shown), "terminal: {}", result.stderr); + } + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!(config["harness"]["codex"]["model"].as_str(), Some("")); + assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("")); +} + +#[test] +fn on_a_terminal_a_codex_display_name_and_capitalised_effort_are_written_as_codex_names_them() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![ + (HARNESS, "codex"), + (CODEX_MODEL, "GPT-6.1-Sol"), + (CODEX_EFFORT, "Max"), + ]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + let config = table(&result); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-6.1-sol") + ); + assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("max")); +} + +#[test] +fn on_a_terminal_an_unknown_codex_model_or_unsupported_effort_is_asked_again_with_the_choices() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![ + (HARNESS, "codex"), + (CODEX_MODEL, "gpt-7"), + (CODEX_MODEL, "GPT-5.5"), + (CODEX_EFFORT, "max"), + (CODEX_EFFORT, "XHigh"), + ]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + for refusal in [ + "the Model gpt-7 is not in Codex's catalog: choose one of gpt-6.1-sol, gpt-6-luna, \ + gpt-5.5", + "the Effort max is not one the Codex Model gpt-5.5 supports: choose one of low, \ + medium, high, xhigh", + ] { + assert!( + result.stderr.contains(refusal), + "terminal: {}", + result.stderr + ); + } + let config = table(&result); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-5.5") + ); + assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("xhigh")); +} + +#[test] +fn on_a_terminal_the_codex_model_and_effort_default_to_the_current_ones_as_codex_names_them() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.user_config_is("[harness.codex]\nmodel = \"GPT-6-Luna\"\neffort = \"High\"\n"); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![(HARNESS, "codex"), (CODEX_MODEL, ""), (CODEX_EFFORT, "")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + for shown in [ + "Model for codex, - for codex's own default [GPT-6-Luna]: ", + "Efforts gpt-6-luna supports: low, medium, high, xhigh, max", + "Effort for codex, - for codex's own default [High]: ", + ] { + assert!(result.stderr.contains(shown), "terminal: {}", result.stderr); + } + let config = table(&result); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-6-luna") + ); + assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("high")); +} From 70ccc3cbd9765176bc50fdaf513bea2e76f6e657 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 01:46:52 -0400 Subject: [PATCH 12/14] Review clean-ups: a Harness's stop signals on Harness, and clearer killed-work tracking (#395) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/harness.rs | 10 ++++++++++ src/progress/codex.rs | 32 +++++++++++++++++++------------- src/session.rs | 12 +----------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/harness.rs b/src/harness.rs index 42cc3a500..4c01c6bb2 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -39,6 +39,16 @@ impl Harness { } } + /// The signals that ask a session on it to stop, in the order they are + /// sent: SIGTERM, after SIGINT for Codex, which stops cleanly only on + /// SIGINT, interrupting its turn. + pub fn stop_signals(self) -> &'static [libc::c_int] { + match self { + Harness::Claude => &[libc::SIGTERM], + Harness::Codex => &[libc::SIGINT, libc::SIGTERM], + } + } + /// Every Harness, in the order Setup lists them. pub const ALL: [Harness; 2] = [Harness::Claude, Harness::Codex]; diff --git a/src/progress/codex.rs b/src/progress/codex.rs index a9d9a5e33..605ac1fcd 100644 --- a/src/progress/codex.rs +++ b/src/progress/codex.rs @@ -64,15 +64,25 @@ impl CodexProgress { } } + /// Track whether the command or sub-agent call `item`, with id `id`, is + /// still running, as `description`. + fn track_running(&mut self, id: &str, item: &Value, description: String) { + self.running.retain(|(running, _)| running != id); + if item["status"] == "in_progress" { + self.running.push((id.to_string(), description)); + } + } + /// Track an item's state, and give its line the first time it is seen. fn item(&mut self, item: &Value) -> Vec<String> { let id = item["id"].as_str().unwrap_or("").to_string(); match item["type"].as_str() { - Some("command_execution" | "collab_tool_call") => { - self.running.retain(|(running, _)| *running != id); - if item["status"] == "in_progress" { - self.running.push((id.clone(), running_work(item))); - } + Some("command_execution") => { + let command = unwrapped(item["command"].as_str().unwrap_or("")); + self.track_running(&id, item, command); + } + Some("collab_tool_call") => { + self.track_running(&id, item, sub_agent_call(item)); } Some("agent_message") => { if let Some(text) = item["text"].as_str() { @@ -210,14 +220,10 @@ impl Stream for CodexProgress { } } -/// How a command or sub-agent call left running is described: a command as -/// it was run, without its shell, and a sub-agent call by its tool and the -/// first line of the prompt it gave, if any, as in `agent spawn_agent: -/// Review the diff`. -fn running_work(item: &Value) -> String { - if item["type"] == "command_execution" { - return unwrapped(item["command"].as_str().unwrap_or("")); - } +/// The sub-agent call `item` as killed background work is described: by its +/// tool and the first line of the prompt it gave, if any, as in `agent +/// spawn_agent: Review the diff`. +fn sub_agent_call(item: &Value) -> String { let tool = item["tool"].as_str().unwrap_or("?"); match item["prompt"] .as_str() diff --git a/src/session.rs b/src/session.rs index 6f42db211..b908010d3 100644 --- a/src/session.rs +++ b/src/session.rs @@ -350,7 +350,7 @@ const STOP_GRACE: Duration = Duration::from_secs(10); /// then SIGKILL. fn stop(child: &mut Child, harness: Harness) { let group = -(child.id() as libc::pid_t); - for &signal in stop_signals(harness) { + for &signal in harness.stop_signals() { // SAFETY: kill has no memory-safety preconditions. unsafe { libc::kill(group, signal) }; let deadline = Instant::now() + STOP_GRACE; @@ -366,16 +366,6 @@ fn stop(child: &mut Child, harness: Harness) { let _ = child.wait(); } -/// The signals that ask a session on `harness` to stop, in the order they -/// are sent: SIGTERM, after SIGINT for Codex, which stops cleanly only on -/// SIGINT, interrupting its turn. -fn stop_signals(harness: Harness) -> &'static [libc::c_int] { - match harness { - Harness::Claude => &[libc::SIGTERM], - Harness::Codex => &[libc::SIGINT, libc::SIGTERM], - } -} - /// Copy every line of `stream` to `log_file` and print the progress lines it /// condenses to, until the stream ends. fn follow( From 07012258bdf3ae0e88c98fbaf6ffd6ec948a496f Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 02:01:11 -0400 Subject: [PATCH 13/14] Review clean-ups for Setup offering Codex (#396) When codex debug models fails, Setup says so and leaves the harness settings as they are, rather than asking the Harness again, which with only codex installed could never end. The catalog's errors no longer speak of checking, as Setup reads it to list the choices, and locals in ask_harness and Catalog::efforts get names that say what they hold. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/harness.rs | 8 +++---- src/questions.rs | 56 ++++++++++++++++++++++++------------------------ tests/setup.rs | 39 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 32 deletions(-) diff --git a/src/harness.rs b/src/harness.rs index e6f3684eb..74c8a4e46 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -299,10 +299,10 @@ impl Catalog { .args(["debug", "models"]) .stdin(Stdio::null()) .output() - .context("could not run codex debug models to check the Model and Effort")?; + .context("could not run codex debug models to read Codex's Models")?; if !output.status.success() { bail!( - "codex debug models failed, so the Model and Effort can't be checked: {}", + "codex debug models failed, so Codex's Models can't be read: {}", said(&output) ); } @@ -351,11 +351,11 @@ impl Catalog { /// Model supports, each once, in the catalog's order. pub fn efforts(&self, slug: Option<&str>) -> Vec<&str> { let mut efforts = Vec::new(); - let of = self + let models = self .models .iter() .filter(|model| slug.is_none_or(|slug| model.slug == slug)); - for each in of.flat_map(|model| &model.efforts) { + for each in models.flat_map(|model| &model.efforts) { if !efforts.contains(&each.as_str()) { efforts.push(each.as_str()); } diff --git a/src/questions.rs b/src/questions.rs index a5bee431a..b9e19b6cd 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -164,9 +164,10 @@ pub fn ask( /// with those `current` sets for it as the defaults, checked as a Run /// checks them, as [`ask_claude`] and [`ask_codex`] ask them. The Harness's /// default is the first one installed, so `claude` when both are. With no -/// Harness to choose, nothing is asked, and the answer is `None`. +/// Harness to choose, or Codex chosen and its catalog unreadable, the answer +/// is `None`. fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndEffort)>> { - let listed: Vec<String> = Harness::ALL + let harnesses: Vec<String> = Harness::ALL .iter() .map(|harness| { if harness.installed() { @@ -176,45 +177,44 @@ fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndE } }) .collect(); - let listed = listed.join(" or "); + let harnesses = harnesses.join(" or "); let default = Harness::ALL.into_iter().find(|harness| harness.installed()); let Some(default) = default else { say(&format!( - "Harness for every Run's sessions: {listed}. Sessions can run on neither here, so \ + "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ the harness settings stay as they are; install claude or codex, then rerun \ `thirdshift setup`." )); return Ok(None); }; - loop { + let harness = loop { let name = answer( - &format!("Harness for every Run's sessions, {listed}"), + &format!("Harness for every Run's sessions, {harnesses}"), Some(default.name()), )?; - let harness = match Harness::named(&name) { - None => { - say(&format!("Choose {}.", harness::NAMES)); - continue; - } + match Harness::named(&name) { + None => say(&format!("Choose {}.", harness::NAMES)), Some(harness) if !harness.installed() => { - say(&format!("{name} is not installed: it isn't on PATH.")); - continue; + say(&format!("{name} is not installed: it isn't on PATH.")) } - Some(harness) => harness, - }; - let current = current.of(harness); - let chosen = match harness { - Harness::Claude => ask_claude(current)?, - Harness::Codex => match Catalog::read() { - Ok(catalog) => ask_codex(&catalog, current)?, - Err(error) => { - say(&format!("{error:#}")); - continue; - } - }, - }; - return Ok(Some((harness, chosen))); - } + Some(harness) => break harness, + } + }; + let set = current.of(harness); + let chosen = match harness { + Harness::Claude => ask_claude(set)?, + Harness::Codex => match Catalog::read() { + Ok(catalog) => ask_codex(&catalog, set)?, + Err(error) => { + say(&format!( + "{error:#}\nThe harness settings stay as they are; rerun `thirdshift \ + setup` once codex debug models works." + )); + return Ok(None); + } + }, + }; + Ok(Some((harness, chosen))) } /// Ask Claude's Model and Effort, with `current` as the defaults. A Model is diff --git a/tests/setup.rs b/tests/setup.rs index 5317c59db..4a2681041 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -1762,3 +1762,42 @@ fn on_a_terminal_the_codex_model_and_effort_default_to_the_current_ones_as_codex ); assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("high")); } + +#[test] +fn on_a_terminal_when_codexs_catalog_cant_be_read_the_harness_is_left_as_it_was() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.user_config_is("[harness.codex]\nmodel = \"gpt-5.5\"\n"); + // The fake is a link to the build every test shares, so it's replaced, + // not written through. + let codex = scenario.path("bin/codex"); + fs::remove_file(&codex).unwrap(); + fs::write(&codex, "#!/bin/sh\necho 'not logged in' >&2\nexit 1\n").unwrap(); + fs::set_permissions(&codex, fs::Permissions::from_mode(0o755)).unwrap(); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![(HARNESS, "codex")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + assert!( + result.stderr.contains( + "codex debug models failed, so Codex's Models can't be read: not logged in\n\ + The harness settings stay as they are; rerun `thirdshift setup` once codex debug \ + models works." + ), + "terminal: {}", + result.stderr + ); + assert!( + !result.stderr.contains(CODEX_MODEL), + "terminal: {}", + result.stderr + ); + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("claude")); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-5.5") + ); +} From 369a2670546eef78d62f1ce3e005a5e2896034e6 Mon Sep 17 00:00:00 2001 From: Jacob Stephens <jacob@stephens.page> Date: Mon, 5 Oct 2026 02:21:51 -0400 Subject: [PATCH 14/14] Review clean-ups for Spec #136: prompts worded for either Harness, Setup keeps the configured Harness The headless and Resume lines no longer name Claude's Bash timeout or TaskStop tool unconditionally, since Codex sessions get them too. Re-running Setup offers the Harness already in the User config as the default answer, when it's installed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- prompts/architecture-review.md | 2 +- prompts/ci-fix-repair-inherited.md | 2 +- prompts/ci-fix-repair.md | 2 +- prompts/conflict-repair.md | 2 +- prompts/continuation-pr.md | 2 +- prompts/continuation.md | 2 +- prompts/foreign-conflict-repair.md | 2 +- prompts/fresh.md | 2 +- prompts/resume.md | 4 ++-- prompts/review-repair.md | 2 +- prompts/spec-review.md | 2 +- site/prompts/index.html | 24 ++++++++++++------------ src/prompt.rs | 12 ++++++------ src/questions.rs | 8 ++++++-- tests/ci.rs | 4 ++-- tests/codex_sessions.rs | 9 ++++++++- tests/fresh_run.rs | 2 +- tests/mergeable.rs | 2 +- tests/setup.rs | 28 ++++++++++++++++++++++++++++ 19 files changed, 76 insertions(+), 37 deletions(-) diff --git a/prompts/architecture-review.md b/prompts/architecture-review.md index b3f32c7fa..2ddb12490 100644 --- a/prompts/architecture-review.md +++ b/prompts/architecture-review.md @@ -24,5 +24,5 @@ Architecture review plan: <URL of the Spec or the single Ticket you published> Architecture review idea: <URL of the issue you filed> Architecture review already filed: <URL of the open issue that already covers it> -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/ci-fix-repair-inherited.md b/prompts/ci-fix-repair-inherited.md index d86f1ba29..55a40f2c4 100644 --- a/prompts/ci-fix-repair-inherited.md +++ b/prompts/ci-fix-repair-inherited.md @@ -18,5 +18,5 @@ Run the affected checks locally, commit, and push <branch>. If a failure is not caused by this branch (it is flaky, or also fails on <base>), do not change code for it. Instead, add it to a "CI notes" section of the pull request body with a one-line explanation. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/ci-fix-repair.md b/prompts/ci-fix-repair.md index 72f58c217..692cd9229 100644 --- a/prompts/ci-fix-repair.md +++ b/prompts/ci-fix-repair.md @@ -15,5 +15,5 @@ Run the affected checks locally, commit, and push <branch>. If a failure is not caused by this branch (it is flaky, or also fails on <base>), do not change code for it. Instead, add it to a "CI notes" section of the pull request body with a one-line explanation. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/conflict-repair.md b/prompts/conflict-repair.md index 150c3506c..e390a56f1 100644 --- a/prompts/conflict-repair.md +++ b/prompts/conflict-repair.md @@ -12,5 +12,5 @@ A merge of origin/<base> into <branch> is in progress in this worktree and has c Resolve the conflicts, finish the merge, and push <branch>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/continuation-pr.md b/prompts/continuation-pr.md index 33e37eec6..b49c30d3b 100644 --- a/prompts/continuation-pr.md +++ b/prompts/continuation-pr.md @@ -21,5 +21,5 @@ In the PR body, add an "Unaddressed findings" section listing each skipped findi Include "Closes #<n>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/continuation.md b/prompts/continuation.md index 8bde6ee19..8e644e936 100644 --- a/prompts/continuation.md +++ b/prompts/continuation.md @@ -21,5 +21,5 @@ In the PR body, add an "Unaddressed findings" section listing each skipped findi Include "Closes #<n>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/foreign-conflict-repair.md b/prompts/foreign-conflict-repair.md index 99a90c54a..9ff9c9857 100644 --- a/prompts/foreign-conflict-repair.md +++ b/prompts/foreign-conflict-repair.md @@ -12,5 +12,5 @@ A merge of origin/<branch> into <branch> is in progress in this worktree and has Resolve the conflicts, finish the merge, and push <branch>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/fresh.md b/prompts/fresh.md index b21fce162..faf2bf246 100644 --- a/prompts/fresh.md +++ b/prompts/fresh.md @@ -11,5 +11,5 @@ Address the Standards and Spec findings you agree with. Push branch <branch> and create a pull request against <base> using the `thirdshift-pr` skill, marked ready for review. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. Include "Closes #<n>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/resume.md b/prompts/resume.md index 38b18f23c..7f330895c 100644 --- a/prompts/resume.md +++ b/prompts/resume.md @@ -7,7 +7,7 @@ Continues any session, once, that ended its turn with background work still runn ``` Your background work (<background work>) was killed when your turn ended, because ending the turn ends the session. -Re-run whatever you were waiting on in the foreground, then finish your job. If the re-run hangs or is moved to the background again, stop it with the `TaskStop` tool, by its task id, and say what could not be run, rather than leaving it running. +Re-run whatever you were waiting on in the foreground, then finish your job. If the re-run hangs or is moved to the background again, stop it by its task id (with the `TaskStop` tool, if you have it), and say what could not be run, rather than leaving it running. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/review-repair.md b/prompts/review-repair.md index 3e230acd8..c53c0cdd2 100644 --- a/prompts/review-repair.md +++ b/prompts/review-repair.md @@ -17,5 +17,5 @@ Add each skipped finding to the "Unaddressed findings" section of the pull reque Commit and push <branch>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/prompts/spec-review.md b/prompts/spec-review.md index a22dd00aa..fcb0b94c4 100644 --- a/prompts/spec-review.md +++ b/prompts/spec-review.md @@ -21,5 +21,5 @@ In the PR body, add an "Unaddressed findings" section listing each skipped findi Include "Closes #<n>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. ``` diff --git a/site/prompts/index.html b/site/prompts/index.html index 31c82a603..a2d8cc5eb 100644 --- a/site/prompts/index.html +++ b/site/prompts/index.html @@ -56,7 +56,7 @@ <h3 id="prompt-fresh-title">Fresh</h3> Push branch <mark class="ph"><branch></mark> and create a pull request against <mark class="ph"><base></mark> using the `thirdshift-pr` skill, marked ready for review. In the PR body, add an "Unaddressed findings" section listing each skipped finding under Standards or Spec, with at least a one-line reason. Include "Closes #<mark class="ph"><n></mark>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-continuation" aria-labelledby="prompt-continuation-title"> @@ -79,7 +79,7 @@ <h3 id="prompt-continuation-title">Continuation, with no pull request</h3> Include "Closes #<mark class="ph"><n></mark>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-continuation-pr" aria-labelledby="prompt-continuation-pr-title"> @@ -102,7 +102,7 @@ <h3 id="prompt-continuation-pr-title">Continuation, with an open pull request</h Include "Closes #<mark class="ph"><n></mark>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-spec-review" aria-labelledby="prompt-spec-review-title"> @@ -125,7 +125,7 @@ <h3 id="prompt-spec-review-title">Spec review</h3> Include "Closes #<mark class="ph"><n></mark>" in the PR body. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-architecture-review" aria-labelledby="prompt-architecture-review-title"> @@ -151,7 +151,7 @@ <h3 id="prompt-architecture-review-title">Architecture review</h3> Architecture review idea: <URL of the issue you filed> Architecture review already filed: <URL of the open issue that already covers it> -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-conflict-repair" aria-labelledby="prompt-conflict-repair-title"> @@ -165,7 +165,7 @@ <h3 id="prompt-conflict-repair-title">Conflict Repair</h3> Resolve the conflicts, finish the merge, and push <mark class="ph"><branch></mark>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-foreign-conflict-repair" aria-labelledby="prompt-foreign-conflict-repair-title"> @@ -179,7 +179,7 @@ <h3 id="prompt-foreign-conflict-repair-title">Conflict Repair, on Foreign commit Resolve the conflicts, finish the merge, and push <mark class="ph"><branch></mark>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-review-repair" aria-labelledby="prompt-review-repair-title"> @@ -198,7 +198,7 @@ <h3 id="prompt-review-repair-title">Review Repair</h3> Commit and push <mark class="ph"><branch></mark>. Do not rebase or force-push. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-ci-fix-repair" aria-labelledby="prompt-ci-fix-repair-title"> @@ -215,7 +215,7 @@ <h3 id="prompt-ci-fix-repair-title">CI-fix Repair</h3> If a failure is not caused by this branch (it is flaky, or also fails on <mark class="ph"><base></mark>), do not change code for it. Instead, add it to a "CI notes" section of the pull request body with a one-line explanation. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-ci-fix-repair-inherited" aria-labelledby="prompt-ci-fix-repair-inherited-title"> @@ -235,7 +235,7 @@ <h3 id="prompt-ci-fix-repair-inherited-title">CI-fix Repair, with Inherited fail If a failure is not caused by this branch (it is flaky, or also fails on <mark class="ph"><base></mark>), do not change code for it. Instead, add it to a "CI notes" section of the pull request body with a one-line explanation. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> <article class="job-sheet" id="prompt-resume" aria-labelledby="prompt-resume-title"> @@ -244,9 +244,9 @@ <h3 id="prompt-resume-title">Resume</h3> <p class="sent-by">Sent by <a href="/#unit-2">2 · M · Implement</a>, <a href="/#unit-3">3 · Y · Review</a> or <a href="/#unit-4">4 · K · Finish</a></p> <pre class="job-text"><code>Your background work (<mark class="ph"><background work></mark>) was killed when your turn ended, because ending the turn ends the session. -Re-run whatever you were waiting on in the foreground, then finish your job. If the re-run hangs or is moved to the background again, stop it with the `TaskStop` tool, by its task id, and say what could not be run, rather than leaving it running. +Re-run whatever you were waiting on in the foreground, then finish your job. If the re-run hangs or is moved to the background again, stop it by its task id (with the `TaskStop` tool, if you have it), and say what could not be run, rather than leaving it running. -You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on. +You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on. </code></pre> </article> </div> diff --git a/src/prompt.rs b/src/prompt.rs index 493e86e09..0e6fdc375 100644 --- a/src/prompt.rs +++ b/src/prompt.rs @@ -4,20 +4,20 @@ use crate::ci::{self, FailedChecks}; use crate::issue::IssueUrl; use crate::labels::{NEEDS_TRIAGE, READY_FOR_AGENT}; -/// Every prompt ends with this. Sessions run with `claude -p`, which exits +/// Every prompt ends with this, worded for either Harness. A session exits /// once the agent ends its turn, killing any background task still running. /// Other sessions on the same machine run the same commands, so a wait on a /// process name can match theirs and outlast the session's own task. /// thirdshift can't tell a task the agent gave up on from one it was waiting -/// on, so the agent is to stop the first kind itself, with `TaskStop`: auto -/// mode may deny it a `kill`. +/// on, so the agent is to stop the first kind itself, on Claude with +/// `TaskStop`: auto mode may deny it a `kill`. const HEADLESS: &str = "You run headless: nobody is watching, and ending your turn ends the session. \ - Run tests and other long commands in the foreground, raising the Bash timeout if needed. \ + Run tests and other long commands in the foreground, raising the command's timeout if needed. \ If a command is moved to the background, wait for that task by its own task id or output file, \ never by process names or patterns (`pgrep`, `ps | grep`, and the like): \ other sessions on this machine run the same commands. \ Never end your turn while a background task you depend on is still running: ending the turn kills it. \ - Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: \ + Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): \ a task still running when your turn ends is taken as work you were waiting on.\n"; /// The fresh prompt, for a run that starts a new Issue branch. @@ -216,7 +216,7 @@ pub fn resume(killed: &[&str]) -> String { "Your background work ({killed}) was killed when your turn ended, because ending the turn ends the session.\n\ \n\ Re-run whatever you were waiting on in the foreground, then finish your job. \ - If the re-run hangs or is moved to the background again, stop it with the `TaskStop` tool, by its task id, \ + If the re-run hangs or is moved to the background again, stop it by its task id (with the `TaskStop` tool, if you have it), \ and say what could not be run, rather than leaving it running.\n\ \n\ {HEADLESS}", diff --git a/src/questions.rs b/src/questions.rs index b9e19b6cd..36bae4812 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -163,7 +163,8 @@ pub fn ask( /// it isn't installed, and refusing that one; then its Model and Effort, /// with those `current` sets for it as the defaults, checked as a Run /// checks them, as [`ask_claude`] and [`ask_codex`] ask them. The Harness's -/// default is the first one installed, so `claude` when both are. With no +/// default is the `current` one if it's installed, else the first one +/// installed, so `claude` when both are. With no /// Harness to choose, or Codex chosen and its catalog unreadable, the answer /// is `None`. fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndEffort)>> { @@ -178,7 +179,10 @@ fn ask_harness(current: &harness::Settings) -> Result<Option<(Harness, ModelAndE }) .collect(); let harnesses = harnesses.join(" or "); - let default = Harness::ALL.into_iter().find(|harness| harness.installed()); + let default = current + .default + .filter(|harness| harness.installed()) + .or_else(|| Harness::ALL.into_iter().find(|harness| harness.installed())); let Some(default) = default else { say(&format!( "Harness for every Run's sessions: {harnesses}. Sessions can run on neither here, so \ diff --git a/tests/ci.rs b/tests/ci.rs index a7dac6145..abf3e2bbf 100644 --- a/tests/ci.rs +++ b/tests/ci.rs @@ -110,7 +110,7 @@ fn red_ci_is_handed_to_a_ci_fix_repair_whose_fix_turns_it_green() { \n\ If a failure is not caused by this branch (it is flaky, or also fails on main), do not change code for it. Instead, add it to a \"CI notes\" section of the pull request body with a one-line explanation.\n\ \n\ - You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.\n" + You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on.\n" ); assert_eq!(calls[1]["cwd"], calls[0]["cwd"]); // thirdshift pushes the fix even though the Repair didn't. @@ -700,7 +700,7 @@ fn with_mixed_failures_the_repair_fixes_only_the_branchs_own_and_the_run_then_fa \n\ If a failure is not caused by this branch (it is flaky, or also fails on main), do not change code for it. Instead, add it to a \"CI notes\" section of the pull request body with a one-line explanation.\n\ \n\ - You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.\n" + You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on.\n" ); let base_commit = scenario.origin_git(&["rev-parse", "main"]); let cause = inherited_failure("test", &base_commit); diff --git a/tests/codex_sessions.rs b/tests/codex_sessions.rs index 646c04797..16f5e5ee9 100644 --- a/tests/codex_sessions.rs +++ b/tests/codex_sessions.rs @@ -87,7 +87,8 @@ fn codex_args(model: &str, effort: &str) -> Vec<String> { /// and `effort`, the `CLAUDE.md` fallback and stdin set to null, its prompt /// last, and found every Factory skill in `.agents/skills/` with none of /// them in `git status`; that its prompt's first line loads its skill with -/// Codex's sigil, if it loads one; and that none ran on `claude`. +/// Codex's sigil, if it loads one, and it names no tool only Claude has; and +/// that none ran on `claude`. fn assert_sessions_on_codex(scenario: &Scenario, model: &str, effort: &str) { let calls = scenario.codex_calls(); assert!(!calls.is_empty(), "no Codex session ran"); @@ -103,6 +104,12 @@ fn assert_sessions_on_codex(scenario: &Scenario, model: &str, effort: &str) { assert_eq!(call["stdin_null"], true, "{args:?}"); assert_eq!(call["git_status"], "", "{args:?}"); assert!(!prompt.starts_with("/thirdshift-"), "{prompt}"); + assert!(!prompt.contains("Bash"), "{prompt}"); + assert_eq!( + prompt.matches("`TaskStop`").count(), + prompt.matches("`TaskStop` tool, if you have it").count(), + "{prompt}" + ); } scenario.assert_every_codex_session_found_the_factory_skills(); } diff --git a/tests/fresh_run.rs b/tests/fresh_run.rs index e33a6dcbe..8cd811176 100644 --- a/tests/fresh_run.rs +++ b/tests/fresh_run.rs @@ -91,7 +91,7 @@ fn gives_the_agent_the_fresh_prompt() { Push branch issue-7 and create a pull request against main using the `thirdshift-pr` skill, marked ready for review.\n\ In the PR body, add an \"Unaddressed findings\" section listing each skipped finding under Standards or Spec, with at least a one-line reason.\n\ Include \"Closes #7\" in the PR body.\n\ - You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.\n" + You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on.\n" ); } diff --git a/tests/mergeable.rs b/tests/mergeable.rs index 2e1c5453f..5950dfddd 100644 --- a/tests/mergeable.rs +++ b/tests/mergeable.rs @@ -114,7 +114,7 @@ fn hands_a_conflicting_merge_to_a_conflict_repair_and_pushes_its_resolution() { \n\ Resolve the conflicts, finish the merge, and push issue-7. Do not rebase or force-push.\n\ \n\ - You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the Bash timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need with the `TaskStop` tool, by its task id: a task still running when your turn ends is taken as work you were waiting on.\n" + You run headless: nobody is watching, and ending your turn ends the session. Run tests and other long commands in the foreground, raising the command's timeout if needed. If a command is moved to the background, wait for that task by its own task id or output file, never by process names or patterns (`pgrep`, `ps | grep`, and the like): other sessions on this machine run the same commands. Never end your turn while a background task you depend on is still running: ending the turn kills it. Before ending your turn, stop every background task you no longer need, by its task id (with the `TaskStop` tool, if you have it): a task still running when your turn ends is taken as work you were waiting on.\n" ); assert_eq!(repair["merging"], true); assert_eq!(repair["branch"], "issue-7"); diff --git a/tests/setup.rs b/tests/setup.rs index 4a2681041..932742e38 100644 --- a/tests/setup.rs +++ b/tests/setup.rs @@ -1679,6 +1679,34 @@ fn on_a_terminal_with_only_codex_installed_codex_is_the_default_answer() { assert_eq!(config["harness"]["codex"]["effort"].as_str(), Some("")); } +#[test] +fn on_a_terminal_rerunning_setup_defaults_to_the_harness_already_configured() { + let scenario = Scenario::new(); + scenario.git_email_is(None); + scenario.user_config_is( + "[harness]\ndefault = \"codex\"\n\n[harness.codex]\nmodel = \"gpt-6.1-sol\"\n", + ); + let path = fakes_only_path(&scenario); + let mut keystrokes = vec![(HARNESS, ""), (CODEX_MODEL, ""), (CODEX_EFFORT, "")]; + keystrokes.extend(NOTHING_ELSE); + + let result = setup_on_terminal(&scenario, &[("PATH", &path)], &keystrokes); + + assert!( + result + .stderr + .contains("Harness for every Run's sessions, claude or codex [codex]: "), + "terminal: {}", + result.stderr + ); + let config = table(&result); + assert_eq!(config["harness"]["default"].as_str(), Some("codex")); + assert_eq!( + config["harness"]["codex"]["model"].as_str(), + Some("gpt-6.1-sol") + ); +} + #[test] fn on_a_terminal_a_codex_display_name_and_capitalised_effort_are_written_as_codex_names_them() { let scenario = Scenario::new();