From b9ffed89212bc29826709167fcbf55b3a392b2e7 Mon Sep 17 00:00:00 2001 From: Cosmin Maria Date: Wed, 4 Nov 2026 14:16:33 +0200 Subject: [PATCH 1/5] feat(simulation): add scripted user turns (simulation.turns) Add `simulation.turns`: a fixed, ordered list of authored user turns that the dialog loop sends verbatim to the agent within ONE session, each exchange recorded as its own TurnRecord. No LLM plays the user. - `turns` is non-empty, replace-merged, and carries the first prompt too, so `initial_prompt` / `initial_prompt_file` are rejected beside it. `persona` and `goal` become optional (required unless `turns` is set); the simulator and criteria-timing fields must keep their defaults, compared by value so a recorded model_dump() still validates. `max_turns` must be >= len(turns). - `ScriptedUser` replays the turns through the surface the loop already uses on `UserSimulator`: no model call, no `claude` CLI, credentials or route, zero simulator tokens, `simulator_model: null`. `evaluate_stop` ends the dialog with the new `script_exhausted` reason after the last turn. - A scripted dialog never reads criteria to continue, so the loop checks them only on a graded run: `coder-eval execute` accepts it (NOT_GRADED) and `evaluate` grades it later. LLM-simulated dialogs are still refused. - A turn that fails after its retries, exhausts `run_limits.max_turns` or breaches a budget ends the script; later turns are not sent. - New task field `working_directory_prefix` (default true). `false` sends each prompt, single-shot or dialog turn, byte for byte. - `network: llm_only` no longer allowlists a simulator route for a script. Docs: Dialog Mode ("Scripted dialogs"), Task Definition Guide, User Guide, Docker Isolation, orchestration notes. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- .claude/notes/orchestration.md | 48 ++- CLAUDE.md | 6 +- docs/DIALOG_MODE.md | 82 +++++- docs/DOCKER_ISOLATION.md | 2 +- docs/TASK_DEFINITION_GUIDE.md | 59 +++- docs/USER_GUIDE.md | 2 +- src/coder_eval/cli/execute_command.py | 3 +- src/coder_eval/cli/run_command.py | 13 +- src/coder_eval/isolation/egress.py | 2 +- src/coder_eval/models/results.py | 3 +- src/coder_eval/models/tasks.py | 118 +++++++- src/coder_eval/orchestrator.py | 81 ++++-- src/coder_eval/simulation/__init__.py | 9 + src/coder_eval/simulation/scripted_user.py | 38 +++ src/coder_eval/simulation/termination.py | 11 +- tests/test_egress_targets.py | 8 +- tests/test_scripted_dialog.py | 324 +++++++++++++++++++++ tests/test_scripted_user.py | 44 +++ tests/test_simulation_config.py | 108 +++++++ tests/test_simulation_termination.py | 42 +++ tests/test_simulation_trials.py | 53 ++++ 21 files changed, 969 insertions(+), 87 deletions(-) create mode 100644 src/coder_eval/simulation/scripted_user.py create mode 100644 tests/test_scripted_dialog.py create mode 100644 tests/test_scripted_user.py diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 7a65e2241..98523ded9 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -50,9 +50,10 @@ path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports/junit.py` still emits `` for an ungraded row it encounters), `--allow-host-grading` (it decides how an ungraded row - is GRADED, and `execute` grades nothing), and simulation tasks (their - turn-continuation logic reads criteria results, so an ungraded dialog would silently - change its own stopping behavior). `stop_early:` blocks are inert under `execute` for + is GRADED, and `execute` grades nothing), and simulation tasks with an LLM-simulated + user (their turn-continuation logic reads criteria results, so an ungraded dialog would + silently change its own stopping behavior). A scripted dialog reads none, so `execute` + accepts it. `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. @@ -95,11 +96,13 @@ the loop over the real execution facts. 2. **The single-shot loop** stops after capturing the trajectory. Returning False keeps `FinalStatus` off SUCCESS and the status chain turns it into NOT_GRADED. The reference-integrity check is skipped too — it protects a grade that is not happening. -3. **The dialog loop** RAISES, and is unreachable today because `execute` rejects - simulation tasks at the CLI: the dialog reads criteria results to decide whether to - keep talking, so an ungraded dialog would silently change its own stopping behavior. - An empty-list "defensive no-op" here would be worse than a refusal — both callers go - straight on to the gate, which treats an empty criteria list as a vacuous pass. +3. **The dialog loop** checks criteria only on a graded run. Its end-of-dialog and + budget-gate checks are skipped when ungraded, so a scripted dialog finalizes + NOT_GRADED. The per-turn check still RAISES, and is unreachable today because + `execute` rejects LLM-simulated tasks at the CLI: that dialog reads criteria results to + decide whether to keep talking, so an ungraded run would silently change its own + stopping behavior. An empty-list "defensive no-op" there would be worse than a + refusal — the gate treats an empty criteria list as a vacuous pass. 4. **The diagnostics path** records nothing: a `not_evaluated` vector would imply criteria we were supposed to run and could not. @@ -632,10 +635,37 @@ unchanged. § Gate selection is fired-only — so `result.early_stop` is never assigned here and an armed simulation task gates strict-AND on a possibly truncated trajectory. -`execute` rejects simulation tasks at the CLI, because the loop reads criteria results to +`execute` rejects LLM-simulated tasks at the CLI, because the loop reads criteria results to decide whether to keep talking: an ungraded dialog would silently change its own stopping behaviour rather than merely skipping the score. +### Scripted dialogs + +`simulation.turns` swaps the LLM user for `ScriptedUser`, which answers the loop's own +`next_user_message` call. `_evaluation_loop` composes it and hands it to the loop as +`user=`; the loop never branches on the kind afterwards, so the stop, telemetry and logging +paths are shared. + +- **The cursor is the loop's history.** `ScriptedUser` returns `turns[len(dialog_pairs)]` + and keeps no counter, so it cannot drift from the exchanges the loop actually completed. + The dialog ends in `evaluate_stop` (`script_exhausted`) before the user is asked past the + last turn. +- **`turns[0]` is the pinned opener.** `_evaluation_loop` hands it to the loop as + `initial_prompt`, so turn 1 is recorded like any pinned opener (no simulator timing, an + accurate `conversation.log` label) and the task carries one source of truth for its + first prompt: `initial_prompt` beside `turns` is rejected. +- **The fields a script ignores are compared by value, not by `model_fields_set`.** + `evaluate` rebuilds the task from a full `model_dump()`, which marks every field as set, + so a "was it set" test would reject every recorded scripted task. +- **`max_turns >= len(turns)` is validated, not derived.** Deriving the cap from the list + would let a variant that swaps the script inherit the task's stale cap, and truncating a + script silently would score a different conversation than the one authored. +- **No criteria steer a script.** `check_criteria` and `stop_on_criteria_pass` stay at their + defaults, so the one criteria check is the end-of-dialog verdict. That is also why + `execute` can accept it: nothing the dialog does depends on a verdict it lacks. +- **No simulator, no simulator egress.** `isolation/egress.py` adds the simulator's model + route only for a simulated user. + ## Embedded commands A recorded config is UNTRUSTED INPUT. `evaluate ` rebuilds the task from the diff --git a/CLAUDE.md b/CLAUDE.md index 977c1772e..6a57fbfd4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,8 +107,8 @@ Each entry is a pointer. Full rationale: `.claude/notes/` (index: `.claude/notes [Sandbox Configuration](docs/TASK_DEFINITION_GUIDE.md#sandbox-configuration). - **Container isolation**: `driver: docker` runs one container per task — see [Docker Isolation](docs/DOCKER_ISOLATION.md). -- **Dialog mode**: `simulation/` drives a multi-turn LLM user — see - [Dialog Mode](docs/DIALOG_MODE.md). +- **Dialog mode**: `simulation/` drives a multi-turn dialog. The user is an LLM or an + authored script (`simulation.turns`) — see [Dialog Mode](docs/DIALOG_MODE.md). ## Success Criteria @@ -275,7 +275,7 @@ Tasks are YAML. Full reference: [Task Definition Guide](docs/TASK_DEFINITION_GUI | Task YAML, criteria, run limits, sandbox | [Task Definition Guide](docs/TASK_DEFINITION_GUIDE.md) | | Variants across the same tasks (A/B) | [A/B Experiments](docs/AB_EXPERIMENTS.md) | | Dataset fan-out and suite thresholds | [Bring Your Own Dataset](docs/DATASETS.md) | -| Multi-turn user simulation | [Dialog Mode](docs/DIALOG_MODE.md) | +| Multi-turn user simulation, scripted turns | [Dialog Mode](docs/DIALOG_MODE.md) | | `driver: docker`, custom images | [Docker Isolation](docs/DOCKER_ISOLATION.md) | | Run/report JSON consumed downstream | [Report Schema](docs/REPORT_SCHEMA.md) | diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index a5cf05ad8..10e8efff5 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -13,7 +13,8 @@ back requirements until asked. Dialog mode replaces the single-shot loop with a conversation between the coding agent and a **simulated user** — a second LLM driven by a persona and a goal. It is the only way to measure the -behaviors that only exist in conversation. +behaviors that only exist in conversation. To replay a fixed list of authored questions instead of +simulating a user, see [Scripted dialogs](#scripted-dialogs). ## When to use it @@ -71,7 +72,8 @@ The mechanics: **Agent-kind constraint.** The *subject* agent can be any registered kind — the dialog driver only calls the agent's `communicate()`, so Codex and plugin agents work. The *simulator*, however, is always a Claude Code agent. A dialog run therefore needs the `claude` CLI on `PATH` and working -Anthropic or Bedrock credentials **even when the agent under test is not Claude**. +Anthropic or Bedrock credentials **even when the agent under test is not Claude**. A +[scripted dialog](#scripted-dialogs) has no simulator and needs neither. **Early stop is not available here.** [Criterion-level `stop_early:` arming](TASK_DEFINITION_GUIDE.md#stop_early-opt-in-early-stop) is single-shot-only; an armed criterion alongside `simulation.enabled` is a hard error at resolution @@ -82,6 +84,61 @@ Every field's default and constraint lives in one place — the [Task Definition Guide's simulation section](TASK_DEFINITION_GUIDE.md#simulation-multi-turn-user-dialog). This page deliberately does not restate them. +## Scripted dialogs + +Sometimes the conversation itself is the benchmark: a fixed list of authored questions, asked in +order within one session, whose answers you read afterwards. A simulated user would paraphrase them +and add an LLM call per turn. Set `turns` instead of `persona` and `goal`, and the dialog replays the +list verbatim: + +```yaml +task_id: onboarding-questions +description: Ask the same three questions in one session and keep each answer. +agent: + type: codex # Any agent: a script needs no `claude` CLI or credentials of its own. +working_directory_prefix: false # Send each turn exactly as written. +simulation: + enabled: true + turns: + - "What does the knowledge base say about onboarding?" + - "And about reimbursements?" + - "Summarize both answers in two sentences." + max_turns: 3 # Must be at least len(turns). +success_criteria: + - type: llm_judge + description: "Each answer is grounded in the knowledge base" + prompt: "Read the DIALOG block and grade whether every answer is grounded in the knowledge base." + include_dialog: true +``` + +What changes against a simulated dialog: + +- **One session, no simulator.** Every turn goes to the same agent instance, so the agent keeps its + context from turn to turn. No LLM plays the user: no `claude` CLI, no credentials, no simulator + route, and `simulation.simulator_model` is `null` with zero simulator tokens. +- **`turns[0]` is the first prompt.** The task sets no `initial_prompt` or `initial_prompt_file`, so + there is one source of truth. Turns are sent byte for byte when `working_directory_prefix: false`; + by default each still carries the `Your working directory is: …` line in front. +- **The script ends the dialog.** After the last turn the dialog stops with + `stop_reason: script_exhausted`, whatever the criteria say. Nothing in a script reads a criteria + result, so `stop_on_criteria_pass` and `check_criteria: every_turn` / `both` make no sense for it, + and neither do `persona`, `goal`, `constraints`, `model` or `stop_token`: each must keep its + default, or the task is rejected at load time. `max_turns` must be at least `len(turns)`; a longer + script is rejected, never truncated. +- **A failed turn ends the dialog.** A turn that still fails after its retries, exhausts + `run_limits.max_turns` or breaches a `run_limits` budget stops the script. Later turns are not sent, + and the turns that ran stay on the record. +- **It runs without grading.** `coder-eval execute` accepts a scripted dialog and finalizes it + `NOT_GRADED`; `coder-eval evaluate ` grades it later. Criteria are checked once, after the + last turn. + +Each exchange is one entry in `iterations` of `task.json` (see [TurnRecord](REPORT_SCHEMA.md#turnrecord)): +that turn's `user_input` (the prompt as the agent received it), `agent_output`, `commands`, +`token_usage` and timings, with the authored turn as `messages[0]`. `agent_output` is the agent's own +rendering of its reply: Claude Code's is a tagged transcript, and its SDK result text is also in +`result_summary.result`. `n_trials: N` repeats the whole script N times, each in its own session, to +measure variance. + ## Designing the persona and goal `persona` is *who is talking*; `goal` is *what they want*. Keep them separate — a persona that @@ -121,15 +178,17 @@ After each exchange the driver evaluates the stop conditions **in this order**, 2. **`stop_on_criteria_pass`** (`criteria_passed`) — every success criterion passes. Requires per-turn checking (`check_criteria: every_turn` or `both`); pairing it with the default `end_of_dialog` is rejected at load time, since there would be nothing to check against. -3. **`max_turns`** (`max_turns`) — the hard cap on exchanges. -4. **`max_total_tokens`** (`budget`) — the dialog-wide budget across simulator **and** agent. The +3. **Script exhausted** (`script_exhausted`) — a [scripted dialog](#scripted-dialogs) has had its + last turn answered. It takes the place of `max_turns` there. +4. **`max_turns`** (`max_turns`) — the hard cap on exchanges. +5. **`max_total_tokens`** (`budget`) — the dialog-wide budget across simulator **and** agent. The dialog ends and the task is **still scored** — unlike [`run_limits.max_total_tokens`](TASK_DEFINITION_GUIDE.md#run-limits), which covers the subject agent only and aborts. -5. **`run_limits.max_turns`** (`agent_max_turns`): the agent used up its own model-call cap inside +6. **`run_limits.max_turns`** (`agent_max_turns`): the agent used up its own model-call cap inside one exchange. That cap restarts on every exchange, so this reason means one exchange ran long, not that the dialog ran out of exchanges. -6. **`stop_token`** (`stop_token`) — only if none of the above fired is the simulator asked for +7. **`stop_token`** (`stop_token`) — only if none of the above fired is the simulator asked for another message; the sentinel token in *that fresh utterance* ends the dialog. This is the workhorse in practice — the simulator decides, in character, that it got what it wanted — but it is evaluated **last**, so a turn that trips `max_turns` or the budget never gets the chance to @@ -192,8 +251,9 @@ with them. That guard is what keeps a chatty simulator from poisoning the grade. Budget roughly `max_turns × n_trials` agent turns per (task, variant) — the worst case, since the dialog usually stops on the stop token first. On top of that: -- **Simulator tokens**, one generation per turn. Small next to the agent's, but not free, and they - are reported separately as `simulation.simulator_input_tokens` / `simulator_output_tokens`. +- **Simulator tokens**, one generation per turn (none in a scripted dialog). Small next to the + agent's, but not free, and they are reported separately as `simulation.simulator_input_tokens` / + `simulator_output_tokens`. - **Criterion cost**, multiplied by turn count under `check_criteria: every_turn` or `both`. Cap it with [`run_limits`](TASK_DEFINITION_GUIDE.md#run-limits) — but note the accounting boundary: @@ -217,9 +277,9 @@ variants: persona: "A terse, impatient user who gives one-line answers." ``` -Productive axes: persona (terse vs. chatty, technical vs. not), how much the goal withholds, and -`n_trials` as a cost/confidence trade. See [A/B Experiments](AB_EXPERIMENTS.md) for the full -experiment layer. +A variant's `turns` replaces the task's script as a whole. Productive axes: persona (terse vs. +chatty, technical vs. not), how much the goal withholds, and `n_trials` as a cost/confidence trade. +See [A/B Experiments](AB_EXPERIMENTS.md) for the full experiment layer. ## Security posture diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index 985d62560..1bfdc63a6 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -108,7 +108,7 @@ call a model, and adds nothing for the parts that do not: | Source | Hosts added | |---|---| -| API backend (`API_BACKEND`), only when a `claude-code` agent, an enabled `simulation:`, or an `llm_judge` / `agent_judge` criterion uses it | `direct`: `api.anthropic.com:443`, `platform.claude.com:443` (Claude Code OAuth refresh). `bedrock`: `bedrock-runtime..amazonaws.com:443`, `bedrock..amazonaws.com:443`. `litellm`: the host of the forwarded `LITELLM_BASE_URL` (a `localhost` value becomes `host.docker.internal`) | +| API backend (`API_BACKEND`), only when a `claude-code` agent, an enabled `simulation:` with a simulated user (not scripted `turns`), or an `llm_judge` / `agent_judge` criterion uses it | `direct`: `api.anthropic.com:443`, `platform.claude.com:443` (Claude Code OAuth refresh). `bedrock`: `bedrock-runtime..amazonaws.com:443`, `bedrock..amazonaws.com:443`. `litellm`: the host of the forwarded `LITELLM_BASE_URL` (a `localhost` value becomes `host.docker.internal`) | | Agent | codex: the host of the forwarded `CODEX_BASE_URL`, else `api.openai.com:443`. antigravity: `generativelanguage.googleapis.com:443`. pi: the host of the `provider/` prefix of `agent.model` (`openrouter`, `anthropic`, `openai`, `google`; anything else or no model gives `openrouter.ai:443`). claude-code, opencode, delegate, none: nothing beyond the backend row | | `system_one_judge` criteria | the host of each `base_url` (default `api.typesafe.ai:443`) | | `egress_allowlist` | your entries | diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 78a0a1257..9be26c793 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -14,6 +14,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [Task YAML Structure](#task-yaml-structure) - [dataset](#dataset) - [skip](#skip) + - [working_directory_prefix](#working_directory_prefix) - [Agent Configuration](#agent-configuration) - [Run Limits](#run-limits) - [Sandbox Configuration](#sandbox-configuration) @@ -41,6 +42,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [Pre-Run Commands](#pre-run-commands) - [Post-Run Commands](#post-run-commands) - [Simulation (Multi-Turn User Dialog)](#simulation-multi-turn-user-dialog) + - [Scripted dialogs](#scripted-dialogs) - [Command Telemetry](#command-telemetry) - [Complete Example](#complete-example) @@ -52,6 +54,7 @@ Every task is a YAML file with this top-level structure: task_id: "my_task" # Unique identifier (required) description: "What this task tests" # Human-readable description (required) initial_prompt: "Instructions..." # Prompt sent to the agent (required) +working_directory_prefix: true # Optional: false sends prompts byte for byte (see below) tags: [smoke, golden, pure-python] # Optional tags for filtering (kebab-case) skip: false # Optional: quarantine this task (see below) @@ -118,6 +121,14 @@ Pair it with a comment naming the blocker — a ticket link, an upstream issue knows what has to change before it can come back. Run quarantined tasks on demand with `coder-eval run --include-skipped`; CI leaves the flag off, so they stay excluded there. +### `working_directory_prefix` + +By default the runner puts `Your working directory is: ` and a blank line in front of +every prompt it sends to the agent: `initial_prompt` in a single-shot task, and each user turn in a +[simulated or scripted dialog](#simulation-multi-turn-user-dialog). `working_directory_prefix: false` +sends the prompt exactly as authored, byte for byte. Use it when the prompt text is itself under test, +as with the fixed questions of a [scripted dialog](#scripted-dialogs). The default is `true`. + ## Tags The `tags` list categorizes tasks for selective execution. Each tag is lowercase kebab-case and may @@ -1703,7 +1714,7 @@ The shipped `experiments/*.yaml` use this to drop sandbox-scoped npm dirs (intro ## Simulation (Multi-Turn User Dialog) -Optional `simulation` block. When present and enabled, the orchestrator replaces the single-shot iteration loop with a multi-turn dialog between the coding agent and a simulated user (a second LLM with a persona and goal). Use this for tasks where the real usage pattern is conversational — clarifying questions, incremental requirements, mid-task corrections — rather than a single fire-and-forget prompt. +Optional `simulation` block. When present and enabled, the orchestrator replaces the single-shot iteration loop with a multi-turn dialog between the coding agent and a simulated user (a second LLM with a persona and goal), or with a [scripted user](#scripted-dialogs) that replays authored turns. Use this for tasks where the real usage pattern is conversational — clarifying questions, incremental requirements, mid-task corrections — rather than a single fire-and-forget prompt. > This section is the field reference. For when to use dialog mode, how to design a persona and > goal, trials and variance, grading, and cost, see **[Dialog Mode](DIALOG_MODE.md)**. @@ -1712,7 +1723,7 @@ Optional `simulation` block. When present and enabled, the orchestrator replaces simulation: enabled: true # Master switch; when false, simulation is skipped entirely. - # Persona and goal (required). + # Persona and goal (required for a simulated user; a scripted one sets `turns` instead). persona: | A non-technical business analyst who knows the outcome they want but not how automation works. Mildly impatient. @@ -1745,10 +1756,11 @@ simulation: | Field | Default | Description | |-------|---------|-------------| | `enabled` | `false` | When false, simulation is skipped; task runs in single-shot mode. | -| `persona` | *required* | Who the simulator is roleplaying. | -| `goal` | *required* | What the simulated user wants. | +| `turns` | *unset* | Scripted user turns, sent verbatim and in order in one session. Replaces the simulator — see [Scripted dialogs](#scripted-dialogs). | +| `persona` | *required* | Who the simulator is roleplaying. Required unless `turns` is set. | +| `goal` | *required* | What the simulated user wants. Required unless `turns` is set. | | `constraints` | `[]` | Behavioral rules the simulator must follow. | -| `max_turns` | `8` | Hard cap on user↔agent exchanges (1–100). | +| `max_turns` | `8` | Hard cap on user↔agent exchanges (1–100). At least `len(turns)` when `turns` is set. | | `stop_token` | `"<<>>"` | Sentinel the simulator emits to end the dialog. | | `stop_on_criteria_pass` | `false` | End when all criteria pass (requires per-turn checking). | | `max_total_tokens` | *unset* | Optional dialog-wide token budget (simulator **plus** agent). Distinct from [`run_limits.max_total_tokens`](#run-limits) — see below. | @@ -1760,16 +1772,17 @@ The simulator runs as a tools-disabled Claude Code agent on its own resolved `Ap **Semantics:** -- The task's `initial_prompt` is the user's *opening* message; the simulator picks up from turn 2. +- The task's `initial_prompt` is the user's *opening* message; the simulator picks up from turn 2. (A scripted dialog has no `initial_prompt`: `turns[0]` opens it.) - `max_turns` caps exchanges. Each exchange also gets a fresh `run_limits.max_turns` of model API calls, so the worst case per trial is the product of the two. Use `n_trials` for variance sampling. - The `reference` solution, if present, is hidden from the simulator (same security posture as for the coding agent). - When `n_trials > 1`, each trial becomes its own `ResolvedTask` with its own zero-padded replicate directory (`runs/////`) and its own `task.json` — the same fan-out mechanism as experiment `repeats`, which `n_trials` takes precedence over when simulation is enabled. Trial-level metadata appears under `simulation.replicate_index` / `simulation.n_trials` on the `EvaluationResult`. **Termination precedence** (evaluated after each exchange, first match wins): `run_limits` breach → -`stop_on_criteria_pass` → `max_turns` → `max_total_tokens` → `stop_token`. The stop token is checked -**last**, on the simulator's next utterance, which is only solicited if nothing above fired — so a -turn that hits `max_turns` or the token budget ends the dialog before the simulator can emit it. -A raising simulator call ends the dialog with `stop_reason: error`. +`stop_on_criteria_pass` → script exhausted (`turns` only) → `max_turns` → `max_total_tokens` → +`stop_token`. The stop token is checked **last**, on the simulator's next utterance, which is only +solicited if nothing above fired — so a turn that hits `max_turns` or the token budget ends the +dialog before the simulator can emit it. A raising simulator call ends the dialog with +`stop_reason: error`. > **`simulation.max_total_tokens` vs. `run_limits.max_total_tokens`.** This one covers the **whole > dialog** (simulator + agent) and ends the conversation gracefully with `stop_reason='budget'`, so @@ -1781,6 +1794,32 @@ A raising simulator call ends the dialog with `stop_reason: error`. **Experiment variants** can override any simulation field (persona, goal, constraints, n_trials, etc.) by setting a partial `simulation:` block on a variant — it is shallow-merged onto the task's simulation block. Useful experiment axes: simulator persona (terse vs. chatty), goal withholding, and n_trials for budget/quality tradeoffs. +### Scripted dialogs + +Set `turns` instead of `persona` and `goal` to replay a fixed list of authored user turns, in order, +within ONE agent session. No LLM plays the user, so the dialog needs no `claude` CLI, no credentials +and no simulator route, and the user side costs nothing. + +```yaml +simulation: + enabled: true + turns: + - "What does the knowledge base say about onboarding?" + - "And about reimbursements?" + max_turns: 8 # Must be at least len(turns). +working_directory_prefix: false # Send each turn byte for byte. +``` + +- `turns[0]` is the first prompt, so the task sets no `initial_prompt` / `initial_prompt_file`. Each turn is sent as written (whitespace and newlines included) and must not be blank. +- `persona`, `goal`, `constraints`, `model`, `stop_token`, `stop_on_criteria_pass` and `check_criteria` have nothing to do in a script, so each must keep its default; a different value is rejected at load time. `max_turns` must be at least `len(turns)`: a longer script is rejected, never truncated. +- The dialog ends after the last turn with `stop_reason: script_exhausted`, whatever the criteria say. Criteria are checked once, after the last turn. `simulator_model` is `null` and the simulator token counts are `0`. +- A turn that still fails after its retries (a crash or a timeout), exhausts `run_limits.max_turns` (`agent_max_turns`) or breaches a `run_limits` budget ends the dialog: later turns are not sent, and what ran is recorded. +- `coder-eval execute` accepts a scripted dialog and finalizes it `NOT_GRADED`; `coder-eval evaluate` grades it afterwards from the recorded run. +- `max_total_tokens` and `n_trials` work as for a simulated dialog. `run_limits.task_timeout` bounds the whole dialog; `run_limits.turn_timeout` bounds each turn. +- `${row.}` placeholders of a [`dataset`](#dataset) are not substituted into `turns`. + +Worked example, and where to read each turn's answer: [Dialog Mode → Scripted dialogs](DIALOG_MODE.md#scripted-dialogs). + ## Command Telemetry The framework automatically tracks all agent commands. No configuration needed. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index b27377f69..59c6dcaae 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -96,7 +96,7 @@ degraded: | --- | --- | | `--junit-xml` | A JUnit report reports verdicts, and there are none. | | `--allow-host-grading` | It decides how an executed-but-ungraded row is *graded*, and `execute` grades nothing. | -| Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | +| Simulation tasks with an LLM-simulated user | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. A [scripted dialog](DIALOG_MODE.md#scripted-dialogs) (`simulation.turns`) reads none and is accepted. | `stop_early:` blocks are also inert here: early stop exists to cut a run once the criteria decide the outcome, and under `execute` the full trajectory is the deliverable. diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index da750bbc1..98b97f788 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -239,7 +239,8 @@ def execute_command( Not supported here: --junit-xml (no verdicts to report), --allow-host-grading (nothing is graded, so there is no host-grading decision to make — pass it to `coder-eval run --resume` or `coder-eval evaluate` instead), and simulation - tasks (their turn-continuation logic reads criteria results). + tasks with an LLM-simulated user (their turn-continuation logic reads criteria + results). A scripted dialog (`simulation.turns`) is supported. Examples: diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index a8ceb6457..92f4e47e1 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -977,25 +977,28 @@ async def _apply_resume( def _reject_simulation_under_execute(resolved: list[ResolvedTask], *, grade: bool) -> None: - """Refuse a simulation task under ``execute`` rather than degrading it. + """Refuse a simulated-user task under ``execute`` rather than degrading it. The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would quietly change its own stopping behaviour and produce a trajectory that is not the one `run` would have produced. A + scripted dialog (``simulation.turns``) never reads them and is accepted. A resolution-time config error (exit 2), sitting with the other resolution-time validations, and it names the offending tasks. """ if grade: return simulated = sorted( - rt.task.task_id for rt in resolved if rt.task.simulation is not None and rt.task.simulation.enabled + rt.task.task_id + for rt in resolved + if rt.task.simulation is not None and rt.task.simulation.enabled and not rt.task.simulation.is_scripted ) if simulated: raise typer.BadParameter( - "`coder-eval execute` does not support simulation tasks (their turn-continuation " - + "logic depends on criteria results): " + "`coder-eval execute` does not support simulation tasks with an LLM-simulated user (their " + + "turn-continuation logic depends on criteria results): " + ", ".join(simulated) - + ". Use `coder-eval run` for these." + + ". Use `coder-eval run` for these, or script the user with `simulation.turns`." ) diff --git a/src/coder_eval/isolation/egress.py b/src/coder_eval/isolation/egress.py index f32a3f622..32a79af4d 100644 --- a/src/coder_eval/isolation/egress.py +++ b/src/coder_eval/isolation/egress.py @@ -127,7 +127,7 @@ def _model_routes(task: TaskDefinition, settings: Settings) -> list[ApiRoute]: routes: list[ApiRoute] = [] if task.agent is not None and task.agent.uses_api_backend: routes.append(agent_route) - if task.simulation is not None and task.simulation.enabled: + if task.simulation is not None and task.simulation.enabled and not task.simulation.is_scripted: routes.append(resolve_evaluation_route(settings, agent_route)) if any(isinstance(c, LLMJudgeCriterion | AgentJudgeCriterion) and c.enabled for c in task.success_criteria): api_route = task.checker_context.api_route if task.checker_context else None diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 802c00f70..f82588d95 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -432,6 +432,7 @@ class SimulationTelemetry(BaseModel): stop_reason: Literal[ "criteria_passed", "stop_token", + "script_exhausted", "max_turns", "agent_max_turns", "budget", @@ -447,7 +448,7 @@ class SimulationTelemetry(BaseModel): description=( "Resolved model that played the simulated user, captured so a persisted " "task.json is self-describing and its cost prices from a fact rather than " - "from the run's route." + "from the run's route. None when the user was scripted (``simulation.turns``)." ), ) diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 3fc3b8477..9120164f8 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -52,6 +52,21 @@ class UnknownTaskFieldWarning(DeprecationWarning): """When success criteria are evaluated inside a simulated dialog.""" +_FIXED_WHEN_SCRIPTED = ( + "persona", + "goal", + "constraints", + "model", + "stop_token", + "stop_on_criteria_pass", + "check_criteria", +) +"""``SimulationConfig`` fields that steer a simulated user or a criteria-driven stop. + +A scripted dialog uses none of them, so ``turns`` requires every one at its default. +""" + + NORMALIZED_CRITERION_ALIASES: dict[str, dict[str, Any]] = { "command_not_executed": {"type": "command_executed", "min_count": 0, "max_count": 0}, } @@ -139,6 +154,9 @@ class SimulationConfig(BaseModel): optional behavioral constraints; it generates each subsequent user prompt after the first (which is still the task's ``initial_prompt``). + ``turns`` replaces the simulator with an authored script: the user is either + scripted (``turns``) or simulated (``persona`` + ``goal``), never both. + Semantics relative to the existing task fields: - ``max_turns`` (below) bounds the intra-dialog agent<->user exchanges. @@ -151,6 +169,19 @@ class SimulationConfig(BaseModel): enabled: bool = Field(default=False, description="Master switch — when false, simulation is skipped entirely.") + turns: list[str] | None = MergeField( + strategy="replace", # a later layer's script wholesale-replaces an earlier one's + default=None, + min_length=1, + description=( + "Scripted user turns, sent verbatim and in order within ONE agent session. turns[0] is the " + "first prompt, so the task sets no initial_prompt. No simulator runs: persona, goal, " + "constraints, model, stop_token, stop_on_criteria_pass and check_criteria must keep their " + "defaults, and max_turns must be at least len(turns). The dialog ends after the last turn " + "with stop_reason='script_exhausted'." + ), + ) + # The simulator shares the coding agent's ApiRoute, so temperature/max_tokens # resolve at the route level. The MODEL is pinned below rather than inherited. model: str = Field( @@ -164,17 +195,20 @@ class SimulationConfig(BaseModel): ), ) - # Persona / goal. - persona: str = Field( + # Persona / goal: required for a simulated user, rejected for a scripted one. + persona: str | None = Field( + default=None, description=( "Who the simulator is roleplaying — background, tone, level of domain knowledge. " - "Passed verbatim into the simulator system prompt." + "Passed verbatim into the simulator system prompt. Required unless ``turns`` is set." ), ) - goal: str = Field( + goal: str | None = Field( + default=None, description=( "What the simulated user wants to achieve. This drives the conversation. " - "You can optionally withhold information (see ``constraints``) to force clarification behavior." + "You can optionally withhold information (see ``constraints``) to force clarification behavior. " + "Required unless ``turns`` is set." ), ) constraints: list[str] = MergeField( @@ -231,13 +265,55 @@ class SimulationConfig(BaseModel): ), ) + @property + def is_scripted(self) -> bool: + """True when the user side is the authored ``turns`` script, not an LLM simulator.""" + return self.turns is not None + @field_validator("persona", "goal") @classmethod - def _non_blank(cls, v: str) -> str: - if not v or not v.strip(): + def _non_blank(cls, v: str | None) -> str | None: + if v is not None and not v.strip(): raise ValueError("must be a non-empty string") return v + @field_validator("turns") + @classmethod + def _non_blank_turns(cls, v: list[str] | None) -> list[str] | None: + if v is not None and any(not turn.strip() for turn in v): + raise ValueError("every turn must be a non-empty string") + return v + + @model_validator(mode="after") + def _validate_user_source(self) -> Self: + """The user is scripted (``turns``) or simulated (``persona`` + ``goal``), never both. + + A scripted dialog never reads a simulator knob or a criteria result to decide + what to say or when to stop, so those fields must keep their defaults rather + than silently do nothing. Compared by value, not by ``model_fields_set``: a + recorded ``model_dump()`` marks every field as set. + """ + if self.turns is None: + if self.persona is None or self.goal is None: + raise ValueError( + "simulation needs `turns` (a scripted user) or both `persona` and `goal` (a simulated user)" + ) + return self + fields = type(self).model_fields + changed = [ + name + for name in _FIXED_WHEN_SCRIPTED + if getattr(self, name) != fields[name].get_default(call_default_factory=True) + ] + if changed: + raise ValueError(f"simulation.turns scripts the user, so these must keep their defaults: {changed}") + if len(self.turns) > self.max_turns: + raise ValueError( + f"simulation.turns has {len(self.turns)} entries but max_turns is {self.max_turns} — " + + f"raise max_turns to at least {len(self.turns)}." + ) + return self + @model_validator(mode="after") def _validate_timing(self) -> Self: if self.stop_on_criteria_pass and self.check_criteria == "end_of_dialog": @@ -455,6 +531,14 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk "Mutually exclusive with initial_prompt." ), ) + working_directory_prefix: bool = Field( + default=True, + description=( + "Prefix every prompt sent to the agent with 'Your working directory is: '. " + "Set false to send each prompt byte-for-byte as authored (initial_prompt, or every " + "simulation.turns entry)." + ), + ) tags: list[str] = MergeField( strategy="replace", # not layer-merged today; replace = the engine default if it ever is default_factory=list, @@ -579,8 +663,8 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk default=None, description=( "Optional multi-turn user-simulation config. When present and enabled, the orchestrator runs " - "the task as a dialog between the coding agent and a simulated user LLM instead of " - "the default single-shot iteration loop." + "the task as a dialog between the coding agent and a simulated user LLM, or an authored " + "script (simulation.turns), instead of the default single-shot iteration loop." ), ) @@ -652,6 +736,22 @@ def check_prompt_fields(self) -> Self: ) return self + @model_validator(mode="after") + def check_scripted_turns_replace_the_prompt(self) -> Self: + """Refuse a prompt field beside ``simulation.turns``: turns[0] is the first prompt.""" + simulation = self.simulation + if ( + simulation is not None + and simulation.enabled + and simulation.is_scripted + and (self.initial_prompt is not None or self.initial_prompt_file is not None) + ): + raise ValueError( + "simulation.turns supplies every user turn, the first included: remove " + + "'initial_prompt' / 'initial_prompt_file' (or drop simulation.turns)." + ) + return self + @model_validator(mode="after") def check_none_agent(self) -> Self: """Validate the no-op agent (``agent: {type: none}``) contract. diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index db264a7ad..99584f3cf 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -78,7 +78,7 @@ ) from .result_metrics import turn_time_buckets, visible_turn_count from .sandbox import Sandbox -from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop +from .simulation import DialogStopReason, DialogUser, ScriptedUser, SimulatorResult, UserSimulator, evaluate_stop from .simulation.utterance import extract_utterance as _extract_utterance from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit from .streaming.events import CriteriaCheckEvent, CriterionSummary @@ -2103,6 +2103,12 @@ def _select_gate(self) -> bool: logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") return self.result.all_criteria_passed(self.task.success_criteria) + def _agent_prompt(self, prompt: str, sandbox_dir: Path) -> str: + """The text the agent receives for ``prompt``: behind the working-directory prefix unless the task opts out.""" + if not self.task.working_directory_prefix: + return prompt + return f"Your working directory is: {sandbox_dir.resolve()}\n\n{prompt}" + async def _evaluation_loop(self) -> bool: """Run the main evaluation loop. @@ -2150,17 +2156,23 @@ async def _evaluation_loop(self) -> bool: self.result.success_criteria_results = criteria_results return self._select_gate() - # Working directory context prepended to every prompt (including feedback). + # Working directory context prepended to every prompt (see `_agent_prompt`). # The agent resumes its session between iterations via session_id. assert self.sandbox is not None and self.sandbox.sandbox_dir is not None sandbox_dir = self.sandbox.sandbox_dir # A simulation block replaces the single-shot loop entirely; a simulated # task runs exactly one dialog per call. - if self.task.simulation is not None and self.task.simulation.enabled: - # initial_prompt is optional in simulation mode — when unset, the - # simulator produces the opening utterance itself. - return await self._simulation_dialog_loop(self.task.initial_prompt, sandbox_dir) + simulation = self.task.simulation + if simulation is not None and simulation.enabled: + if simulation.turns is None: + # initial_prompt is optional in simulation mode — when unset, the + # simulator produces the opening utterance itself. + return await self._simulation_dialog_loop(self.task.initial_prompt, sandbox_dir) + # A scripted dialog opens with its first turn; ScriptedUser replays the rest. + return await self._simulation_dialog_loop( + simulation.turns[0], sandbox_dir, user=ScriptedUser(simulation.turns) + ) # Guaranteed for real agents by check_prompt_fields; a no-op task runs with # no prompt, which NoOpAgent ignores, returning an empty turn. @@ -2170,13 +2182,13 @@ async def _evaluation_loop(self) -> bool: self.result.iteration_count = iteration # Communicate with agent (with retry logic) - prompt_with_cwd = f"Your working directory is: {sandbox_dir.resolve()}\n\n{current_prompt}" + agent_prompt = self._agent_prompt(current_prompt, sandbox_dir) logger.debug(f"Sending prompt: {current_prompt[:100]}...") # The agent owns its lifecycle events now (AgentStartEvent fires inside # communicate()); the orchestrator is a pure consumer of the stream. turn_record = await self._communicate_with_retry( - prompt=prompt_with_cwd, + prompt=agent_prompt, iteration=iteration, operation_label="Agent communication", ) @@ -2289,16 +2301,17 @@ async def _run_dialog_criteria_check( """ assert self.result is not None assert self.success_checker is not None - # Grading site 3 of 4. Unreachable today, and RAISES rather than returning - # an empty list: both callers go straight on to the gate, which treats an - # empty criteria list as a vacuous pass, so a silent no-op here would - # produce a criteria-free dialog scoring as though nothing was asked. + # Grading site 3 of 4. The loop skips its end-of-dialog and budget-gate checks + # when ungraded; the per-turn check is reachable ungraded only past the CLI + # refusal, and RAISES rather than returning an empty list: the gate treats an + # empty criteria list as a vacuous pass, so a silent no-op here would produce + # a criteria-free dialog scoring as though nothing was asked. # Rationale: .claude/notes/orchestration.md § The four grading sites if not self.grade: raise ValueError( "Grading is disabled but the simulation dialog path requires criteria results to " - + "decide turn continuation. `execute` refuses simulation tasks at the CLI; reaching " - + "here means that refusal was bypassed." + + "decide turn continuation. `execute` refuses LLM-simulated dialogs at the CLI; " + + "reaching here means that refusal was bypassed." ) await self._verify_reference_integrity() criteria_results = await self.success_checker.check_all_async( @@ -2338,7 +2351,7 @@ def _user_message_from_sim( async def _solicit_user_message( self, - simulator: UserSimulator, + simulator: DialogUser, history: list[tuple[str, str]], sim_model_id: str | None, failure_log_msg: str, @@ -2369,7 +2382,7 @@ async def _solicit_user_message( ) async def _acquire_opener( - self, simulator: UserSimulator, sim_config: SimulationConfig, sim_model_id: str | None + self, simulator: DialogUser, sim_config: SimulationConfig, sim_model_id: str | None ) -> _OpenerOutcome: """Pure-simulation opener (``initial_prompt is None``): ask the simulator to generate turn 1 from empty history. @@ -2424,12 +2437,15 @@ async def _acquire_opener( failures=0, ) - async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: Path) -> bool: # noqa: PLR0915 — sequential dialog driver; decomposed into helpers, residual length is irreducible without a _DialogState rewrite (see plan Phase 5). Statement count ratcheted by CE022. + async def _simulation_dialog_loop( # noqa: PLR0915 — sequential dialog driver; decomposed into helpers, residual length is irreducible without a _DialogState rewrite (see plan Phase 5). Statement count ratcheted by CE022. + self, initial_prompt: str | None, sandbox_dir: Path, *, user: DialogUser | None = None + ) -> bool: """Run the task as a multi-turn dialog driven by an LLM user simulator. Replaces the criteria-feedback iteration loop for tasks that define a ``simulation`` block. One invocation runs exactly ONE dialog trajectory; - parallel trials are expanded upstream. + parallel trials are expanded upstream. ``user`` replaces the LLM simulator + (a scripted dialog passes its ``ScriptedUser``). Lifecycle: obtain the opening utterance (pinned ``initial_prompt``, else ask the simulator), send it as turn 1, then after each agent reply @@ -2437,7 +2453,7 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: budget), and ask the simulator for the next message — breaking on a passing criteria check with ``stop_on_criteria_pass``, on the simulator's stop token, or on a simulator exception. A final criteria check runs afterwards - unless one just did. + unless one just did, and only when graded. Emits the same streaming events as the single-shot loop, so downstream renderers work unchanged. Simulator telemetry lands on @@ -2452,13 +2468,17 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: assert self.task.agent is not None sim_config = self.task.simulation - simulator = UserSimulator( - config=sim_config, - task_description=self.task.description, - initial_prompt=initial_prompt, - # Resolved independently of eval_route and checker_context: the - # simulator is a real CLI subprocess, not a checker concern. - route=self.simulator_route, + simulator = ( + user + if user is not None + else UserSimulator( + config=sim_config, + task_description=self.task.description, + initial_prompt=initial_prompt, + # Resolved independently of eval_route and checker_context: the + # simulator is a real CLI subprocess, not a checker concern. + route=self.simulator_route, + ) ) await simulator.start() @@ -2503,7 +2523,7 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: else: current_prompt = initial_prompt pending_user_turn = UserMessage(text=initial_prompt) - self._log_conversation("USER", 1, current_prompt, metadata="pinned initial_prompt") + self._log_conversation("USER", 1, current_prompt, metadata="pinned opener") # Clean (user, agent) pairs, deliberately without the # working-directory prefix the framework prepends to agent prompts. dialog_pairs: list[tuple[str, str]] = [] @@ -2523,11 +2543,10 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: self.result.iteration_count = turns_completed logger.info("Simulation turn %s/%s", turns_completed, sim_config.max_turns) - prompt_with_cwd = f"Your working directory is: {sandbox_dir.resolve()}\n\n{current_prompt}" # Agent emits its own AgentStartEvent inside communicate(). agent_turn_attempted = True turn_record = await self._communicate_with_retry( - prompt=prompt_with_cwd, + prompt=self._agent_prompt(current_prompt, sandbox_dir), iteration=turns_completed, operation_label=f"Agent communication (sim turn {turns_completed})", ) @@ -2574,7 +2593,7 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: self._check_run_limits(iteration=turns_completed) except BudgetExceededError: stop_reason = DialogStopReason.RUN_LIMIT_EXCEEDED - if not criteria_checked_this_turn: + if self.grade and not criteria_checked_this_turn: # Partial-credit side effects only; this fallback site # neither sets all_passed nor emits an event. await self._run_dialog_criteria_check(judge_usage_accum) @@ -2634,7 +2653,7 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: # Final criteria check for end_of_dialog (or both) modes, or when # every_turn mode did not run a check for the final turn. - end_of_dialog_needed = ( + end_of_dialog_needed = self.grade and ( sim_config.check_criteria in ("end_of_dialog", "both") or not criteria_checked_this_turn ) if end_of_dialog_needed: diff --git a/src/coder_eval/simulation/__init__.py b/src/coder_eval/simulation/__init__.py index d9d7d1943..0175c149d 100644 --- a/src/coder_eval/simulation/__init__.py +++ b/src/coder_eval/simulation/__init__.py @@ -9,18 +9,27 @@ Public API: - ``UserSimulator`` — wraps an LLM invoker (Anthropic direct or AWS Bedrock) and produces the next simulated-user utterance. + - ``ScriptedUser`` — replays the task's authored ``simulation.turns`` + instead; no model call. + - ``DialogUser`` — either of the two, as the dialog loop sees them. - ``SimulatorResult`` — simulator output + token accounting + stop flag. - ``DialogStopReason`` — enum for why a dialog ended. - ``evaluate_stop`` — pure predicate that decides whether a dialog should terminate after a given turn. """ +from coder_eval.simulation.scripted_user import ScriptedUser from coder_eval.simulation.termination import DialogStopReason, evaluate_stop from coder_eval.simulation.user_simulator import SimulatorResult, UserSimulator +type DialogUser = UserSimulator | ScriptedUser + + __all__ = [ "DialogStopReason", + "DialogUser", + "ScriptedUser", "SimulatorResult", "UserSimulator", "evaluate_stop", diff --git a/src/coder_eval/simulation/scripted_user.py b/src/coder_eval/simulation/scripted_user.py new file mode 100644 index 000000000..b4f049475 --- /dev/null +++ b/src/coder_eval/simulation/scripted_user.py @@ -0,0 +1,38 @@ +"""Scripted user for dialogs whose turns are authored, not simulated. + +Stands in for ``UserSimulator`` in the dialog loop with the same +``start`` / ``next_user_message`` / ``stop`` / ``model`` surface, and makes no +model call, so a scripted dialog needs no ``claude`` CLI, no credentials and +costs the user side nothing. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from coder_eval.simulation.user_simulator import SimulatorResult + + +class ScriptedUser: + """Replays ``SimulationConfig.turns`` in order, byte for byte. + + The next turn is the one after the exchanges already completed, so the dialog + loop's own history is the only cursor. Asking past the last turn raises + ``IndexError``; the loop stops first, on ``DialogStopReason.SCRIPT_EXHAUSTED``. + """ + + model: str | None = None + + def __init__(self, turns: Sequence[str]) -> None: + self._turns = tuple(turns) + + async def start(self) -> None: + """Nothing to start: there is no simulator agent.""" + + async def stop(self) -> None: + """Nothing to stop: there is no simulator agent.""" + + async def next_user_message(self, dialog_pairs: list[tuple[str, str]]) -> SimulatorResult: + """The scripted turn that follows ``dialog_pairs`` (one pair per completed exchange).""" + text = self._turns[len(dialog_pairs)] + return SimulatorResult(text=text, raw_text=text, stop_requested=False, input_tokens=None, output_tokens=None) diff --git a/src/coder_eval/simulation/termination.py b/src/coder_eval/simulation/termination.py index 4cc96f046..3478af040 100644 --- a/src/coder_eval/simulation/termination.py +++ b/src/coder_eval/simulation/termination.py @@ -13,6 +13,7 @@ class DialogStopReason(StrEnum): CRITERIA_PASSED = "criteria_passed" STOP_TOKEN = "stop_token" + SCRIPT_EXHAUSTED = "script_exhausted" MAX_TURNS = "max_turns" AGENT_MAX_TURNS = "agent_max_turns" BUDGET = "budget" @@ -43,9 +44,10 @@ def evaluate_stop( Precedence (first matching wins): 1. criteria passed AND ``stop_on_criteria_pass`` — CRITERIA_PASSED - 2. ``turns_completed >= max_turns`` — MAX_TURNS - 3. ``max_total_tokens`` exceeded — BUDGET - 4. otherwise — keep going + 2. every scripted turn (``turns``) answered — SCRIPT_EXHAUSTED + 3. ``turns_completed >= max_turns`` — MAX_TURNS + 4. ``max_total_tokens`` exceeded — BUDGET + 5. otherwise — keep going Stop-token detection is NOT handled here: the orchestrator checks ``SimulatorResult.stop_requested`` directly on the fresh simulator output @@ -67,6 +69,9 @@ def evaluate_stop( if config.stop_on_criteria_pass and criteria_all_passed: return StopDecision(stop=True, reason=DialogStopReason.CRITERIA_PASSED) + if config.turns is not None and turns_completed >= len(config.turns): + return StopDecision(stop=True, reason=DialogStopReason.SCRIPT_EXHAUSTED) + if turns_completed >= config.max_turns: return StopDecision(stop=True, reason=DialogStopReason.MAX_TURNS) diff --git a/tests/test_egress_targets.py b/tests/test_egress_targets.py index 7719b11e6..366268e63 100644 --- a/tests/test_egress_targets.py +++ b/tests/test_egress_targets.py @@ -56,6 +56,7 @@ def _targets( env: dict[str, str] | None = None, criteria: list[Any] | None = None, simulation: bool | None = None, + scripted: bool = False, route: ApiRouteContext | None = None, bedrock_token: str | None = "tok", **docker: Any, @@ -71,7 +72,11 @@ def _targets( if agent is None: task.agent = None if simulation is not None: - task.simulation = SimulationConfig(enabled=simulation, persona="p", goal="g") + task.simulation = ( + SimulationConfig(enabled=simulation, turns=["p"]) + if scripted + else SimulationConfig(enabled=simulation, persona="p", goal="g") + ) if route is not None: task.checker_context = CheckerContext(api_route=route) settings = Settings.model_construct( @@ -121,6 +126,7 @@ def _targets( ), "simulation-enabled": (CODEX, {"simulation": True}, sorted([*DIRECT, OPENAI])), "simulation-disabled": (CODEX, {"simulation": False}, [OPENAI]), + "simulation-scripted-calls-no-simulator-model": (CODEX, {"simulation": True, "scripted": True}, [OPENAI]), "litellm-simulator-pinned": (CODEX, {"backend": LITELLM, "simulation": True}, sorted([*DIRECT, OPENAI])), } diff --git a/tests/test_scripted_dialog.py b/tests/test_scripted_dialog.py new file mode 100644 index 000000000..8613bd1e4 --- /dev/null +++ b/tests/test_scripted_dialog.py @@ -0,0 +1,324 @@ +"""End-to-end tests for scripted dialogs: authored user turns, one agent session, no simulator. + +These run the real ``Orchestrator`` (and, for ``execute`` / ``evaluate``, the real CLI) +against a ``MockAgent`` that records the prompts it receives, so they cover the whole +dialog loop without any LLM, ``claude`` CLI or credentials. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app, run_helpers +from coder_eval.errors import TurnTimeoutError +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + RunLimits, + SandboxConfig, + SimulationConfig, + TaskDefinition, + TokenUsage, + TurnRecord, + parse_agent_config, + simulator_cost_usd, +) +from coder_eval.orchestrator import Orchestrator +from tests.fixtures.mock_agent import MockAgent + + +runner = CliRunner() + +TURNS = [ + "What does the knowledge base say about onboarding?", + " And about\nreimbursements? ", + "Thanks — one more: «ünïcode» ✓", +] + + +class _SessionAgent(MockAgent): + """A ``MockAgent`` that remembers every prompt, and can time out or run out of turns on cue.""" + + def __init__( + self, + task: TaskDefinition, + *, + time_out_on_call: int | None = None, + exhaust_on_call: int | None = None, + tokens_per_call: int = 0, + ) -> None: + super().__init__(task, scenario="success") + self.prompts: list[str] = [] + self._time_out_on_call = time_out_on_call + self._exhaust_on_call = exhaust_on_call + self._tokens_per_call = tokens_per_call + + async def communicate(self, user_input: str, **kwargs: Any) -> TurnRecord: + self.prompts.append(user_input) + call = len(self.prompts) + if call == self._time_out_on_call: + raise TurnTimeoutError(30.0, task_id=self.task.task_id, iteration=call) + turn = await super().communicate(user_input, **kwargs) + if self._tokens_per_call: + turn.token_usage = TokenUsage( + uncached_input_tokens=self._tokens_per_call, output_tokens=self._tokens_per_call + ) + turn.max_turns_exhausted = call == self._exhaust_on_call + return turn + + +def _install_agent(monkeypatch: pytest.MonkeyPatch, **agent_kwargs: Any) -> list[_SessionAgent]: + """Make every orchestrator build a ``_SessionAgent``; the returned list collects them.""" + created: list[_SessionAgent] = [] + + async def _create(self: Orchestrator) -> _SessionAgent: + created.append(_SessionAgent(self.task, **agent_kwargs)) + return created[-1] + + monkeypatch.setattr(Orchestrator, "_create_agent", _create) + return created + + +def _task(turns: list[str] = TURNS, *, verbatim: bool = True, **overrides: Any) -> TaskDefinition: + """A scripted task; ``verbatim=False`` leaves ``working_directory_prefix`` at its default.""" + kwargs: dict[str, Any] = { + "task_id": "scripted", + "description": "A scripted dialog", + "agent": parse_agent_config(type=AgentKind.CLAUDE_CODE), + "sandbox": SandboxConfig(driver="tempdir"), + "success_criteria": [FileExistsCriterion(path="test.txt", description="file must exist")], + } + if verbatim: + kwargs["working_directory_prefix"] = False + if "simulation" not in overrides: + kwargs["simulation"] = SimulationConfig(enabled=True, turns=turns, max_turns=len(turns)) + kwargs.update(overrides) + return TaskDefinition(**kwargs) + + +async def _run(tmp_path: Path, task: TaskDefinition, *, grade: bool = True) -> EvaluationResult: + return await Orchestrator(task=task, run_dir=tmp_path / "run", variant_id="default", grade=grade).run() + + +# -------------------------------------------------------------------------- +# What the agent receives +# -------------------------------------------------------------------------- + + +async def test_turns_reach_the_agent_verbatim_and_in_order_within_one_session(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + + result = await _run(tmp_path, _task()) + + assert len(created) == 1, "one agent instance is one session" + assert created[0].prompts == TURNS + assert [turn.user_input for turn in result.iterations] == TURNS + assert [turn.messages[0].text for turn in result.iterations] == TURNS + + +async def test_each_turn_keeps_the_working_directory_prefix_unless_the_task_opts_out(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + + await _run(tmp_path, _task(verbatim=False)) + + prefix = f"Your working directory is: {created[0].working_directory.resolve()}\n\n" + assert created[0].prompts == [prefix + turn for turn in TURNS] + + +async def test_a_single_shot_prompt_can_drop_the_working_directory_prefix(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + single_shot = _task(simulation=None, initial_prompt="Say exactly this.") + + await _run(tmp_path, single_shot) + + assert created[0].prompts == ["Say exactly this."] + + +async def test_a_single_shot_prompt_keeps_the_working_directory_prefix_by_default(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + single_shot = _task(simulation=None, initial_prompt="Say exactly this.", verbatim=False) + + await _run(tmp_path, single_shot) + + prefix = f"Your working directory is: {created[0].working_directory.resolve()}\n\n" + assert created[0].prompts == [prefix + "Say exactly this."] + + +# -------------------------------------------------------------------------- +# How the dialog ends, and what it records +# -------------------------------------------------------------------------- + + +async def test_a_finished_script_reports_script_exhausted_and_costs_the_user_side_nothing(tmp_path, monkeypatch): + _install_agent(monkeypatch) + + result = await _run(tmp_path, _task()) + + sim = result.simulation + assert sim is not None + assert sim.stop_reason == "script_exhausted" + assert sim.total_turns == len(TURNS) + assert (sim.simulator_input_tokens, sim.simulator_output_tokens, sim.simulator_failures) == (0, 0, 0) + assert sim.simulator_model is None + assert simulator_cost_usd(result) is None + assert result.iteration_count == len(TURNS) + assert len(result.iterations) == len(TURNS), "no trailing record for a user turn nobody answered" + + +async def test_the_script_runs_to_its_end_even_when_the_criteria_already_pass(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + checks: list[None] = [] + original = Orchestrator._run_dialog_criteria_check + + async def _counting(self: Orchestrator, accum: Any) -> Any: + checks.append(None) + return await original(self, accum) + + monkeypatch.setattr(Orchestrator, "_run_dialog_criteria_check", _counting) + + result = await _run(tmp_path, _task()) + + # The mock agent writes test.txt on its first reply, so the criteria pass from turn 1. + assert created[0].prompts == TURNS + assert result.final_status == FinalStatus.SUCCESS + assert len(checks) == 1, "criteria are read once, after the last turn, never to steer the dialog" + + +async def test_the_conversation_log_shows_the_script_as_authored_not_simulated(tmp_path, monkeypatch): + _install_agent(monkeypatch) + + await _run(tmp_path, _task()) + + log = (tmp_path / "run" / "conversation.log").read_text(encoding="utf-8") + user_headers = re.findall(r"^=== USER \(turn (\d+)\)(?: — (.*))? ===$", log, flags=re.MULTILINE) + assert user_headers == [("1", "pinned opener"), ("2", ""), ("3", "")] + + +async def test_no_simulator_is_built_for_a_script(tmp_path, monkeypatch): + _install_agent(monkeypatch) + + def _forbidden(*args: Any, **kwargs: Any) -> None: + raise AssertionError("a scripted dialog must not build an LLM simulator") + + monkeypatch.setattr("coder_eval.orchestrator.UserSimulator", _forbidden) + + result = await _run(tmp_path, _task()) + + assert result.simulation is not None + assert result.simulation.stop_reason == "script_exhausted" + + +# -------------------------------------------------------------------------- +# Without grading (`execute`) +# -------------------------------------------------------------------------- + + +async def test_a_script_runs_in_full_ungraded_and_finalizes_not_graded(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + + result = await _run(tmp_path, _task(), grade=False) + + assert created[0].prompts == TURNS + assert result.final_status == FinalStatus.NOT_GRADED + assert result.weighted_score is None + assert result.success_criteria_results == [] + assert result.simulation is not None + assert result.simulation.stop_reason == "script_exhausted" + + +async def test_a_budget_breach_ends_an_ungraded_script_with_its_own_status(tmp_path, monkeypatch): + """The partial-credit check a breach normally runs must not fire (and raise) when nothing is graded.""" + created = _install_agent(monkeypatch, tokens_per_call=100) + task = _task(run_limits=RunLimits(max_total_tokens=150)) + + result = await _run(tmp_path, task, grade=False) + + assert created[0].prompts == TURNS[:1] + assert result.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED + assert result.simulation is not None + assert result.simulation.stop_reason == "run_limit_exceeded" + + +_SCRIPTED_YAML = """task_id: scripted-cli +description: A scripted dialog run through the CLI +agent: + type: claude-code +working_directory_prefix: false +simulation: + enabled: true + turns: + - First question. + - Second question. + max_turns: 2 +success_criteria: + - type: file_exists + path: test.txt + description: file must exist +""" + + +def _single_task_dir(run_dir: Path) -> Path: + (task_json,) = run_dir.glob("**/task.json") + return task_json.parent + + +def test_execute_accepts_a_script_and_evaluate_grades_it_afterwards(tmp_path, monkeypatch): + monkeypatch.setattr(run_helpers.settings, "runs_dir", tmp_path / "default-runs") + created = _install_agent(monkeypatch) + task_file = tmp_path / "task.yaml" + task_file.write_text(_SCRIPTED_YAML, encoding="utf-8") + run_dir = tmp_path / "run" + + executed = runner.invoke(app, ["execute", str(task_file), "--run-dir", str(run_dir)]) + + assert executed.exit_code == 0, executed.output + assert created[0].prompts == ["First question.", "Second question."] + task_dir = _single_task_dir(run_dir) + row = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + assert row["final_status"] == "NOT_GRADED" + assert row["simulation"]["stop_reason"] == "script_exhausted" + + graded = runner.invoke(app, ["evaluate", str(task_dir)]) + + assert graded.exit_code == 0, graded.output + row = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + assert row["final_status"] == "SUCCESS" + assert row["simulation"]["stop_reason"] == "script_exhausted" + assert len(row["iterations"]) == 2 + + +# -------------------------------------------------------------------------- +# A turn that fails +# -------------------------------------------------------------------------- + + +async def test_a_failed_turn_ends_the_script_and_later_turns_are_never_sent(tmp_path, monkeypatch): + created = _install_agent(monkeypatch, time_out_on_call=2) + + result = await _run(tmp_path, _task()) + + assert created[0].prompts == TURNS[:2], "turn 2 was sent and failed; turn 3 never was" + assert result.final_status == FinalStatus.ERROR + assert "timed out" in (result.error_message or "") + assert result.simulation is not None + assert result.simulation.stop_reason == "error" + assert TURNS[1] in [turn.user_input for turn in result.iterations], "the turn that failed stays on record" + + +async def test_an_agent_that_runs_out_of_its_own_turns_ends_the_script(tmp_path, monkeypatch): + created = _install_agent(monkeypatch, exhaust_on_call=1) + + result = await _run(tmp_path, _task()) + + assert created[0].prompts == TURNS[:1] + assert result.max_turns_exhausted is True + assert result.simulation is not None + assert result.simulation.stop_reason == "agent_max_turns" diff --git a/tests/test_scripted_user.py b/tests/test_scripted_user.py new file mode 100644 index 000000000..759305579 --- /dev/null +++ b/tests/test_scripted_user.py @@ -0,0 +1,44 @@ +"""Unit tests for ``ScriptedUser``: authored turns replayed without a model.""" + +from __future__ import annotations + +import pytest + +from coder_eval.simulation import ScriptedUser + + +TURNS = ["First question.", " Second, with spaces.\n", "Third question."] + + +async def test_serves_the_turn_after_the_completed_exchanges(): + user = ScriptedUser(TURNS) + + assert (await user.next_user_message([])).text == "First question." + assert (await user.next_user_message([("First question.", "answer")])).text == " Second, with spaces.\n" + history = [("First question.", "answer"), (" Second, with spaces.\n", "answer")] + assert (await user.next_user_message(history)).text == "Third question." + + +async def test_the_turn_is_an_ordinary_utterance_that_cost_nothing(): + result = await ScriptedUser(TURNS).next_user_message([]) + + assert result.raw_text == result.text + assert result.stop_requested is False + assert result.input_tokens is None + assert result.output_tokens is None + + +async def test_asking_past_the_last_turn_raises(): + history = [(turn, "answer") for turn in TURNS] + + with pytest.raises(IndexError): + await ScriptedUser(TURNS).next_user_message(history) + + +async def test_needs_no_simulator_agent_to_start_or_stop(): + user = ScriptedUser(TURNS) + + await user.start() + await user.stop() + + assert user.model is None diff --git a/tests/test_simulation_config.py b/tests/test_simulation_config.py index 7d8c9d16a..d04e8dd79 100644 --- a/tests/test_simulation_config.py +++ b/tests/test_simulation_config.py @@ -156,3 +156,111 @@ def test_initial_prompt_still_required_when_simulation_disabled(self): def test_initial_prompt_still_required_when_no_simulation_block(self): with pytest.raises(ValidationError, match="initial_prompt"): self._task(initial_prompt=None) + + +def _scripted_kwargs(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = {"enabled": True, "turns": ["First question.", "Second question."]} + base.update(overrides) + return base + + +class TestScriptedTurns: + """``turns`` scripts the user, so the simulator's knobs have nothing to do and must not be set.""" + + def test_persona_and_goal_are_not_needed_to_script_the_user(self): + cfg = SimulationConfig(**_scripted_kwargs()) + assert cfg.is_scripted is True + assert cfg.turns == ["First question.", "Second question."] + assert cfg.persona is None + assert cfg.goal is None + + def test_a_simulated_user_is_not_scripted(self): + assert SimulationConfig(**_minimal_sim_kwargs()).is_scripted is False + + def test_a_simulated_user_still_needs_persona_and_goal(self): + with pytest.raises(ValidationError, match=r"persona.*goal"): + SimulationConfig(enabled=True) + with pytest.raises(ValidationError, match=r"persona.*goal"): + SimulationConfig(enabled=True, persona="A user") + + def test_turns_are_kept_verbatim(self): + turns = [" leading and trailing space\n", "Line one\n\nLine two"] + assert SimulationConfig(**_scripted_kwargs(turns=turns)).turns == turns + + def test_an_empty_script_is_rejected(self): + with pytest.raises(ValidationError, match="at least 1"): + SimulationConfig(**_scripted_kwargs(turns=[])) + + def test_a_blank_turn_is_rejected(self): + with pytest.raises(ValidationError, match="non-empty"): + SimulationConfig(**_scripted_kwargs(turns=["Fine.", " "])) + + @pytest.mark.parametrize( + ("field", "value"), + [ + ("persona", "A business analyst."), + ("goal", "Build an invoice pipeline."), + ("constraints", ["Do not paste code."]), + ("model", "anthropic.claude-haiku-4-5"), + ("stop_token", "<<>>"), + ("stop_on_criteria_pass", True), + ("check_criteria", "every_turn"), + ], + ) + def test_a_simulator_or_criteria_knob_beside_turns_is_rejected(self, field: str, value: Any): + with pytest.raises(ValidationError, match=field): + SimulationConfig(**_scripted_kwargs(**{field: value})) + + def test_a_recorded_config_validates_again(self): + """``evaluate`` rebuilds the task from a full ``model_dump``, defaults included.""" + recorded = SimulationConfig(**_scripted_kwargs()).model_dump() + assert SimulationConfig(**recorded).turns == ["First question.", "Second question."] + + def test_more_turns_than_max_turns_is_rejected_rather_than_truncated(self): + turns = [f"Question {i}." for i in range(9)] + with pytest.raises(ValidationError, match="max_turns"): + SimulationConfig(**_scripted_kwargs(turns=turns)) + assert SimulationConfig(**_scripted_kwargs(turns=turns, max_turns=9)).max_turns == 9 + + def test_token_budget_and_trials_still_apply_to_a_script(self): + cfg = SimulationConfig(**_scripted_kwargs(max_total_tokens=50_000, n_trials=3)) + assert (cfg.max_total_tokens, cfg.n_trials) == (50_000, 3) + + +class TestScriptedTurnsOnTaskDefinition: + def _task(self, **overrides: Any) -> TaskDefinition: + kwargs: dict[str, Any] = { + "task_id": "scripted-task", + "description": "A scripted task", + "agent": parse_agent_config(type=AgentKind.CLAUDE_CODE), + "sandbox": SandboxConfig(driver="tempdir"), + "success_criteria": [FileExistsCriterion(path="out.txt", description="file exists")], + } + if "simulation" not in overrides: + kwargs["simulation"] = SimulationConfig(**_scripted_kwargs()) + kwargs.update(overrides) + return TaskDefinition(**kwargs) + + def test_no_initial_prompt_is_needed(self): + task = self._task() + assert task.initial_prompt is None + + def test_initial_prompt_beside_turns_is_rejected(self): + simulation = SimulationConfig(**_scripted_kwargs()) + with pytest.raises(ValidationError, match="turns"): + self._task(simulation=simulation, initial_prompt="Hello.") + + def test_initial_prompt_file_beside_turns_is_rejected(self): + simulation = SimulationConfig(**_scripted_kwargs()) + with pytest.raises(ValidationError, match="turns"): + self._task(simulation=simulation, initial_prompt_file="prompt.md") + + def test_a_disabled_script_leaves_the_prompt_contract_alone(self): + simulation = SimulationConfig(**_scripted_kwargs(enabled=False)) + assert self._task(initial_prompt="Hello.", simulation=simulation).initial_prompt == "Hello." + + def test_the_working_directory_prefix_is_on_by_default(self): + assert self._task().working_directory_prefix is True + + def test_the_working_directory_prefix_can_be_switched_off(self): + assert self._task(working_directory_prefix=False).working_directory_prefix is False diff --git a/tests/test_simulation_termination.py b/tests/test_simulation_termination.py index e6f0e2a00..5cd1b0ba7 100644 --- a/tests/test_simulation_termination.py +++ b/tests/test_simulation_termination.py @@ -96,6 +96,48 @@ def test_precedence_max_turns_over_budget(self): assert d.reason == DialogStopReason.MAX_TURNS +def _scripted_cfg(**overrides): + base = {"enabled": True, "turns": ["one", "two", "three"], "max_turns": 3} + base.update(overrides) + return SimulationConfig(**base) + + +class TestEvaluateStopScripted: + """A script ends when its last turn is answered, never on a criteria result.""" + + def test_keeps_going_while_turns_remain(self): + d = evaluate_stop(config=_scripted_cfg(), turns_completed=2, total_tokens_used=0, criteria_all_passed=True) + assert d.stop is False + + def test_script_exhausted_after_the_last_turn(self): + d = evaluate_stop(config=_scripted_cfg(), turns_completed=3, total_tokens_used=0, criteria_all_passed=False) + assert d.stop is True + assert d.reason == DialogStopReason.SCRIPT_EXHAUSTED + + def test_script_exhausted_is_reported_instead_of_max_turns(self): + """The script fills ``max_turns`` exactly here; the more specific reason wins.""" + d = evaluate_stop(config=_scripted_cfg(), turns_completed=3, total_tokens_used=0, criteria_all_passed=False) + assert d.reason == DialogStopReason.SCRIPT_EXHAUSTED + + def test_script_exhausted_is_reported_instead_of_budget(self): + d = evaluate_stop( + config=_scripted_cfg(max_total_tokens=100), + turns_completed=3, + total_tokens_used=200, + criteria_all_passed=False, + ) + assert d.reason == DialogStopReason.SCRIPT_EXHAUSTED + + def test_budget_still_cuts_a_script_short(self): + d = evaluate_stop( + config=_scripted_cfg(max_total_tokens=100), + turns_completed=1, + total_tokens_used=200, + criteria_all_passed=False, + ) + assert d.reason == DialogStopReason.BUDGET + + class TestStripStopToken: def test_removes_token(self): assert strip_stop_token("Thanks. <<>>", "<<>>") == "Thanks." diff --git a/tests/test_simulation_trials.py b/tests/test_simulation_trials.py index 82dd254ac..1f1cfbf21 100644 --- a/tests/test_simulation_trials.py +++ b/tests/test_simulation_trials.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Any +import pytest + from coder_eval.models import ( ExperimentDefaults, ExperimentDefinition, @@ -80,6 +82,22 @@ def test_variant_overrides_persona_only(self): assert resolved.simulation.goal == "task goal" assert resolved.simulation.stop_token == "<<>>" + def test_variant_turns_replace_the_task_script_wholesale(self): + """A script is one ordered whole: a variant swaps it, it never splices into it.""" + task_kwargs = _base_task_kwargs() + del task_kwargs["initial_prompt"] + task = TaskDefinition( + **task_kwargs, + simulation=SimulationConfig(enabled=True, turns=["task one", "task two", "task three"], max_turns=3), + ) + exp = ExperimentDefinition( + experiment_id="test", + variants=[ExperimentVariant(variant_id="short", simulation={"turns": ["variant one"]})], + ) + resolved, *_ = resolve_task_for_variant(self._default_exp(), task, exp, exp.variants[0]) + assert resolved.simulation is not None + assert resolved.simulation.turns == ["variant one"] + def test_no_simulation_anywhere_stays_none(self): """No layer provides simulation — resolved stays None (single-shot).""" task = TaskDefinition(**_base_task_kwargs()) @@ -107,6 +125,41 @@ def test_variant_can_enable_simulation_on_plain_task(self): assert resolved.simulation.enabled is True +class TestScriptedTaskUnderAVariantPrompt: + def test_variant_initial_prompt_cannot_override_a_script(self, tmp_path: Path): + """The script already holds the first prompt; a second one would be silently dropped.""" + task_yaml = tmp_path / "task.yaml" + task_yaml.write_text( + """ +task_id: scripted-task +description: Scripted test +agent: + type: claude-code +sandbox: + driver: tempdir +success_criteria: + - type: file_exists + path: out.txt + description: file +simulation: + enabled: true + turns: [one, two] +""".strip() + ) + default_exp = ExperimentDefinition( + experiment_id="default", + defaults=ExperimentDefaults(agent={"type": "claude-code", "permission_mode": "acceptEdits"}), + variants=[ExperimentVariant(variant_id="default")], + ) + experiment = ExperimentDefinition( + experiment_id="test", + variants=[ExperimentVariant(variant_id="rewrite", initial_prompt="A different opener.")], + ) + + with pytest.raises(ValueError, match=r"simulation\.turns"): + resolve_all_tasks([task_yaml], experiment, default_exp, BatchRunConfig(run_dir=tmp_path / "runs")) + + class TestResolveAllTasksWithTrials: def test_end_to_end_expansion(self, tmp_path: Path): """Verify resolve_all_tasks expands trials across the full pipeline.""" From 1565ea39e1e3d4ba4e1fabb4952b6deafff35fbc Mon Sep 17 00:00:00 2001 From: Cosmin Maria Date: Wed, 4 Nov 2026 15:10:54 +0200 Subject: [PATCH 2/5] fix(simulation): export DialogUser as a plain alias for CodeQL CodeQL's Python extractor does not treat a PEP 695 `type` statement as a definition, so it reported `DialogUser` in `__all__` as undefined (code scanning alert 203). A plain assignment is the same alias to pyright and is not flagged; `typing.TypeAlias` would trip ruff's UP040. Co-Authored-By: Claude Opus 5.5 --- src/coder_eval/simulation/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coder_eval/simulation/__init__.py b/src/coder_eval/simulation/__init__.py index 0175c149d..fee227bef 100644 --- a/src/coder_eval/simulation/__init__.py +++ b/src/coder_eval/simulation/__init__.py @@ -23,7 +23,8 @@ from coder_eval.simulation.user_simulator import SimulatorResult, UserSimulator -type DialogUser = UserSimulator | ScriptedUser +# A plain alias: CodeQL does not read a `type` statement as defining its `__all__` name. +DialogUser = UserSimulator | ScriptedUser __all__ = [ From ffa1713f0f8540075ee95739c4dfc3db57071571 Mon Sep 17 00:00:00 2001 From: Cosmin Maria Date: Wed, 4 Nov 2026 15:10:54 +0200 Subject: [PATCH 3/5] fix(deps): bump pyasn1 to 0.6.5 for four High advisories osv-scanner fails the Quality Gate on pyasn1 0.6.4 (GHSA-2qm8-x737-gmw5, GHSA-9m75-vc3g-4x39, GHSA-fm9c-5rcf-2q5p, GHSA-mm57-4v3p-rchx), on main as well. 0.6.5 fixes all four. Lockfile only. Co-Authored-By: Claude Opus 5.5 --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 3dfa40491..4912a67a4 100644 --- a/uv.lock +++ b/uv.lock @@ -2252,11 +2252,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]] From 31f4c633ac28ee423a378b95c1d249705d3a1a4b Mon Sep 17 00:00:00 2001 From: Cosmin Maria Date: Thu, 5 Nov 2026 18:32:35 +0200 Subject: [PATCH 4/5] fix(simulation): retain exhaustion facts and expand scripted dataset turns --- docs/DATASETS.md | 7 ++-- src/coder_eval/models/tasks.py | 11 +++--- src/coder_eval/orchestration/task_loader.py | 6 ++- src/coder_eval/orchestrator.py | 4 +- tests/test_dataset_expansion.py | 42 ++++++++++++++++++++ tests/test_scripted_dialog.py | 44 ++++++++++++++++++--- 6 files changed, 99 insertions(+), 15 deletions(-) diff --git a/docs/DATASETS.md b/docs/DATASETS.md index 498addab4..0e6896d6f 100644 --- a/docs/DATASETS.md +++ b/docs/DATASETS.md @@ -71,10 +71,11 @@ Each line is one JSON object — the same shape as an inline row: ## Substituting row values -`${row.}` placeholders are replaced with that row's value in exactly two places: +`${row.}` placeholders are replaced with that row's value in: -1. the task's `initial_prompt`, and -2. every **string leaf** of every `success_criteria` entry (at any nesting depth). +1. the task's `initial_prompt`, +2. each scripted `simulation.turns` entry, and +3. every **string leaf** of every `success_criteria` entry (at any nesting depth). Nothing else is substituted — not the task-level `description`, not `pre_run`, not `sandbox`. Keep row-varying data in the prompt and the criteria. (A *criterion's* own `description` is a string leaf diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 9120164f8..6c145b8c3 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -397,10 +397,11 @@ class Dataset(BaseModel): stable row identifier and becomes a suffix on the task_id ("/"). - Row values are substituted into the task's ``initial_prompt`` and into - string fields of each ``success_criteria`` entry using ``${row.}`` - syntax. Substitution happens in ``task_loader.expand_dataset`` before - variant resolution, so variants cannot override the dataset. + Row values are substituted into the task's ``initial_prompt``, each + ``simulation.turns`` entry, and string fields of each ``success_criteria`` + entry using ``${row.}`` syntax. Substitution happens in + ``task_loader.expand_dataset`` before variant resolution, so variants + cannot override the dataset. """ model_config = ConfigDict(extra="forbid") @@ -644,7 +645,7 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk default=None, description=( "Optional dataset to fan out this task into one sub-task per row. " - "Row values substitute into initial_prompt and success_criteria via ${row.}. " + "Row values substitute into initial_prompt, simulation.turns, and success_criteria via ${row.}. " "Expansion happens before variant resolution, so variants cannot override the dataset." ), ) diff --git a/src/coder_eval/orchestration/task_loader.py b/src/coder_eval/orchestration/task_loader.py index 02e6b5d1b..fb482b2e2 100644 --- a/src/coder_eval/orchestration/task_loader.py +++ b/src/coder_eval/orchestration/task_loader.py @@ -372,7 +372,8 @@ def expand_dataset( Each expanded task has its task_id rewritten to ``"/"``, its ``dataset`` cleared (preventing re-expansion downstream), and ``${row.}`` substituted in - ``initial_prompt`` and every string leaf of ``success_criteria``. + ``initial_prompt``, each ``simulation.turns`` entry, and every string leaf + of ``success_criteria``. Row ids are validated against a safe pattern, since they become directory names under the run dir. @@ -436,6 +437,9 @@ def expand_dataset( data = task.model_dump(exclude_unset=True) if isinstance(data.get("initial_prompt"), str): data["initial_prompt"] = _substitute_row_in_str(data["initial_prompt"], row) + simulation = data.get("simulation") + if isinstance(simulation, dict) and isinstance(simulation.get("turns"), list): + simulation["turns"] = _substitute_row_in_tree(simulation["turns"], row) if isinstance(data.get("success_criteria"), list): data["success_criteria"] = [_substitute_row_in_tree(c, row) for c in data["success_criteria"]] data["suite_id"] = task.task_id diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 99584f3cf..efb945529 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2554,6 +2554,9 @@ async def _simulation_dialog_loop( # noqa: PLR0915 — sequential dialog driver turn_record.messages.insert(0, pending_user_turn) pending_user_turn = None self.result.iterations.append(turn_record) + # Retain the agent fact even when the script's stop wins below. + if turn_record.max_turns_exhausted: + self.result.max_turns_exhausted = True self._sync_sandbox_command_path_with_agent() dialog_pairs.append((current_prompt, _extract_utterance(turn_record.agent_output or ""))) agent_meta_parts = [] @@ -2616,7 +2619,6 @@ async def _simulation_dialog_loop( # noqa: PLR0915 — sequential dialog driver self._check_expected_turns(iteration=turns_completed) if turn_record.max_turns_exhausted: - self.result.max_turns_exhausted = True stop_reason = DialogStopReason.AGENT_MAX_TURNS logger.warning( "Agent exhausted its inner max_turns during simulation turn %s; ending dialog.", diff --git a/tests/test_dataset_expansion.py b/tests/test_dataset_expansion.py index 08693cfb5..9cbace7a3 100644 --- a/tests/test_dataset_expansion.py +++ b/tests/test_dataset_expansion.py @@ -52,6 +52,48 @@ def test_passthrough_when_no_dataset(self, tmp_path: Path) -> None: assert expanded[0] is task # same object, no copy +class TestExpandDatasetScripted: + @staticmethod + def _task(turns: list[str], rows: list[dict[str, Any]]) -> TaskDefinition: + data = _base_task_dict() + del data["initial_prompt"] + data["simulation"] = {"enabled": True, "turns": turns, "max_turns": len(turns)} + data["success_criteria"] = [] + data["dataset"] = {"rows": rows} + return TaskDefinition(**data) + + def test_scalar_values_preserve_whitespace_and_original_task(self, tmp_path: Path) -> None: + turns = [" ${row.text}\n", "${row.number}/${row.flag}/${row.optional}"] + task = self._task(turns, [{"id": "r1", "text": "ALPHA", "number": 3, "flag": True, "optional": None}]) + before = task.model_dump() + + expanded = expand_dataset(task, tmp_path) + + assert expanded[0].simulation.turns == [" ALPHA\n", "3/True/"] + assert task.model_dump() == before + + def test_unknown_field_in_later_turn_is_rejected(self, tmp_path: Path) -> None: + task = self._task(["First", "${row.missing}"], [{"id": "r1"}]) + with pytest.raises(KeyError, match=r"row\.missing"): + expand_dataset(task, tmp_path) + + @pytest.mark.parametrize("value", [{"nested": "value"}, ["nested"]]) + def test_non_scalar_turn_value_is_rejected(self, tmp_path: Path, value: Any) -> None: + task = self._task(["First", "${row.value}"], [{"id": "r1", "value": value}]) + with pytest.raises(TypeError, match="must be a scalar"): + expand_dataset(task, tmp_path) + + def test_empty_substitution_keeps_existing_turn_validation(self, tmp_path: Path) -> None: + task = self._task(["First", "${row.value}"], [{"id": "r1", "value": None}]) + with pytest.raises(ValueError, match="every turn must be a non-empty string"): + expand_dataset(task, tmp_path) + + def test_no_dataset_keeps_scripted_task_unchanged(self, tmp_path: Path) -> None: + task = self._task(["First", "Literal ${row.value}"], [{"id": "r1"}]) + task.dataset = None + assert expand_dataset(task, tmp_path)[0] is task + + class TestExpandDatasetInline: def test_expands_rows_with_substitution(self, tmp_path: Path) -> None: task = _make_task_with_dataset( diff --git a/tests/test_scripted_dialog.py b/tests/test_scripted_dialog.py index 8613bd1e4..99e3173ec 100644 --- a/tests/test_scripted_dialog.py +++ b/tests/test_scripted_dialog.py @@ -21,6 +21,7 @@ AgentKind, EvaluationResult, FileExistsCriterion, + FileMatchesRegexCriterion, FinalStatus, RunLimits, SandboxConfig, @@ -31,6 +32,7 @@ parse_agent_config, simulator_cost_usd, ) +from coder_eval.orchestration.task_loader import expand_dataset from coder_eval.orchestrator import Orchestrator from tests.fixtures.mock_agent import MockAgent @@ -124,6 +126,27 @@ async def test_turns_reach_the_agent_verbatim_and_in_order_within_one_session(tm assert [turn.messages[0].text for turn in result.iterations] == TURNS +async def test_dataset_turns_reach_each_row_agent_after_substitution(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + data = _task( + turns=["Question: ${row.question}", "Next: ${row.followup}"], + sandbox=SandboxConfig(driver="tempdir", python=None), + ).model_dump() + data["dataset"] = { + "rows": [ + {"id": "r1", "question": "ALPHA", "followup": "ONE"}, + {"id": "r2", "question": "BETA", "followup": "TWO"}, + ] + } + task = TaskDefinition.model_validate(data) + + for row in expand_dataset(task, tmp_path): + await Orchestrator(task=row, run_dir=tmp_path / row.row_id, variant_id="default").run() + + assert [agent.prompts for agent in created] == [["Question: ALPHA", "Next: ONE"], ["Question: BETA", "Next: TWO"]] + assert task.simulation.turns == ["Question: ${row.question}", "Next: ${row.followup}"] + + async def test_each_turn_keeps_the_working_directory_prefix_unless_the_task_opts_out(tmp_path, monkeypatch): created = _install_agent(monkeypatch) @@ -313,12 +336,23 @@ async def test_a_failed_turn_ends_the_script_and_later_turns_are_never_sent(tmp_ assert TURNS[1] in [turn.user_input for turn in result.iterations], "the turn that failed stays on record" -async def test_an_agent_that_runs_out_of_its_own_turns_ends_the_script(tmp_path, monkeypatch): - created = _install_agent(monkeypatch, exhaust_on_call=1) +@pytest.mark.parametrize("exhaust_on_call", [1, len(TURNS)]) +@pytest.mark.parametrize("grade", [True, False]) +async def test_an_agent_that_runs_out_of_its_own_turns_ends_the_script(tmp_path, monkeypatch, exhaust_on_call, grade): + created = _install_agent(monkeypatch, exhaust_on_call=exhaust_on_call) + task = _task( + sandbox=SandboxConfig(driver="tempdir", python=None), + success_criteria=[ + FileMatchesRegexCriterion(path="not-created.txt", pattern="required", description="Missing file") + ], + ) - result = await _run(tmp_path, _task()) + result = await _run(tmp_path, task, grade=grade) - assert created[0].prompts == TURNS[:1] + assert created[0].prompts == TURNS[:exhaust_on_call] + assert result.iterations[-1].max_turns_exhausted is True assert result.max_turns_exhausted is True + assert result.final_status == (FinalStatus.MAX_TURNS_EXHAUSTED if grade else FinalStatus.NOT_GRADED) assert result.simulation is not None - assert result.simulation.stop_reason == "agent_max_turns" + assert result.simulation.stop_reason == ("agent_max_turns" if exhaust_on_call < len(TURNS) else "script_exhausted") + assert json.loads((tmp_path / "run" / "task.json").read_text())["max_turns_exhausted"] is True From 31d030dd5da929da3573308cc919ae1969257811 Mon Sep 17 00:00:00 2001 From: Cosmin Maria Date: Thu, 5 Nov 2026 22:28:27 +0200 Subject: [PATCH 5/5] fix: reject scripted prompt overrides and derive dialog length --- .claude/notes/orchestration.md | 6 +-- docs/DIALOG_MODE.md | 5 ++- docs/TASK_DEFINITION_GUIDE.md | 7 ++- src/coder_eval/models/tasks.py | 11 ++--- src/coder_eval/orchestration/experiment.py | 17 ++++++++ tests/test_experiment_resolver.py | 28 ++++++++++++ tests/test_scripted_dialog.py | 50 ++++++++++++++++++++++ tests/test_simulation_config.py | 21 +++++++-- 8 files changed, 125 insertions(+), 20 deletions(-) diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 98523ded9..4d6d0260c 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -657,9 +657,9 @@ paths are shared. - **The fields a script ignores are compared by value, not by `model_fields_set`.** `evaluate` rebuilds the task from a full `model_dump()`, which marks every field as set, so a "was it set" test would reject every recorded scripted task. -- **`max_turns >= len(turns)` is validated, not derived.** Deriving the cap from the list - would let a variant that swaps the script inherit the task's stale cap, and truncating a - script silently would score a different conversation than the one authored. +- **Script length owns the dialog length.** Derive `max_turns` after simulation config + validation, including merged variant scripts, so a replacement cannot inherit a stale cap. + Prompt overrides are rejected before mutation because a script owns every user turn. - **No criteria steer a script.** `check_criteria` and `stop_on_criteria_pass` stay at their defaults, so the one criteria check is the end-of-dialog verdict. That is also why `execute` can accept it: nothing the dialog does depends on a verdict it lacks. diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index 10e8efff5..c9ab64084 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -123,8 +123,9 @@ What changes against a simulated dialog: `stop_reason: script_exhausted`, whatever the criteria say. Nothing in a script reads a criteria result, so `stop_on_criteria_pass` and `check_criteria: every_turn` / `both` make no sense for it, and neither do `persona`, `goal`, `constraints`, `model` or `stop_token`: each must keep its - default, or the task is rejected at load time. `max_turns` must be at least `len(turns)`; a longer - script is rejected, never truncated. + default, or the task is rejected at load time. `max_turns` is derived from the script length + (1–100 turns), so an authored script needs no second dialog-length setting. Variant prompt + overrides and mutations are rejected; replace `simulation.turns` to change the script. - **A failed turn ends the dialog.** A turn that still fails after its retries, exhausts `run_limits.max_turns` or breaches a `run_limits` budget stops the script. Later turns are not sent, and the turns that ran stay on the record. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 9be26c793..d27325224 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1760,7 +1760,7 @@ simulation: | `persona` | *required* | Who the simulator is roleplaying. Required unless `turns` is set. | | `goal` | *required* | What the simulated user wants. Required unless `turns` is set. | | `constraints` | `[]` | Behavioral rules the simulator must follow. | -| `max_turns` | `8` | Hard cap on user↔agent exchanges (1–100). At least `len(turns)` when `turns` is set. | +| `max_turns` | `8` | Hard cap on simulated user↔agent exchanges (1–100); derived from `len(turns)` for scripts. | | `stop_token` | `"<<>>"` | Sentinel the simulator emits to end the dialog. | | `stop_on_criteria_pass` | `false` | End when all criteria pass (requires per-turn checking). | | `max_total_tokens` | *unset* | Optional dialog-wide token budget (simulator **plus** agent). Distinct from [`run_limits.max_total_tokens`](#run-limits) — see below. | @@ -1806,17 +1806,16 @@ simulation: turns: - "What does the knowledge base say about onboarding?" - "And about reimbursements?" - max_turns: 8 # Must be at least len(turns). working_directory_prefix: false # Send each turn byte for byte. ``` - `turns[0]` is the first prompt, so the task sets no `initial_prompt` / `initial_prompt_file`. Each turn is sent as written (whitespace and newlines included) and must not be blank. -- `persona`, `goal`, `constraints`, `model`, `stop_token`, `stop_on_criteria_pass` and `check_criteria` have nothing to do in a script, so each must keep its default; a different value is rejected at load time. `max_turns` must be at least `len(turns)`: a longer script is rejected, never truncated. +- `persona`, `goal`, `constraints`, `model`, `stop_token`, `stop_on_criteria_pass` and `check_criteria` have nothing to do in a script, so each must keep its default; a different value is rejected at load time. `max_turns` is derived from the 1–100 authored turns, so no second length setting is needed. Variant prompt overrides and mutations are rejected; replace `simulation.turns` instead. - The dialog ends after the last turn with `stop_reason: script_exhausted`, whatever the criteria say. Criteria are checked once, after the last turn. `simulator_model` is `null` and the simulator token counts are `0`. - A turn that still fails after its retries (a crash or a timeout), exhausts `run_limits.max_turns` (`agent_max_turns`) or breaches a `run_limits` budget ends the dialog: later turns are not sent, and what ran is recorded. - `coder-eval execute` accepts a scripted dialog and finalizes it `NOT_GRADED`; `coder-eval evaluate` grades it afterwards from the recorded run. - `max_total_tokens` and `n_trials` work as for a simulated dialog. `run_limits.task_timeout` bounds the whole dialog; `run_limits.turn_timeout` bounds each turn. -- `${row.}` placeholders of a [`dataset`](#dataset) are not substituted into `turns`. +- `${row.}` placeholders of a [`dataset`](#dataset) are substituted into each scripted turn. Worked example, and where to read each turn's answer: [Dialog Mode → Scripted dialogs](DIALOG_MODE.md#scripted-dialogs). diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 6c145b8c3..df174567c 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -173,11 +173,12 @@ class SimulationConfig(BaseModel): strategy="replace", # a later layer's script wholesale-replaces an earlier one's default=None, min_length=1, + max_length=100, description=( "Scripted user turns, sent verbatim and in order within ONE agent session. turns[0] is the " "first prompt, so the task sets no initial_prompt. No simulator runs: persona, goal, " "constraints, model, stop_token, stop_on_criteria_pass and check_criteria must keep their " - "defaults, and max_turns must be at least len(turns). The dialog ends after the last turn " + "defaults. max_turns is derived from len(turns). The dialog ends after the last turn " "with stop_reason='script_exhausted'." ), ) @@ -225,7 +226,7 @@ class SimulationConfig(BaseModel): default=8, ge=1, le=100, - description="Hard cap on simulated user<->agent exchanges within a single dialog.", + description="Hard cap on simulated user<->agent exchanges; derived from len(turns) for scripted users.", ) stop_token: str = Field( default=DEFAULT_SIMULATION_STOP_TOKEN, @@ -307,11 +308,7 @@ def _validate_user_source(self) -> Self: ] if changed: raise ValueError(f"simulation.turns scripts the user, so these must keep their defaults: {changed}") - if len(self.turns) > self.max_turns: - raise ValueError( - f"simulation.turns has {len(self.turns)} entries but max_turns is {self.max_turns} — " - + f"raise max_turns to at least {len(self.turns)}." - ) + self.max_turns = len(self.turns) return self @model_validator(mode="after") diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index cbc10026e..2b24266b5 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -274,6 +274,23 @@ def _apply_prompt_overrides( variant: The specific variant being resolved. lineage: Config lineage dict to update. """ + simulation = task.simulation + if ( + simulation is not None + and simulation.enabled + and simulation.is_scripted + and ( + variant.initial_prompt is not None + or variant.initial_prompt_file is not None + or variant.prompt_mutations + or (experiment.defaults is not None and experiment.defaults.prompt_mutations) + ) + ): + raise ValueError( + "simulation.turns supplies every user turn: initial_prompt, initial_prompt_file, " + + "and prompt_mutations cannot override a scripted task." + ) + # Full replacement — skip all mutations if variant.initial_prompt is not None: task.initial_prompt = variant.initial_prompt diff --git a/tests/test_experiment_resolver.py b/tests/test_experiment_resolver.py index c37f7e2ae..b7b11acc5 100644 --- a/tests/test_experiment_resolver.py +++ b/tests/test_experiment_resolver.py @@ -16,6 +16,7 @@ TemplateDirSource, ) from coder_eval.orchestration.experiment import _apply_prompt_overrides, resolve_task_for_variant +from coder_eval.orchestration.task_loader import resolve_variant_initial_prompt_file def _make_task(agent: dict | None = None, **kwargs) -> TaskDefinition: @@ -668,6 +669,33 @@ def test_defaults_mutations_applied_before_variant(self): # defaults prefix first, then variant suffix assert resolved.initial_prompt == "DEFAULT\n\nbase\n\nVARIANT" + @pytest.mark.parametrize("kind", ["inline", "file", "default-mutation", "variant-mutation"]) + def test_scripted_prompt_overrides_refuse_before_mutation(self, tmp_path, kind): + task = _make_task( + agent={"type": "claude-code"}, + initial_prompt=None, + simulation={"enabled": True, "turns": ["first", "second"]}, + ) + variant = ExperimentVariant(variant_id="review") + defaults = None + if kind == "inline": + variant.initial_prompt = "replacement" + elif kind == "file": + (tmp_path / "prompt.txt").write_text("replacement") + variant.initial_prompt_file = "prompt.txt" + resolve_variant_initial_prompt_file(variant, tmp_path) + elif kind == "default-mutation": + defaults = ExperimentDefaults(prompt_mutations=[PromptPrefix(content="prefix")]) + else: + variant.prompt_mutations = [PromptPrefix(content="prefix")] + experiment = ExperimentDefinition(experiment_id="review", defaults=defaults, variants=[variant]) + before = task.model_dump() + lineage = {} + with pytest.raises(ValueError, match=r"simulation\.turns"): + _apply_prompt_overrides(task, experiment, variant, lineage) + assert task.model_dump() == before + assert lineage == {} + def test_variant_initial_prompt_overrides_task_prompt(self): default_exp = _make_default_experiment() task = _make_task(agent={"type": "claude-code"}, initial_prompt="original prompt") diff --git a/tests/test_scripted_dialog.py b/tests/test_scripted_dialog.py index 99e3173ec..39265922a 100644 --- a/tests/test_scripted_dialog.py +++ b/tests/test_scripted_dialog.py @@ -20,6 +20,8 @@ from coder_eval.models import ( AgentKind, EvaluationResult, + ExperimentDefinition, + ExperimentVariant, FileExistsCriterion, FileMatchesRegexCriterion, FinalStatus, @@ -32,6 +34,8 @@ parse_agent_config, simulator_cost_usd, ) +from coder_eval.orchestration.experiment import _apply_prompt_overrides +from coder_eval.orchestration.regrade import task_from_prior from coder_eval.orchestration.task_loader import expand_dataset from coder_eval.orchestrator import Orchestrator from tests.fixtures.mock_agent import MockAgent @@ -356,3 +360,49 @@ async def test_an_agent_that_runs_out_of_its_own_turns_ends_the_script(tmp_path, assert result.simulation is not None assert result.simulation.stop_reason == ("agent_max_turns" if exhaust_on_call < len(TURNS) else "script_exhausted") assert json.loads((tmp_path / "run" / "task.json").read_text())["max_turns_exhausted"] is True + + +@pytest.mark.parametrize("exhaust_on_call", [None, 9]) +@pytest.mark.parametrize("grade", [True, False]) +async def test_nine_turn_script_needs_no_dialog_cap_and_roundtrips(tmp_path, monkeypatch, exhaust_on_call, grade): + turns = [f"Exact question {i}" for i in range(9)] + created = _install_agent(monkeypatch, exhaust_on_call=exhaust_on_call) + task = _task( + turns=turns, + simulation=SimulationConfig(enabled=True, turns=turns), + sandbox=SandboxConfig(driver="tempdir", python=None), + success_criteria=[ + FileMatchesRegexCriterion(path="not-created.txt", pattern="required", description="Missing file") + ], + ) + result = await _run(tmp_path, task, grade=grade) + assert len(created) == 1 and created[0].prompts == turns + assert result.iteration_count == len(result.iterations) == 9 + assert result.simulation.stop_reason == "script_exhausted" + exhausted = exhaust_on_call == 9 + assert result.iterations[-1].max_turns_exhausted is exhausted + assert result.max_turns_exhausted is exhausted + expected = FinalStatus.MAX_TURNS_EXHAUSTED if exhausted else FinalStatus.FAILURE + assert result.final_status == (expected if grade else FinalStatus.NOT_GRADED) + stored = json.loads((tmp_path / "run" / "task.json").read_text(encoding="utf-8")) + assert stored["max_turns_exhausted"] is exhausted + restored = TaskDefinition.model_validate(stored["task_config"]["resolved"]) + detached, _ = task_from_prior(result, tmp_path / "run") + assert restored.simulation.turns == detached.simulation.turns == turns + assert restored.simulation.max_turns == detached.simulation.max_turns == 9 + + +async def test_disabled_script_still_runs_the_overridden_normal_prompt(tmp_path, monkeypatch): + created = _install_agent(monkeypatch) + task = _task( + initial_prompt="original", + simulation=SimulationConfig(enabled=False, turns=TURNS), + sandbox=SandboxConfig(driver="tempdir", python=None), + ) + variant = ExperimentVariant(variant_id="review", initial_prompt="normal replacement") + experiment = ExperimentDefinition(experiment_id="review", variants=[variant]) + _apply_prompt_overrides(task, experiment, variant, {}) + result = await _run(tmp_path, task, grade=False) + assert created[0].prompts == ["normal replacement"] + assert result.simulation is None + assert result.final_status == FinalStatus.NOT_GRADED diff --git a/tests/test_simulation_config.py b/tests/test_simulation_config.py index d04e8dd79..2b3a800ca 100644 --- a/tests/test_simulation_config.py +++ b/tests/test_simulation_config.py @@ -216,11 +216,24 @@ def test_a_recorded_config_validates_again(self): recorded = SimulationConfig(**_scripted_kwargs()).model_dump() assert SimulationConfig(**recorded).turns == ["First question.", "Second question."] - def test_more_turns_than_max_turns_is_rejected_rather_than_truncated(self): - turns = [f"Question {i}." for i in range(9)] + @pytest.mark.parametrize("count", [1, 9, 100]) + @pytest.mark.parametrize("specified", [None, 1, 100]) + def test_script_length_sets_the_dialog_length(self, count, specified): + kwargs = _scripted_kwargs(turns=[f"Question {i}." for i in range(count)]) + if specified is not None: + kwargs["max_turns"] = specified + cfg = SimulationConfig(**kwargs) + assert cfg.max_turns == count + assert SimulationConfig.model_validate(cfg.model_dump()).max_turns == count + + @pytest.mark.parametrize("invalid", [0, 101, 1.5, "invalid"]) + def test_raw_dialog_limit_still_validates_before_derivation(self, invalid): with pytest.raises(ValidationError, match="max_turns"): - SimulationConfig(**_scripted_kwargs(turns=turns)) - assert SimulationConfig(**_scripted_kwargs(turns=turns, max_turns=9)).max_turns == 9 + SimulationConfig(**_scripted_kwargs(max_turns=invalid)) + + def test_script_keeps_the_existing_effective_upper_bound(self): + with pytest.raises(ValidationError, match="turns"): + SimulationConfig(**_scripted_kwargs(turns=["question"] * 101)) def test_token_budget_and_trials_still_apply_to_a_script(self): cfg = SimulationConfig(**_scripted_kwargs(max_total_tokens=50_000, n_trials=3))