diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 20772f90..ecb4307f 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -242,6 +242,13 @@ row per model-provider error the agent's own client reported, including errors i retried itself; filled by the codex adapter, empty on other backends), `crashed`, `crash_reason`. +`model_used` depends on the route. On a Bedrock route the Claude Code agent records the +resolved inference-profile id (e.g. `eu.anthropic.claude-sonnet-5`), and the SDK's alias +stays on each `messages[].model`. On every other route it is the id the agent reports. +`EvaluationResult.model_used` takes the last iteration's value, else the configured +model unresolved. Strip the region and `anthropic.` prefixes before you group rows by +model. + > **Token invariant.** Summing the four token buckets across `messages` > (assistant + the synthetic `reconciliation` entry) equals `token_usage` exactly. > The `reconciliation` message carries the residual the per-message stream diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 4145e99b..81a79306 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -64,6 +64,11 @@ subprocess. `BEDROCK_MODEL` is the route-level default; an explicit `--model` / `-D agent.model=…` / task `agent.model` always wins over it, and bare aliases are auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `global.`). +On Bedrock, `model_used` (per iteration and on the task) records that resolved +inference-profile id, e.g. `eu.anthropic.claude-sonnet-5`. The SDK reports a friendly +alias (`claude-sonnet-5`) instead; that raw value stays on each `messages[].model`. +See [Report Schema → TurnRecord](../REPORT_SCHEMA.md#turnrecord) for the per-route contract. + > **For official benchmarking use the direct API** — the SDK reports accurate > token/cost there. See [User Guide → API Routing](../USER_GUIDE.md#api-routing--benchmarking). diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index aa34da93..5f3103cf 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -700,6 +700,18 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage + @property + def model_used(self) -> str | None: + """The model recorded on ``AgentEndEvent.model_used``: the resolved route model on Bedrock, else the SDK's. + + On Bedrock the SDK reports a friendly alias (``claude-sonnet-5``) for the + inference profile the run was pinned to (``eu.anthropic.claude-sonnet-5``). + That raw alias stays on each ``AssistantMessage.model``. + """ + if isinstance(self._agent.route, BedrockRoute) and self.effective_model: + return self.effective_model + return self.sdk_model_used + def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -757,7 +769,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso iteration=self.iteration, user_input=self.user_input, agent_output=agent_output, - model_used=self.sdk_model_used, + model_used=self.model_used, assistant_turn_count=self.assistant_turn_count, messages=list(self.sdk_messages), num_turns=self.num_turns, diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 802c00f7..3f4805ec 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -343,7 +343,10 @@ class TurnRecord(BaseModel): default=None, description="Token usage for this turn (if available from agent SDK)" ) model_used: str | None = Field( - default=None, description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514')" + default=None, + description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514'). On a Bedrock " + + "route the Claude Code agent records the resolved inference-profile id (e.g., " + + "'eu.anthropic.claude-sonnet-5'), not the SDK's alias, so normalize the id before grouping across routes.", ) assistant_turn_count: int = Field( default=0, @@ -536,7 +539,10 @@ class EvaluationResult(BaseModel): variant_id: str = Field(default="default", description="ID of the experiment variant") agent_type: str = Field(description="Type of agent used (registered kind string, e.g. 'claude-code')") model_used: str | None = Field( - default=None, description="Model identifier used for the evaluation (resolved from iterations or agent config)" + default=None, + description="Model identifier used for the evaluation: the last iteration's TurnRecord.model_used, else " + + "the configured agent model. The config fallback is not route-resolved, so a Bedrock row with no " + + "recorded iteration carries the bare configured alias.", ) # Execution metadata diff --git a/tests/test_agent.py b/tests/test_agent.py index ef975c0c..60c4a98d 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -12,7 +12,7 @@ from coder_eval.agent import AgentState from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.models import AgentKind, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute, parse_agent_config def test_claude_agent_initialization(): @@ -1944,6 +1944,57 @@ def test_on_user_message_resolves_pending_command(self): assert len(tool_ends) == 1 assert tool_ends[0].status == ToolEndStatus.OK + def _run_alias_turn(self, route: ApiRoute, config_model: str | None, *, crashed: bool = False): + """Drive one turn whose SDK reports the alias ``claude-sonnet-5``; a crash ends it as a TIMEOUT.""" + from coder_eval.streaming.events import AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route) + state, events = self._state(agent) + state.effective_model = agent._resolve_effective_model(config_model, {}, None) + + state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5")) + status = AgentEndStatus.TIMEOUT if crashed else AgentEndStatus.COMPLETED + state.finalize(status, crashed=crashed, crash_reason="boom" if crashed else None) + return agent, state, events + + @pytest.mark.parametrize( + ("route", "config_model", "expected"), + [ + pytest.param( + BedrockRoute(region="eu-north-1"), + "claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + id="bedrock-records-qualified-alias", + ), + pytest.param( + BedrockRoute(region="eu-north-1"), + None, + "claude-sonnet-5", + id="bedrock-without-resolved-model-falls-back-to-sdk", + ), + pytest.param(DirectRoute(), "sonnet", "claude-sonnet-5", id="direct-records-sdk-model"), + pytest.param( + LiteLLMRoute(model="zai.glm-5"), "zai.glm-5", "claude-sonnet-5", id="litellm-records-sdk-model" + ), + ], + ) + def test_model_used_on_agent_end(self, route: ApiRoute, config_model: str | None, expected: str): + from coder_eval.streaming.events import AgentEndEvent + + _, state, events = self._run_alias_turn(route, config_model) + + (end,) = [e for e in events if isinstance(e, AgentEndEvent)] + assert end.model_used == expected + assert state.collector.build_turn_record().model_used == expected + assert state.sdk_messages[0].model == "claude-sonnet-5" + + def test_model_used_on_crashed_partial_turn(self): + agent, _, _ = self._run_alias_turn(BedrockRoute(region="eu-north-1"), "claude-sonnet-5", crashed=True) + + assert agent.pending_turn is not None + assert agent.pending_turn.model_used == "eu.anthropic.claude-sonnet-5" + class TestAgentBaseHelpers: """Unit tests for the shared mid-turn failure kernels on the Agent base."""