From 6e9db40ce2a581f14566eeed0fef5e6539d2ab94 Mon Sep 17 00:00:00 2001 From: NopeCharon <115794254+NopeCharon@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:06:17 +0530 Subject: [PATCH 1/2] fix(claude-code): record the Bedrock route model as model_used On a Bedrock route the Claude SDK reports a friendly alias on AssistantMessage.model (for example `claude-sonnet-5`) even when the run was pinned to a fully qualified inference profile such as `eu.anthropic.claude-sonnet-5`. That alias became TurnRecord.model_used and then task.json model_used, so runs on distinct Bedrock routes grouped under one non-route-specific key, and a campaign that pins qualified ids could not prove the pin from model_used alone. The turn state now reports the resolved route model (`_resolve_effective_model`, already used for cost backfill and ANTHROPIC_MODEL) as model_used on a Bedrock route. Other routes keep the SDK-reported id, which there is the precise dated model. If no model was resolved on Bedrock, the SDK value is still used. The raw SDK alias stays on each messages[].model. Pricing is unaffected: both pricing.py and the evalboard normalize the region and vendor prefixes away before the rate lookup. Fixes #99 --- docs/agents/CLAUDE_CODE.md | 4 +++ src/coder_eval/agents/claude_code_agent.py | 13 +++++++- tests/test_agent.py | 35 +++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 4145e99be..024807040 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -64,6 +64,10 @@ subprocess. `BEDROCK_MODEL` is the route-level default; an explicit `--model` / `-D agent.model=…` / task `agent.model` always wins over it, and bare aliases are auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `global.`). +On Bedrock, `model_used` (per iteration and on the task) records that resolved +inference-profile id, e.g. `eu.anthropic.claude-sonnet-5`. The SDK reports a friendly +alias (`claude-sonnet-5`) instead; that raw value stays on each `messages[].model`. + > **For official benchmarking use the direct API** — the SDK reports accurate > token/cost there. See [User Guide → API Routing](../USER_GUIDE.md#api-routing--benchmarking). diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index aa34da93b..cbd4bb253 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -700,6 +700,17 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage + def model_used(self) -> str | None: + """The turn's comparison-facing model: the resolved route model on Bedrock, else the SDK's. + + On Bedrock the SDK reports a friendly alias (``claude-sonnet-5``) for the + inference profile the run was pinned to (``eu.anthropic.claude-sonnet-5``). + That raw alias stays on each ``AssistantMessage.model``. + """ + if isinstance(self._agent.route, BedrockRoute) and self.effective_model: + return self.effective_model + return self.sdk_model_used + def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -757,7 +768,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso iteration=self.iteration, user_input=self.user_input, agent_output=agent_output, - model_used=self.sdk_model_used, + model_used=self.model_used(), assistant_turn_count=self.assistant_turn_count, messages=list(self.sdk_messages), num_turns=self.num_turns, diff --git a/tests/test_agent.py b/tests/test_agent.py index ef975c0c5..a510b24de 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -12,7 +12,7 @@ from coder_eval.agent import AgentState from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.models import AgentKind, BedrockRoute, DirectRoute, parse_agent_config def test_claude_agent_initialization(): @@ -1944,6 +1944,39 @@ def test_on_user_message_resolves_pending_command(self): assert len(tool_ends) == 1 assert tool_ends[0].status == ToolEndStatus.OK + @pytest.mark.parametrize( + ("route", "effective_model", "expected"), + [ + pytest.param( + BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-5"), + "eu.anthropic.claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + id="bedrock-records-route-model", + ), + pytest.param( + BedrockRoute(region="eu-north-1"), + None, + "claude-sonnet-5", + id="bedrock-without-resolved-model-falls-back-to-sdk", + ), + pytest.param(DirectRoute(), "sonnet", "claude-sonnet-5", id="direct-records-sdk-model"), + ], + ) + def test_model_used_on_agent_end(self, route, effective_model, expected): + from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route) + state, events = self._state(agent) + state.effective_model = effective_model + + state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5")) + state.finalize(AgentEndStatus.COMPLETED) + + (end,) = [e for e in events if isinstance(e, AgentEndEvent)] + assert end.model_used == expected + assert state.sdk_messages[0].model == "claude-sonnet-5" + class TestAgentBaseHelpers: """Unit tests for the shared mid-turn failure kernels on the Agent base.""" From 5b2b2c7a1fe82034ee8ed554973a32693899b6d8 Mon Sep 17 00:00:00 2001 From: NopeCharon <115794254+NopeCharon@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:39:53 +0530 Subject: [PATCH 2/2] fix(claude-code): tighten model_used review follow-ups Make _ClaudeTurnState.model_used a property so it has the same shape as the delegate agent's turn state. Resolve the Bedrock case through _resolve_effective_model so the test covers alias qualification, add a LiteLLM case for the other side of the BedrockRoute guard, assert the collector's TurnRecord, and cover the crashed partial turn. Annotate the parametrized arguments. Document the per-route meaning of model_used in the TurnRecord and EvaluationResult field descriptions and in REPORT_SCHEMA.md, including that the agent-config fallback is not route-resolved. --- docs/REPORT_SCHEMA.md | 7 ++++ docs/agents/CLAUDE_CODE.md | 1 + src/coder_eval/agents/claude_code_agent.py | 5 ++- src/coder_eval/models/results.py | 10 ++++- tests/test_agent.py | 46 +++++++++++++++------- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 20772f909..ecb4307f0 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -242,6 +242,13 @@ row per model-provider error the agent's own client reported, including errors i retried itself; filled by the codex adapter, empty on other backends), `crashed`, `crash_reason`. +`model_used` depends on the route. On a Bedrock route the Claude Code agent records the +resolved inference-profile id (e.g. `eu.anthropic.claude-sonnet-5`), and the SDK's alias +stays on each `messages[].model`. On every other route it is the id the agent reports. +`EvaluationResult.model_used` takes the last iteration's value, else the configured +model unresolved. Strip the region and `anthropic.` prefixes before you group rows by +model. + > **Token invariant.** Summing the four token buckets across `messages` > (assistant + the synthetic `reconciliation` entry) equals `token_usage` exactly. > The `reconciliation` message carries the residual the per-message stream diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 024807040..81a79306d 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -67,6 +67,7 @@ auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `globa On Bedrock, `model_used` (per iteration and on the task) records that resolved inference-profile id, e.g. `eu.anthropic.claude-sonnet-5`. The SDK reports a friendly alias (`claude-sonnet-5`) instead; that raw value stays on each `messages[].model`. +See [Report Schema → TurnRecord](../REPORT_SCHEMA.md#turnrecord) for the per-route contract. > **For official benchmarking use the direct API** — the SDK reports accurate > token/cost there. See [User Guide → API Routing](../USER_GUIDE.md#api-routing--benchmarking). diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index cbd4bb253..5f3103cf2 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -700,8 +700,9 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage + @property def model_used(self) -> str | None: - """The turn's comparison-facing model: the resolved route model on Bedrock, else the SDK's. + """The model recorded on ``AgentEndEvent.model_used``: the resolved route model on Bedrock, else the SDK's. On Bedrock the SDK reports a friendly alias (``claude-sonnet-5``) for the inference profile the run was pinned to (``eu.anthropic.claude-sonnet-5``). @@ -768,7 +769,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso iteration=self.iteration, user_input=self.user_input, agent_output=agent_output, - model_used=self.model_used(), + model_used=self.model_used, assistant_turn_count=self.assistant_turn_count, messages=list(self.sdk_messages), num_turns=self.num_turns, diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 802c00f70..3f4805ec4 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -343,7 +343,10 @@ class TurnRecord(BaseModel): default=None, description="Token usage for this turn (if available from agent SDK)" ) model_used: str | None = Field( - default=None, description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514')" + default=None, + description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514'). On a Bedrock " + + "route the Claude Code agent records the resolved inference-profile id (e.g., " + + "'eu.anthropic.claude-sonnet-5'), not the SDK's alias, so normalize the id before grouping across routes.", ) assistant_turn_count: int = Field( default=0, @@ -536,7 +539,10 @@ class EvaluationResult(BaseModel): variant_id: str = Field(default="default", description="ID of the experiment variant") agent_type: str = Field(description="Type of agent used (registered kind string, e.g. 'claude-code')") model_used: str | None = Field( - default=None, description="Model identifier used for the evaluation (resolved from iterations or agent config)" + default=None, + description="Model identifier used for the evaluation: the last iteration's TurnRecord.model_used, else " + + "the configured agent model. The config fallback is not route-resolved, so a Bedrock row with no " + + "recorded iteration carries the bare configured alias.", ) # Execution metadata diff --git a/tests/test_agent.py b/tests/test_agent.py index a510b24de..60c4a98d5 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -12,7 +12,7 @@ from coder_eval.agent import AgentState from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AgentKind, BedrockRoute, DirectRoute, parse_agent_config +from coder_eval.models import AgentKind, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute, parse_agent_config def test_claude_agent_initialization(): @@ -1944,14 +1944,28 @@ def test_on_user_message_resolves_pending_command(self): assert len(tool_ends) == 1 assert tool_ends[0].status == ToolEndStatus.OK + def _run_alias_turn(self, route: ApiRoute, config_model: str | None, *, crashed: bool = False): + """Drive one turn whose SDK reports the alias ``claude-sonnet-5``; a crash ends it as a TIMEOUT.""" + from coder_eval.streaming.events import AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route) + state, events = self._state(agent) + state.effective_model = agent._resolve_effective_model(config_model, {}, None) + + state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5")) + status = AgentEndStatus.TIMEOUT if crashed else AgentEndStatus.COMPLETED + state.finalize(status, crashed=crashed, crash_reason="boom" if crashed else None) + return agent, state, events + @pytest.mark.parametrize( - ("route", "effective_model", "expected"), + ("route", "config_model", "expected"), [ pytest.param( - BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-5"), - "eu.anthropic.claude-sonnet-5", + BedrockRoute(region="eu-north-1"), + "claude-sonnet-5", "eu.anthropic.claude-sonnet-5", - id="bedrock-records-route-model", + id="bedrock-records-qualified-alias", ), pytest.param( BedrockRoute(region="eu-north-1"), @@ -1960,23 +1974,27 @@ def test_on_user_message_resolves_pending_command(self): id="bedrock-without-resolved-model-falls-back-to-sdk", ), pytest.param(DirectRoute(), "sonnet", "claude-sonnet-5", id="direct-records-sdk-model"), + pytest.param( + LiteLLMRoute(model="zai.glm-5"), "zai.glm-5", "claude-sonnet-5", id="litellm-records-sdk-model" + ), ], ) - def test_model_used_on_agent_end(self, route, effective_model, expected): - from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus - from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock - - agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route) - state, events = self._state(agent) - state.effective_model = effective_model + def test_model_used_on_agent_end(self, route: ApiRoute, config_model: str | None, expected: str): + from coder_eval.streaming.events import AgentEndEvent - state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5")) - state.finalize(AgentEndStatus.COMPLETED) + _, state, events = self._run_alias_turn(route, config_model) (end,) = [e for e in events if isinstance(e, AgentEndEvent)] assert end.model_used == expected + assert state.collector.build_turn_record().model_used == expected assert state.sdk_messages[0].model == "claude-sonnet-5" + def test_model_used_on_crashed_partial_turn(self): + agent, _, _ = self._run_alias_turn(BedrockRoute(region="eu-north-1"), "claude-sonnet-5", crashed=True) + + assert agent.pending_turn is not None + assert agent.pending_turn.model_used == "eu.anthropic.claude-sonnet-5" + class TestAgentBaseHelpers: """Unit tests for the shared mid-turn failure kernels on the Agent base."""