diff --git a/strix/config/models.py b/strix/config/models.py index 102d132cfd..66063d3d1a 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -45,6 +45,7 @@ from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_input from strix.config.tool_call_limits import TurnToolCallLimiter from strix.llm import request_log +from strix.llm.content_blocks import tolerate_content_blocks if TYPE_CHECKING: @@ -632,7 +633,7 @@ def get_model(self, model_name: str | None) -> Model: base_url=None, ) else: - model = super().get_model(model_name) + model = tolerate_content_blocks(super().get_model(model_name)) resolved_name = model_name or llm.model or "unknown" if _routes_via_litellm(model): # LiteLLM's callbacks log every reply; only a cancelled attempt diff --git a/strix/llm/content_blocks.py b/strix/llm/content_blocks.py new file mode 100644 index 0000000000..5ee5f9ab51 --- /dev/null +++ b/strix/llm/content_blocks.py @@ -0,0 +1,196 @@ +"""Accept chat completions whose message content is a list of blocks. + +Some OpenAI-compatible endpoints return ``message.content`` as a list of typed +blocks instead of a string. Mistral does this for its reasoning models, for +example ``[{"type": "thinking", "thinking": [...]}, {"type": "text", "text": +"OK"}]``, both in full responses and in each streamed ``delta.content``. The +SDK's chat-completions converter and stream handler assume a string and fail +validation on the first call, which surfaces as LLM CONNECTION FAILED even +though the endpoint answered normally. + +:class:`BlockTolerantChatCompletionsModel` flattens that content before the SDK +reads it. Text blocks become the string content. Thinking blocks become +``reasoning_content``, which the SDK already turns into a reasoning item and +does not replay to non-DeepSeek models. Block types Strix does not know are +skipped rather than treated as an error. String content passes through as is. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Literal, cast, overload + +from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel + + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from agents.agent_output import AgentOutputSchemaBase + from agents.handoffs import Handoff + from agents.items import TResponseInputItem + from agents.model_settings import ModelSettings + from agents.models.interface import Model, ModelTracing + from agents.tool import Tool + from agents.tracing import GenerationSpanData, Span + from openai import AsyncStream + from openai.types.chat import ChatCompletion, ChatCompletionChunk + from openai.types.responses import Response, ResponsePromptParam + + +logger = logging.getLogger(__name__) + +_TEXT_BLOCK_TYPES = frozenset({"text", "output_text"}) +_THINKING_BLOCK_TYPES = frozenset({"thinking", "reasoning"}) + + +def _field(block: Any, name: str) -> Any: + if isinstance(block, dict): + return cast("dict[str, Any]", block).get(name) + return getattr(block, name, None) + + +def _text_of(value: Any) -> str: + """The text in a block field: a string, or a list of text chunks (Mistral's thinking).""" + if isinstance(value, str): + return value + if isinstance(value, list): + return "".join( + part if isinstance(part, str) else _text_of(_field(part, "text")) + for part in cast("Iterable[Any]", value) + ) + return "" + + +def split_content_blocks(blocks: Iterable[Any]) -> tuple[str, str]: + """Split list-shaped message content into ``(text, thinking)``.""" + text: list[str] = [] + thinking: list[str] = [] + for block in blocks: + if isinstance(block, str): + text.append(block) + continue + kind = _field(block, "type") + if kind in _TEXT_BLOCK_TYPES: + text.append(_text_of(_field(block, "text"))) + elif kind in _THINKING_BLOCK_TYPES: + body = _field(block, "thinking") + thinking.append(_text_of(body if body is not None else _field(block, "text"))) + else: + logger.debug("Skipping unsupported content block type %r", kind) + return "".join(text), "".join(thinking) + + +def flatten_message_content(message: Any) -> None: + """Rewrite a list-valued ``content`` on a message or stream delta in place.""" + content = getattr(message, "content", None) + if not isinstance(content, list): + return + text, thinking = split_content_blocks(cast("Iterable[Any]", content)) + message.content = text or None + if thinking and not getattr(message, "reasoning_content", None): + message.reasoning_content = thinking + + +class _FlattenedStream: + """An ``AsyncStream`` of chunks whose deltas have been flattened.""" + + def __init__(self, stream: AsyncStream[ChatCompletionChunk]) -> None: + self._stream = stream + + def __aiter__(self) -> AsyncIterator[ChatCompletionChunk]: + return self._chunks() + + async def _chunks(self) -> AsyncIterator[ChatCompletionChunk]: + async for chunk in self._stream: + for choice in chunk.choices: + flatten_message_content(choice.delta) + yield chunk + + async def aclose(self) -> None: + await self._stream.close() + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + +class BlockTolerantChatCompletionsModel(OpenAIChatCompletionsModel): + """Chat-completions model that accepts list-shaped message content.""" + + @classmethod + def from_stock(cls, model: OpenAIChatCompletionsModel) -> BlockTolerantChatCompletionsModel: + return cls( + model=model.model, + openai_client=model._client, + should_replay_reasoning_content=model.should_replay_reasoning_content, + strict_feature_validation=model._strict_feature_validation, + buffer_streamed_tool_calls=model._buffer_streamed_tool_calls, + ) + + @overload + async def _fetch_response( + self, + system_instructions: str | None, + input: str | list[TResponseInputItem], + model_settings: ModelSettings, + tools: list[Tool], + output_schema: AgentOutputSchemaBase | None, + handoffs: list[Handoff], + span: Span[GenerationSpanData], + tracing: ModelTracing, + stream: Literal[True], + prompt: ResponsePromptParam | None = None, + ) -> tuple[Response, AsyncStream[ChatCompletionChunk]]: ... + + @overload + async def _fetch_response( + self, + system_instructions: str | None, + input: str | list[TResponseInputItem], + model_settings: ModelSettings, + tools: list[Tool], + output_schema: AgentOutputSchemaBase | None, + handoffs: list[Handoff], + span: Span[GenerationSpanData], + tracing: ModelTracing, + stream: Literal[False], + prompt: ResponsePromptParam | None = None, + ) -> ChatCompletion: ... + + async def _fetch_response( + self, + system_instructions: str | None, + input: str | list[TResponseInputItem], # noqa: A002 - matches the SDK signature + model_settings: ModelSettings, + tools: list[Tool], + output_schema: AgentOutputSchemaBase | None, + handoffs: list[Handoff], + span: Span[GenerationSpanData], + tracing: ModelTracing, + stream: bool = False, + prompt: ResponsePromptParam | None = None, + ) -> ChatCompletion | tuple[Response, AsyncStream[ChatCompletionChunk]]: + args = (system_instructions, input, model_settings, tools, output_schema, handoffs) + if stream: + response, chunks = await super()._fetch_response( + *args, span, tracing, stream=True, prompt=prompt + ) + # handle_stream only iterates and closes the stream. + return response, cast("AsyncStream[ChatCompletionChunk]", _FlattenedStream(chunks)) + completion = await super()._fetch_response( + *args, span, tracing, stream=False, prompt=prompt + ) + for choice in completion.choices: + flatten_message_content(choice.message) + return completion + + +def tolerate_content_blocks(model: Model) -> Model: + """Swap the SDK's stock chat-completions model for the block-tolerant one. + + Only the exact stock class is replaced, so a caller's own subclass is left + alone. Every other model is returned unchanged. + """ + if type(model) is not OpenAIChatCompletionsModel: + return model + return BlockTolerantChatCompletionsModel.from_stock(model) diff --git a/tests/test_content_blocks.py b/tests/test_content_blocks.py new file mode 100644 index 0000000000..038c16fdb5 --- /dev/null +++ b/tests/test_content_blocks.py @@ -0,0 +1,308 @@ +"""Chat completions whose message content is a list of blocks (#1357). + +Mistral serves its reasoning models with ``content`` as a list such as +``[{"type": "thinking", ...}, {"type": "text", "text": "OK"}]``, in full +responses and in every streamed delta. The stock SDK model fails validation on +both paths. A local server that answers the way Mistral does shows the stock +model failing and the block-tolerant model returning the text, with the +thinking kept as a reasoning item. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import TYPE_CHECKING, Any + +import litellm +import pytest +from agents.model_settings import ModelSettings +from agents.models import _openai_shared +from agents.models.interface import ModelTracing +from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel +from openai import AsyncOpenAI +from openai.types.responses import ( + ResponseCompletedEvent, + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, +) +from pydantic import ValidationError + +from strix.config.models import ( + StrixProvider, + _NonStreamingModel, + _TurnGuardModel, + configure_sdk_model_defaults, +) +from strix.config.settings import Settings +from strix.llm.content_blocks import ( + BlockTolerantChatCompletionsModel, + split_content_blocks, + tolerate_content_blocks, +) +from strix.llm.request_log import RequestLoggingModel + + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + + +def _thinking(text: str) -> dict[str, Any]: + # Mistral nests the thinking text in a list of text chunks. + return {"type": "thinking", "thinking": [{"type": "text", "text": text}]} + + +def _completion(message: dict[str, Any], finish_reason: str = "stop") -> dict[str, Any]: + return { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "magistral", + "choices": [{"index": 0, "finish_reason": finish_reason, "message": message}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + + +def _chunk(content: Any) -> dict[str, Any]: + return { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "magistral", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": content}}], + } + + +_REPLY: dict[str, Any] = {} +_STREAM_DELTAS: list[Any] = [] + + +class _Handler(BaseHTTPRequestHandler): + """An endpoint that answers like Mistral's reasoning models.""" + + def log_message(self, *args: Any) -> None: + pass + + def do_POST(self) -> None: + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + if body.get("stream"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for content in _STREAM_DELTAS: + self.wfile.write(f"data: {json.dumps(_chunk(content))}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + return + payload = json.dumps(_REPLY).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +@pytest.fixture +def endpoint() -> Iterator[str]: + _REPLY.clear() + _REPLY.update( + _completion( + { + "role": "assistant", + "content": [_thinking("user wants OK"), {"type": "text", "text": "OK"}], + } + ) + ) + _STREAM_DELTAS[:] = [ + [_thinking("user ")], + [_thinking("wants OK")], + [{"type": "text", "text": "O"}], + [{"type": "text", "text": "K"}], + ] + server = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +def _client(base_url: str) -> AsyncOpenAI: + return AsyncOpenAI(api_key="tok", base_url=base_url) + + +def _call_kwargs() -> dict[str, Any]: + return { + "system_instructions": "s", + "input": "hi", + "model_settings": ModelSettings(), + "tools": [], + "output_schema": None, + "handoffs": [], + "tracing": ModelTracing.DISABLED, + "previous_response_id": None, + "conversation_id": None, + "prompt": None, + } + + +async def _drain(gen: AsyncIterator[Any]) -> list[Any]: + return [event async for event in gen] + + +def _text(item: Any) -> str: + assert isinstance(item, ResponseOutputMessage) + return "".join(part.text for part in item.content) + + +def _reasoning(item: Any) -> str: + assert isinstance(item, ResponseReasoningItem) + return "".join(part.text for part in item.summary) + + +@pytest.mark.parametrize( + ("blocks", "expected"), + [ + ([_thinking("plan"), {"type": "text", "text": "OK"}], ("OK", "plan")), + ( + [{"type": "thinking", "thinking": "plan"}, {"type": "text", "text": "OK"}], + ("OK", "plan"), + ), + ([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], ("ab", "")), + (["plain", {"type": "text", "text": " text"}], ("plain text", "")), + ([{"type": "image_url", "image_url": {}}, {"type": "text", "text": "OK"}], ("OK", "")), + ([], ("", "")), + ], +) +def test_split_content_blocks(blocks: list[Any], expected: tuple[str, str]) -> None: + assert split_content_blocks(blocks) == expected + + +@pytest.mark.asyncio +async def test_stock_model_rejects_block_content(endpoint: str) -> None: + # The failure #1357 reports, on both SDK paths. + model = OpenAIChatCompletionsModel(model="magistral", openai_client=_client(endpoint)) + with pytest.raises(ValidationError, match="ResponseOutputText"): + await model.get_response(**_call_kwargs()) + with pytest.raises(ValidationError, match="ResponseTextDeltaEvent"): + await _drain(model.stream_response(**_call_kwargs())) + + +@pytest.mark.asyncio +async def test_block_content_get_response(endpoint: str) -> None: + model = BlockTolerantChatCompletionsModel(model="magistral", openai_client=_client(endpoint)) + response = await model.get_response(**_call_kwargs()) + + reasoning, message = response.output + assert _reasoning(reasoning) == "user wants OK" + assert _text(message) == "OK" + + +@pytest.mark.asyncio +async def test_block_content_stream_response(endpoint: str) -> None: + model = BlockTolerantChatCompletionsModel(model="magistral", openai_client=_client(endpoint)) + events = await _drain(model.stream_response(**_call_kwargs())) + + completed = events[-1] + assert isinstance(completed, ResponseCompletedEvent) + reasoning, message = completed.response.output + assert _reasoning(reasoning) == "user wants OK" + assert _text(message) == "OK" + + +@pytest.mark.asyncio +async def test_thinking_only_content_keeps_the_tool_call(endpoint: str) -> None: + # A turn that thinks and then calls a tool has no text block at all. + _REPLY.clear() + _REPLY.update( + _completion( + { + "role": "assistant", + "content": [_thinking("need the tool")], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "do_thing", "arguments": '{"n": 1}'}, + } + ], + }, + finish_reason="tool_calls", + ) + ) + model = BlockTolerantChatCompletionsModel(model="magistral", openai_client=_client(endpoint)) + response = await model.get_response(**_call_kwargs()) + + reasoning, tool_call = response.output + assert _reasoning(reasoning) == "need the tool" + assert isinstance(tool_call, ResponseFunctionToolCall) + assert tool_call.name == "do_thing" + + +@pytest.mark.asyncio +async def test_string_content_is_untouched(endpoint: str) -> None: + _REPLY.clear() + _REPLY.update(_completion({"role": "assistant", "content": "plain reply"})) + _STREAM_DELTAS[:] = ["plain ", "reply"] + model = BlockTolerantChatCompletionsModel(model="magistral", openai_client=_client(endpoint)) + + response = await model.get_response(**_call_kwargs()) + assert [_text(item) for item in response.output] == ["plain reply"] + + events = await _drain(model.stream_response(**_call_kwargs())) + assert [_text(item) for item in events[-1].response.output] == ["plain reply"] + + +def test_tolerate_content_blocks_keeps_the_model_configuration() -> None: + client = AsyncOpenAI(api_key="tok", base_url="https://gateway.example/v1") + stock = OpenAIChatCompletionsModel( + model="magistral", + openai_client=client, + strict_feature_validation=True, + buffer_streamed_tool_calls=True, + ) + wrapped = tolerate_content_blocks(stock) + + assert isinstance(wrapped, BlockTolerantChatCompletionsModel) + assert wrapped.model == "magistral" + assert wrapped._client is client + assert wrapped._strict_feature_validation is True + assert wrapped._buffer_streamed_tool_calls is True + + +def test_tolerate_content_blocks_leaves_subclasses_alone() -> None: + class _Custom(OpenAIChatCompletionsModel): + pass + + custom = _Custom(model="m", openai_client=AsyncOpenAI(api_key="tok")) + assert tolerate_content_blocks(custom) is custom + + +def test_provider_serves_chat_completions_through_the_tolerant_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The #1357 setup: an openai/ model id on a custom base URL. + # configure_sdk_model_defaults writes these process-wide; register them so + # they are put back and later tests don't pick up the fake endpoint. + monkeypatch.setattr(_openai_shared, "_use_responses_by_default", True) + monkeypatch.setattr(_openai_shared, "_default_openai_client", None) + monkeypatch.setattr(_openai_shared, "_default_openai_key", None) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "headers", litellm.headers) + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setenv("OPENAI_BASE_URL", "") + monkeypatch.setenv("STRIX_LLM", "openai/zai-glm-latest") + monkeypatch.setenv("LLM_API_KEY", "test-key") + monkeypatch.setenv("LLM_API_BASE", "https://api.mistral.example/v1") + monkeypatch.setenv("STRIX_API_TYPE", "chat_completions") + configure_sdk_model_defaults(Settings()) + + model = StrixProvider().get_model("openai/zai-glm-latest") + while isinstance(model, _NonStreamingModel | _TurnGuardModel | RequestLoggingModel): + model = model._inner + assert isinstance(model, BlockTolerantChatCompletionsModel)