From bd8a8510fe04980b11a2a0d9928be99bf1229e64 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Mon, 2 Nov 2026 23:34:48 +0800 Subject: [PATCH] [feat] models: configure agent input modalities Signed-off-by: Weiming Liao --- apps/api/src/opendirector_api/main.py | 7 +- apps/api/src/opendirector_api/models.py | 3 + .../src/opendirector/domain/capabilities.py | 15 ++ .../opendirector/domain/creation/models.py | 6 +- .../infrastructure/model_configs.py | 12 +- src/backend/models.ts | 1 + src/backend/rules.ts | 1 + src/canvas/ModelServiceForm.tsx | 8 + src/canvas/model-settings.css | 5 + src/contracts/creation.schema.json | 155 ++++++++++++++++++ src/contracts/generated.ts | 4 + tests/test_model_configs.py | 32 +++- 12 files changed, 240 insertions(+), 9 deletions(-) diff --git a/apps/api/src/opendirector_api/main.py b/apps/api/src/opendirector_api/main.py index 2b24bd40..77372260 100644 --- a/apps/api/src/opendirector_api/main.py +++ b/apps/api/src/opendirector_api/main.py @@ -116,7 +116,8 @@ def save_model_batch(value: ModelBatchInput): return model_configs.save_many(value.kind, value.url, value.modelIds, value.apiKey.get_secret_value(), str(value.configId) if value.configId else None, connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode, video_adapter=value.videoAdapter, - video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None) + video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None, + input_modalities=value.inputModalities) @app.post('/api/model-configs/discover') async def discover_models(value: ModelDiscoveryRequest): @@ -158,7 +159,7 @@ async def test_agent_model(value: AgentModelTest): def create_model_config(value: ModelConfigInput): return model_configs.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode, video_adapter=value.videoAdapter, video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None, - capabilities_supplied='videoCapabilities' in value.model_fields_set) + capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities) @app.post('/api/model-configs/capabilities') async def model_capabilities(value: ModelCapabilityRequest): @@ -179,7 +180,7 @@ async def model_capabilities(value: ModelCapabilityRequest): def update_model_config(config_id: UUID, value: ModelConfigInput): return model_configs.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), str(config_id), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode, video_adapter=value.videoAdapter, video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None, - capabilities_supplied='videoCapabilities' in value.model_fields_set) + capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities) @app.delete("/api/model-configs/{config_id}", status_code=204) def delete_model_config(config_id: UUID): diff --git a/apps/api/src/opendirector_api/models.py b/apps/api/src/opendirector_api/models.py index 581d6d9e..85ba3bf7 100644 --- a/apps/api/src/opendirector_api/models.py +++ b/apps/api/src/opendirector_api/models.py @@ -3,6 +3,7 @@ from pydantic import model_validator, BaseModel, ConfigDict, Field, SecretStr from opendirector.domain.creation.editors import ToolDraftBinding +from opendirector.domain.capabilities import AgentInputModality from opendirector.domain.video_capabilities import VideoAdapter, VideoCapabilities @@ -25,6 +26,7 @@ class ModelConfigInput(ModelConnection): url: str = Field(max_length=2048) modelId: str = Field(max_length=200) apiKey: SecretStr = SecretStr("") + inputModalities: list[AgentInputModality] | None = Field(default=None, min_length=1, max_length=2) videoAdapter: VideoAdapter | None = None videoCapabilities: VideoCapabilities | None = None @@ -54,6 +56,7 @@ class ModelConfigPublic(ModelConnection): providerId: UUID | None = Field(default=None, exclude_if=lambda value: value is None) videoAdapter: VideoAdapter = 'auto' videoCapabilities: VideoCapabilities | None = None + inputModalities: list[AgentInputModality] | None = None inputRules: dict = Field(default_factory=dict) id: UUID kind: Literal["agent", "image", "video", "audio", "vision"] diff --git a/packages/core/src/opendirector/domain/capabilities.py b/packages/core/src/opendirector/domain/capabilities.py index a1c8cc04..730a4101 100644 --- a/packages/core/src/opendirector/domain/capabilities.py +++ b/packages/core/src/opendirector/domain/capabilities.py @@ -8,6 +8,8 @@ from dataclasses import dataclass from typing import Literal +from opendirector.domain.projects import ProjectError + Capability = Literal[ 'text.chat', 'image.generate', 'image.edit', 'image.cutout', 'video.generate', 'video.first_frame', 'video.edit', 'audio.speech', 'audio.music', 'image.segment', 'video.matting', @@ -151,3 +153,16 @@ class ChatRequest: class MediaResult: data: bytes suffix: str + + +AgentInputModality = Literal['text', 'image'] + + +def agent_input_modalities(kind, value): + """Declared agent inputs are separate from media-generation reference rules.""" + if value is None: + return ['text'] if kind == 'agent' else None + if (kind != 'agent' or not isinstance(value, (list, tuple)) or 'text' not in value + or len(value) != len(set(value)) or any(item not in {'text', 'image'} for item in value)): + raise ProjectError('对话模型输入模态必须包含文本，可选图片，且不能重复。') + return ['text', *(['image'] if 'image' in value else [])] diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 9364b955..2216cc79 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -5,6 +5,7 @@ from pydantic import Field, SecretStr, field_validator, model_validator +from opendirector.domain.capabilities import agent_input_modalities from opendirector.domain.video_capabilities import VideoCapabilities as ExistingVideoCapabilities from opendirector.domain.video_capabilities import VideoAdapter, check_video_config from opendirector.domain.projects import ProjectError @@ -63,6 +64,7 @@ class ModelConnection(Contract): connectionType: Literal['api', 'self_hosted'] protocol: Protocol authMode: Literal['none', 'bearer'] + inputModalities: tuple[Literal['text', 'image'], ...] | None = None videoAdapter: VideoAdapter = 'auto' videoCapabilities: VideoCapabilities | None = None @@ -96,6 +98,7 @@ def connection(self): if self.protocol not in allowed[self.kind]: raise ValueError('Protocol is incompatible with model category') try: + agent_input_modalities(self.kind, self.inputModalities) check_video_config(self.kind, self.protocol, self.videoAdapter, self.videoCapabilities.model_dump(exclude_none=True) if self.videoCapabilities else None) except ProjectError as error: @@ -129,6 +132,7 @@ class ModelSnapshot(Contract): url: str = Field(min_length=1, max_length=2048) protocol: Protocol connectionType: Literal['api', 'self_hosted'] + inputModalities: tuple[Literal['text', 'image'], ...] | None = Field(default=None, exclude_if=lambda value: value is None) videoAdapter: VideoAdapter = 'auto' inputRules: InputRules | None = None @@ -146,5 +150,5 @@ def model_name(cls, value): def connection(self): ModelConnection(kind=self.kind, url=self.url, modelId=self.modelId, connectionType=self.connectionType, protocol=self.protocol, - authMode='none', videoAdapter=self.videoAdapter) + authMode='none', videoAdapter=self.videoAdapter, inputModalities=self.inputModalities) return self diff --git a/packages/core/src/opendirector/infrastructure/model_configs.py b/packages/core/src/opendirector/infrastructure/model_configs.py index aab687c5..e2134349 100644 --- a/packages/core/src/opendirector/infrastructure/model_configs.py +++ b/packages/core/src/opendirector/infrastructure/model_configs.py @@ -11,6 +11,7 @@ from opendirector.domain.projects import ProjectError, identifier from opendirector.domain.input_rules import input_rules +from opendirector.domain.capabilities import agent_input_modalities from opendirector.domain.video_capabilities import check_video_config from opendirector.infrastructure.local_projects import child, read_json, write_json @@ -78,7 +79,7 @@ def _read(self) -> list[dict]: item.update(services[item['providerId']]) seen = set() for item in value["models"]: - if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'videoAdapter', 'videoCapabilities', 'providerId'}: + if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'videoAdapter', 'videoCapabilities', 'providerId', 'inputModalities'}: raise ValueError() identifier(item["id"]) if item["id"] in seen or item["kind"] not in MODEL_KINDS: @@ -91,6 +92,7 @@ def _read(self) -> list[dict]: if 'videoAdapter' not in item: item['videoAdapter'] = ('minimax_h3' if item['modelId'].rsplit('/', 1)[-1].lower() == 'minimax-h3' else 'fasth3') if item['kind'] == 'video' and item['protocol'] == 'vllm_omni' else 'auto' + item['inputModalities'] = agent_input_modalities(item['kind'], item.get('inputModalities')) item['videoCapabilities'] = check_video_config(item['kind'], item['protocol'], item['videoAdapter'], item.get('videoCapabilities')) if item['authMode'] == 'bearer' and not fields['apiKey']: raise ValueError() @@ -100,7 +102,7 @@ def _read(self) -> list[dict]: @staticmethod def _public(item: dict) -> dict: - return {**({'providerId': item['providerId']} if 'providerId' in item else {}), **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities")}, 'inputRules': input_rules(item)} + return {**({'providerId': item['providerId']} if 'providerId' in item else {}), **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities", "inputModalities")}, 'inputRules': input_rules(item)} def list(self) -> list[dict]: with self.lock: @@ -117,7 +119,7 @@ def resolve(self, config_id: str, kind: str) -> dict: raise ProjectError("所选模型不存在或类别不符，请在设置中配置并重新选择。", 409) return dict(item) - def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False) -> dict: + def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False, input_modalities=None) -> dict: if kind not in MODEL_KINDS: raise ProjectError("无效的模型类别。") fields = model_fields(url, model_id, api_key) @@ -139,6 +141,7 @@ def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str raise ProjectError('请填写 API Key，或将鉴权方式设为无需鉴权。') fields['apiKey'] = existing['apiKey'] item = {'id': config_id or str(uuid4()), 'kind': kind, **fields, **connection} + item['inputModalities'] = agent_input_modalities(kind, input_modalities if input_modalities is not None else (existing or {}).get('inputModalities')) item['videoAdapter'] = video_adapter if video_adapter is not None else (existing or {}).get('videoAdapter', 'auto') raw = video_capabilities if capabilities_supplied or video_capabilities is not None else (existing or {}).get('videoCapabilities') item['videoCapabilities'] = check_video_config(kind, connection['protocol'], item['videoAdapter'], raw) @@ -177,7 +180,7 @@ def _write(self, models: list[dict]) -> None: def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, - auth_mode=None, video_adapter=None, video_capabilities=None) -> list[dict]: + auth_mode=None, video_adapter=None, video_capabilities=None, input_modalities=None) -> list[dict]: if kind not in MODEL_KINDS or not 1 <= len(model_ids) <= 200: raise ProjectError('请选择 1—200 个模型。') connection = connection_fields(kind, connection_type, protocol, auth_mode) @@ -209,6 +212,7 @@ def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, old = existing item = {'id': old['id'] if old else str(uuid4()), 'kind': kind, **field, **connection, 'apiKey': key, 'providerId': provider_id, 'videoAdapter': video_adapter or 'auto'} + item['inputModalities'] = agent_input_modalities(kind, input_modalities if input_modalities is not None else (old or {}).get('inputModalities')) item['videoCapabilities'] = check_video_config(kind, connection['protocol'], item['videoAdapter'], video_capabilities) profile = input_rules(item)['profile'] if item['videoCapabilities'] is not None and profile in {'fasth3-omni', 'vllm-omni-h3'}: diff --git a/src/backend/models.ts b/src/backend/models.ts index 0651de12..72a134b2 100644 --- a/src/backend/models.ts +++ b/src/backend/models.ts @@ -25,6 +25,7 @@ export interface ModelConfigInput { connectionType: "api" | "self_hosted"; protocol: NonNullable; authMode: "none" | "bearer"; + inputModalities?: ("text" | "image")[] | null; videoAdapter?: VideoAdapter; videoCapabilities?: VideoCapabilities | null; } diff --git a/src/backend/rules.ts b/src/backend/rules.ts index b3c7c9ac..acedb701 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -53,6 +53,7 @@ export interface ModelConfig { | null; authMode: "none" | "bearer" | null; inputRules: InputRules; + inputModalities?: ("text" | "image")[] | null; videoAdapter?: VideoAdapter; videoCapabilities?: VideoCapabilities | null; } diff --git a/src/canvas/ModelServiceForm.tsx b/src/canvas/ModelServiceForm.tsx index 0a72e291..a37c733f 100644 --- a/src/canvas/ModelServiceForm.tsx +++ b/src/canvas/ModelServiceForm.tsx @@ -79,6 +79,7 @@ export function ModelServiceForm({ : (existing?.protocol ?? PROTOCOLS[kind][0][0]), authMode: existing?.authMode ?? "none", videoAdapter: existing?.videoAdapter ?? "auto", + inputModalities: kind === "agent" ? existing?.inputModalities ?? ["text"] : undefined, }); const [selectedModels, setSelectedModels] = useState(() => existing @@ -370,6 +371,13 @@ export function ModelServiceForm({

+ {kind === "agent" &&
+ 输入能力 + + +

应用于本次选中的模型。仅在模型及接口支持图片理解时启用。

+
} summary svg { transform: rotate(180deg); } + +.cw-model-modalities { border: 0; border-top: 1px solid var(--settings-border); margin: 0 0 20px; padding: 16px 0 0; display: flex; flex-wrap: wrap; gap: 12px 24px; } +.cw-model-modalities legend { padding: 0 8px 0 0; font-size: var(--settings-label-size); } +.cw-model-modalities label { display: flex; align-items: center; gap: 6px; font-size: var(--settings-control-size); } +.cw-model-modalities .cw-model-note { flex-basis: 100%; margin: 0; } diff --git a/src/contracts/creation.schema.json b/src/contracts/creation.schema.json index 86d291e7..1fab0c76 100644 --- a/src/contracts/creation.schema.json +++ b/src/contracts/creation.schema.json @@ -6597,6 +6597,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -6708,6 +6727,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -6810,6 +6848,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -6894,6 +6951,25 @@ "title": "Connectiontype", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -18838,6 +18914,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -18876,6 +18971,7 @@ "connectionType", "protocol", "authMode", + "inputModalities", "videoAdapter", "videoCapabilities", "id", @@ -18938,6 +19034,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -18983,6 +19098,7 @@ "connectionType", "protocol", "authMode", + "inputModalities", "videoAdapter", "videoCapabilities", "apiKey" @@ -19043,6 +19159,25 @@ "title": "Authmode", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ @@ -19073,6 +19208,7 @@ "connectionType", "protocol", "authMode", + "inputModalities", "videoAdapter", "videoCapabilities" ], @@ -19129,6 +19265,25 @@ "title": "Connectiontype", "type": "string" }, + "inputModalities": { + "anyOf": [ + { + "items": { + "enum": [ + "text", + "image" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inputmodalities" + }, "videoAdapter": { "default": "auto", "enum": [ diff --git a/src/contracts/generated.ts b/src/contracts/generated.ts index 6ce3f22e..3d6674b7 100644 --- a/src/contracts/generated.ts +++ b/src/contracts/generated.ts @@ -756,6 +756,7 @@ export interface ModelConnectionInput extends ContractInput { readonly connectionType: "api" | "self_hosted"; readonly protocol: "openai" | "volcengine" | "vllm_omni" | "vllm_omni_segmentation" | "vllm_omni_music"; readonly authMode: "none" | "bearer"; + readonly inputModalities?: ReadonlyArray<"text" | "image"> | null; readonly videoAdapter?: "auto" | "fasth3" | "minimax_h3" | "standard"; readonly videoCapabilities?: VideoCapabilitiesInput | null; } @@ -767,6 +768,7 @@ export interface ModelSnapshotInput extends ContractInput { readonly url: string; readonly protocol: "openai" | "volcengine" | "vllm_omni" | "vllm_omni_segmentation" | "vllm_omni_music"; readonly connectionType: "api" | "self_hosted"; + readonly inputModalities?: ReadonlyArray<"text" | "image"> | null; readonly videoAdapter?: "auto" | "fasth3" | "minimax_h3" | "standard"; readonly inputRules?: InputRulesInput | null; } @@ -2124,6 +2126,7 @@ export interface ModelConnection extends Contract { readonly connectionType: "api" | "self_hosted"; readonly protocol: "openai" | "volcengine" | "vllm_omni" | "vllm_omni_segmentation" | "vllm_omni_music"; readonly authMode: "none" | "bearer"; + readonly inputModalities: ReadonlyArray<"text" | "image"> | null; readonly videoAdapter: "auto" | "fasth3" | "minimax_h3" | "standard"; readonly videoCapabilities: VideoCapabilities | null; } @@ -2135,6 +2138,7 @@ export interface ModelSnapshot extends Contract { readonly url: string; readonly protocol: "openai" | "volcengine" | "vllm_omni" | "vllm_omni_segmentation" | "vllm_omni_music"; readonly connectionType: "api" | "self_hosted"; + readonly inputModalities?: ReadonlyArray<"text" | "image"> | null; readonly videoAdapter: "auto" | "fasth3" | "minimax_h3" | "standard"; readonly inputRules: InputRules | null; } diff --git a/tests/test_model_configs.py b/tests/test_model_configs.py index 8a583135..3c9a5054 100644 --- a/tests/test_model_configs.py +++ b/tests/test_model_configs.py @@ -30,7 +30,7 @@ def test_four_categories_survive_restart_without_exposing_secrets(api): response = client.post("/api/model-configs", json=payload(kind)) assert response.status_code == 200 assert "test-secret" not in response.text - assert set(response.json()) == {"id", "kind", "url", "modelId", "inputRules", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities"} + assert set(response.json()) == {"id", "kind", "url", "modelId", "inputRules", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities", "inputModalities"} configs.append(response.json()) assert len({item["id"] for item in configs}) == 5 assert client.get("/api/model-configs").json() == configs @@ -322,3 +322,33 @@ def test_batch_edit_legacy_segmentation_preserves_identity(api): assert response.status_code == 200, response.text assert response.json()[0]['id'] == saved['id'] assert len(client.get('/api/model-configs').json()) == 2 + + +def test_agent_image_input_persists_through_batch_edits_and_restart(api): + client, store = api + saved = client.post('/api/model-configs/batch', json=payload(modelIds=['vision-agent'], + inputModalities=['text', 'image'])).json()[0] + assert saved['inputModalities'] == ['text', 'image'] + resolved = ModelConfigs(store.repository).resolve(saved['id'], 'agent') + assert resolved['inputModalities'] == ['text', 'image'] + unchanged = client.patch('/api/model-configs/' + saved['id'], json=payload(modelId='vision-agent', apiKey='')) + assert unchanged.status_code == 200 + assert unchanged.json()['inputModalities'] == ['text', 'image'] + disabled = client.post('/api/model-configs/batch', json=payload(configId=saved['id'], + modelIds=['vision-agent'], apiKey='', inputModalities=['text'])) + assert disabled.status_code == 200 + assert disabled.json()[0]['inputModalities'] == ['text'] + + +@pytest.mark.parametrize('modalities', [[], ['image'], ['text', 'text'], ['text', 'video']]) +def test_invalid_agent_input_modalities_do_not_write(api, modalities): + client, store = api + response = client.post('/api/model-configs', json=payload(inputModalities=modalities)) + assert response.status_code in {400, 422} + assert ModelConfigs(store.repository).list() == [] + + +def test_media_generation_model_cannot_declare_agent_modalities(api): + client, _ = api + response = client.post('/api/model-configs', json=payload(kind='image', inputModalities=['text', 'image'])) + assert response.status_code == 400