[feat] models: configure agent input modalities

apps/api/src/opendirector_api/main.py

@@ -116,7 +116,8 @@ def save_model_batch(value: ModelBatchInput):
         return model_configs.save_many(value.kind, value.url, value.modelIds, value.apiKey.get_secret_value(),
                                        str(value.configId) if value.configId else None, connection_type=value.connectionType,
                                        protocol=value.protocol, auth_mode=value.authMode, video_adapter=value.videoAdapter,
-                                       video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None)
+                                       video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None,
+                                       input_modalities=value.inputModalities)
 
     @app.post('/api/model-configs/discover')
     async def discover_models(value: ModelDiscoveryRequest):
@@ -158,7 +159,7 @@ async def test_agent_model(value: AgentModelTest):
     def create_model_config(value: ModelConfigInput):
         return model_configs.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode,
                                   video_adapter=value.videoAdapter, video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None,
-                                  capabilities_supplied='videoCapabilities' in value.model_fields_set)
+                                  capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities)
 
     @app.post('/api/model-configs/capabilities')
     async def model_capabilities(value: ModelCapabilityRequest):
@@ -179,7 +180,7 @@ async def model_capabilities(value: ModelCapabilityRequest):
     def update_model_config(config_id: UUID, value: ModelConfigInput):
         return model_configs.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), str(config_id), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode,
                                   video_adapter=value.videoAdapter, video_capabilities=value.videoCapabilities.model_dump(exclude_none=True) if value.videoCapabilities else None,
-                                  capabilities_supplied='videoCapabilities' in value.model_fields_set)
+                                  capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities)
 
     @app.delete("/api/model-configs/{config_id}", status_code=204)
     def delete_model_config(config_id: UUID):

apps/api/src/opendirector_api/models.py

@@ -3,6 +3,7 @@
 
 from pydantic import model_validator, BaseModel, ConfigDict, Field, SecretStr
 from opendirector.domain.creation.editors import ToolDraftBinding
+from opendirector.domain.capabilities import AgentInputModality
 from opendirector.domain.video_capabilities import VideoAdapter, VideoCapabilities
 
 
@@ -25,6 +26,7 @@ class ModelConfigInput(ModelConnection):
     url: str = Field(max_length=2048)
     modelId: str = Field(max_length=200)
     apiKey: SecretStr = SecretStr("")
+    inputModalities: list[AgentInputModality] | None = Field(default=None, min_length=1, max_length=2)
     videoAdapter: VideoAdapter | None = None
     videoCapabilities: VideoCapabilities | None = None
 
@@ -54,6 +56,7 @@ class ModelConfigPublic(ModelConnection):
     providerId: UUID | None = Field(default=None, exclude_if=lambda value: value is None)
     videoAdapter: VideoAdapter = 'auto'
     videoCapabilities: VideoCapabilities | None = None
+    inputModalities: list[AgentInputModality] | None = None
     inputRules: dict = Field(default_factory=dict)
     id: UUID
     kind: Literal["agent", "image", "video", "audio", "vision"]

packages/core/src/opendirector/domain/capabilities.py

@@ -8,6 +8,8 @@
 from dataclasses import dataclass
 from typing import Literal
 
+from opendirector.domain.projects import ProjectError
+
 Capability = Literal[
     'text.chat', 'image.generate', 'image.edit', 'image.cutout', 'video.generate', 'video.first_frame', 'video.edit',
     'audio.speech', 'audio.music', 'image.segment', 'video.matting',
@@ -151,3 +153,16 @@ class ChatRequest:
 class MediaResult:
     data: bytes
     suffix: str
+
+
+AgentInputModality = Literal['text', 'image']
+
+
+def agent_input_modalities(kind, value):
+    """Declared agent inputs are separate from media-generation reference rules."""
+    if value is None:
+        return ['text'] if kind == 'agent' else None
+    if (kind != 'agent' or not isinstance(value, (list, tuple)) or 'text' not in value
+            or len(value) != len(set(value)) or any(item not in {'text', 'image'} for item in value)):
+        raise ProjectError('对话模型输入模态必须包含文本，可选图片，且不能重复。')
+    return ['text', *(['image'] if 'image' in value else [])]

packages/core/src/opendirector/domain/creation/models.py

@@ -5,6 +5,7 @@
 
 from pydantic import Field, SecretStr, field_validator, model_validator
 
+from opendirector.domain.capabilities import agent_input_modalities
 from opendirector.domain.video_capabilities import VideoCapabilities as ExistingVideoCapabilities
 from opendirector.domain.video_capabilities import VideoAdapter, check_video_config
 from opendirector.domain.projects import ProjectError
@@ -63,6 +64,7 @@ class ModelConnection(Contract):
     connectionType: Literal['api', 'self_hosted']
     protocol: Protocol
     authMode: Literal['none', 'bearer']
+    inputModalities: tuple[Literal['text', 'image'], ...] | None = None
     videoAdapter: VideoAdapter = 'auto'
     videoCapabilities: VideoCapabilities | None = None
 
@@ -96,6 +98,7 @@ def connection(self):
         if self.protocol not in allowed[self.kind]:
             raise ValueError('Protocol is incompatible with model category')
         try:
+            agent_input_modalities(self.kind, self.inputModalities)
             check_video_config(self.kind, self.protocol, self.videoAdapter,
                                self.videoCapabilities.model_dump(exclude_none=True) if self.videoCapabilities else None)
         except ProjectError as error:
@@ -129,6 +132,7 @@ class ModelSnapshot(Contract):
     url: str = Field(min_length=1, max_length=2048)
     protocol: Protocol
     connectionType: Literal['api', 'self_hosted']
+    inputModalities: tuple[Literal['text', 'image'], ...] | None = Field(default=None, exclude_if=lambda value: value is None)
     videoAdapter: VideoAdapter = 'auto'
     inputRules: InputRules | None = None
 
@@ -146,5 +150,5 @@ def model_name(cls, value):
     def connection(self):
         ModelConnection(kind=self.kind, url=self.url, modelId=self.modelId,
                         connectionType=self.connectionType, protocol=self.protocol,
-                        authMode='none', videoAdapter=self.videoAdapter)
+                        authMode='none', videoAdapter=self.videoAdapter, inputModalities=self.inputModalities)
         return self

packages/core/src/opendirector/infrastructure/model_configs.py

@@ -11,6 +11,7 @@
 
 from opendirector.domain.projects import ProjectError, identifier
 from opendirector.domain.input_rules import input_rules
+from opendirector.domain.capabilities import agent_input_modalities
 from opendirector.domain.video_capabilities import check_video_config
 from opendirector.infrastructure.local_projects import child, read_json, write_json
 
@@ -78,7 +79,7 @@ def _read(self) -> list[dict]:
                     item.update(services[item['providerId']])
             seen = set()
             for item in value["models"]:
-                if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'videoAdapter', 'videoCapabilities', 'providerId'}:
+                if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'videoAdapter', 'videoCapabilities', 'providerId', 'inputModalities'}:
                     raise ValueError()
                 identifier(item["id"])
                 if item["id"] in seen or item["kind"] not in MODEL_KINDS:
@@ -91,6 +92,7 @@ def _read(self) -> list[dict]:
                 if 'videoAdapter' not in item:
                     item['videoAdapter'] = ('minimax_h3' if item['modelId'].rsplit('/', 1)[-1].lower() == 'minimax-h3'
                                             else 'fasth3') if item['kind'] == 'video' and item['protocol'] == 'vllm_omni' else 'auto'
+                item['inputModalities'] = agent_input_modalities(item['kind'], item.get('inputModalities'))
                 item['videoCapabilities'] = check_video_config(item['kind'], item['protocol'], item['videoAdapter'], item.get('videoCapabilities'))
                 if item['authMode'] == 'bearer' and not fields['apiKey']:
                     raise ValueError()
@@ -100,7 +102,7 @@ def _read(self) -> list[dict]:
 
     @staticmethod
     def _public(item: dict) -> dict:
-        return {**({'providerId': item['providerId']} if 'providerId' in item else {}), **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities")}, 'inputRules': input_rules(item)}
+        return {**({'providerId': item['providerId']} if 'providerId' in item else {}), **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "videoAdapter", "videoCapabilities", "inputModalities")}, 'inputRules': input_rules(item)}
 
     def list(self) -> list[dict]:
         with self.lock:
@@ -117,7 +119,7 @@ def resolve(self, config_id: str, kind: str) -> dict:
                 raise ProjectError("所选模型不存在或类别不符，请在设置中配置并重新选择。", 409)
             return dict(item)
 
-    def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False) -> dict:
+    def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False, input_modalities=None) -> dict:
         if kind not in MODEL_KINDS:
             raise ProjectError("无效的模型类别。")
         fields = model_fields(url, model_id, api_key)
@@ -139,6 +141,7 @@ def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str
                     raise ProjectError('请填写 API Key，或将鉴权方式设为无需鉴权。')
                 fields['apiKey'] = existing['apiKey']
             item = {'id': config_id or str(uuid4()), 'kind': kind, **fields, **connection}
+            item['inputModalities'] = agent_input_modalities(kind, input_modalities if input_modalities is not None else (existing or {}).get('inputModalities'))
             item['videoAdapter'] = video_adapter if video_adapter is not None else (existing or {}).get('videoAdapter', 'auto')
             raw = video_capabilities if capabilities_supplied or video_capabilities is not None else (existing or {}).get('videoCapabilities')
             item['videoCapabilities'] = check_video_config(kind, connection['protocol'], item['videoAdapter'], raw)
@@ -177,7 +180,7 @@ def _write(self, models: list[dict]) -> None:
 
     def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str,
                   config_id: str | None = None, *, connection_type='api', protocol=None,
-                  auth_mode=None, video_adapter=None, video_capabilities=None) -> list[dict]:
+                  auth_mode=None, video_adapter=None, video_capabilities=None, input_modalities=None) -> list[dict]:
         if kind not in MODEL_KINDS or not 1 <= len(model_ids) <= 200:
             raise ProjectError('请选择 1—200 个模型。')
         connection = connection_fields(kind, connection_type, protocol, auth_mode)
@@ -209,6 +212,7 @@ def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str,
                     old = existing
                 item = {'id': old['id'] if old else str(uuid4()), 'kind': kind, **field, **connection,
                         'apiKey': key, 'providerId': provider_id, 'videoAdapter': video_adapter or 'auto'}
+                item['inputModalities'] = agent_input_modalities(kind, input_modalities if input_modalities is not None else (old or {}).get('inputModalities'))
                 item['videoCapabilities'] = check_video_config(kind, connection['protocol'], item['videoAdapter'], video_capabilities)
                 profile = input_rules(item)['profile']
                 if item['videoCapabilities'] is not None and profile in {'fasth3-omni', 'vllm-omni-h3'}:

src/backend/models.ts

@@ -25,6 +25,7 @@ export interface ModelConfigInput {
   connectionType: "api" | "self_hosted";
   protocol: NonNullable<ModelConfig["protocol"]>;
   authMode: "none" | "bearer";
+  inputModalities?: ("text" | "image")[] | null;
   videoAdapter?: VideoAdapter;
   videoCapabilities?: VideoCapabilities | null;
 }

src/backend/rules.ts

@@ -53,6 +53,7 @@ export interface ModelConfig {
     | null;
   authMode: "none" | "bearer" | null;
   inputRules: InputRules;
+  inputModalities?: ("text" | "image")[] | null;
   videoAdapter?: VideoAdapter;
   videoCapabilities?: VideoCapabilities | null;
 }

src/canvas/ModelServiceForm.tsx

@@ -79,6 +79,7 @@ export function ModelServiceForm({
         : (existing?.protocol ?? PROTOCOLS[kind][0][0]),
     authMode: existing?.authMode ?? "none",
     videoAdapter: existing?.videoAdapter ?? "auto",
+    inputModalities: kind === "agent" ? existing?.inputModalities ?? ["text"] : undefined,
   });
   const [selectedModels, setSelectedModels] = useState<string[]>(() =>
     existing
@@ -370,6 +371,13 @@ export function ModelServiceForm({
             </p>
           </fieldset>
         </details>
+        {kind === "agent" && <fieldset className="cw-model-modalities" disabled={locked}>
+          <legend>输入能力</legend>
+          <label><input type="checkbox" checked disabled />文本</label>
+          <label><input type="checkbox" checked={value.inputModalities?.includes("image") ?? false}
+            onChange={event => update({inputModalities: event.target.checked ? ["text", "image"] : ["text"]})} />图片</label>
+          <p className="cw-model-note">应用于本次选中的模型。仅在模型及接口支持图片理解时启用。</p>
+        </fieldset>}
         <ModelPicker
           modelPlaceholder={placeholders[1]}
           available={modelIds}

src/canvas/model-settings.css

@@ -642,3 +642,8 @@
 .cw-model-connection-details[open] > summary svg {
   transform: rotate(180deg);
 }
+
+.cw-model-modalities { border: 0; border-top: 1px solid var(--settings-border); margin: 0 0 20px; padding: 16px 0 0; display: flex; flex-wrap: wrap; gap: 12px 24px; }
+.cw-model-modalities legend { padding: 0 8px 0 0; font-size: var(--settings-label-size); }
+.cw-model-modalities label { display: flex; align-items: center; gap: 6px; font-size: var(--settings-control-size); }
+.cw-model-modalities .cw-model-note { flex-basis: 100%; margin: 0; }